From ae9bbc7131f6b4231308345be877cf0257685a38 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Thu, 30 Jul 2026 11:01:14 +0000 Subject: [PATCH 01/45] [cub] Add runtime-sized shared-memory histogram tier --- cub/cub/agent/agent_histogram.cuh | 55 ++++++++- .../device/dispatch/dispatch_histogram.cuh | 115 ++++++++++++++++-- .../dispatch/kernels/kernel_histogram.cuh | 75 ++++++++++++ cub/test/catch2_test_device_histogram.cu | 8 ++ 4 files changed, 241 insertions(+), 12 deletions(-) diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index daed613be77d..b4cc4bfd4cce 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -187,6 +187,9 @@ _CCCL_DEVICE _CCCL_FORCEINLINE auto NativePointer(IteratorT itr) //! //! @tparam OffsetT //! Signed integer type for global offsets +//! +//! @tparam UseDynamicSmemHistogram +//! Whether the privatized histogram is supplied separately in dynamic shared memory. template + typename OffsetT, + bool UseDynamicSmemHistogram = false> struct AgentHistogram { static constexpr int vec_size = AgentHistogramPolicyT::VEC_SIZE; @@ -230,10 +234,13 @@ struct AgentHistogram BlockLoad; using BlockLoadVecT = BlockLoad; + using HistogramsStorageT = ::cuda::std:: + _If; + struct _TempStorage { - // Smem needed for block-privatized smem histogram (with 1 word of padding) - CounterT histograms[NumActiveChannels][PrivatizedSmemBins + 1]; + // Static histogram storage or pointers into the separately allocated dynamic storage. + HistogramsStorageT histograms; int tile_idx; union @@ -631,6 +638,10 @@ struct AgentHistogram : // prefer gmem privatized histograms blockIdx.x & 1) // prefer blended privatized histograms { + static_assert(!UseDynamicSmemHistogram, + "AgentHistogram with UseDynamicSmemHistogram=true requires the dynamic-SMEM " + "constructor that takes an extern __shared__ base pointer."); + const int blockId = static_cast((blockIdx.y * gridDim.x) + blockIdx.x); // TODO(bgruber): d_privatized_histograms seems only used when !prefer_smem, can we skip it if prefer_smem? @@ -642,6 +653,44 @@ struct AgentHistogram } } + //! @brief Constructor for a histogram stored in dynamic shared memory + _CCCL_DEVICE _CCCL_FORCEINLINE AgentHistogram( + TempStorage& temp_storage, + SampleIteratorT d_samples, + const int* num_output_bins, + const int* num_privatized_bins, + CounterT** d_output_histograms, + CounterT** d_privatized_histograms, + const OutputDecodeOpT* output_decode_op, + const PrivatizedDecodeOpT* privatized_decode_op, + CounterT* dyn_smem_histogram_base) + : temp_storage(temp_storage.Alias()) + , d_wrapped_samples(d_samples) + , d_native_samples(NativePointer(d_wrapped_samples)) + , num_output_bins(num_output_bins) + , num_privatized_bins(num_privatized_bins) + , d_output_histograms(d_output_histograms) + , output_decode_op(output_decode_op) + , privatized_decode_op(privatized_decode_op) + , prefer_smem(true) + { + static_assert(UseDynamicSmemHistogram, + "Dynamic-SMEM AgentHistogram constructor requires UseDynamicSmemHistogram=true."); + + for (int ch = 0; ch < NumActiveChannels; ++ch) + { + this->d_privatized_histograms[ch] = d_privatized_histograms[ch]; + } + + CounterT* p = dyn_smem_histogram_base; + _CCCL_PRAGMA_UNROLL_FULL() + for (int ch = 0; ch < NumActiveChannels; ++ch) + { + this->temp_storage.histograms[ch] = p; + p += num_privatized_bins[ch]; + } + } + //! @brief Consume image //! //! @param num_row_pixels diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 673be75f6fd7..32430c77628c 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -56,6 +56,11 @@ namespace detail::histogram // Maximum number of bins per channel for which we will use a privatized smem strategy static constexpr int max_privatized_smem_bins = 256; +// Use one runtime-sized shared-memory histogram above the static tier. Keep the +// initial budget conservative so the path is portable across supported devices. +static constexpr int dynamic_smem_histogram_bytes = 32 * 1024; +static constexpr int dynamic_smem_histogram_tag = dynamic_smem_histogram_bytes / sizeof(unsigned int); + template ; } + template + _CCCL_HIDE_FROM_ABI CUB_RUNTIME_FUNCTION static constexpr auto HistogramSweepDynamicSmemKernel() + { + return &DeviceHistogramSweepDynamicSmemKernel< + PolicyT, + PRIVATIZED_SMEM_BINS, + NUM_CHANNELS, + NUM_ACTIVE_CHANNELS, + SampleIteratorT, + CounterT, + PrivatizedDecodeOpT, + OutputDecodeOpT, + OffsetT>; + } + /// Returns the device-init histogram sweep kernel that initializes decode operators from level arrays in the kernel. template (); - auto sweep_kernel = [&] { - if constexpr (IsDeviceInit) + const auto init_kernel = kernel_source.template HistogramInitKernel(); + constexpr bool use_dynamic_smem = PRIVATIZED_SMEM_BINS == dynamic_smem_histogram_tag; + auto sweep_kernel = [&] { + if constexpr (use_dynamic_smem) + { + static_assert(!IsDeviceInit, "Dynamic shared-memory histograms require host-initialized transforms"); + using output_decode_op_t = typename FirstLevelArrayT::value_type; + using privatized_decode_op_t = typename SecondLevelArrayT::value_type; + return kernel_source.template HistogramSweepDynamicSmemKernel(); + } + else if constexpr (IsDeviceInit) { return kernel_source.template HistogramSweepKernelDeviceInit< PolicySelector, @@ -235,6 +266,15 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( const int threads_per_block = active_policy.threads_per_block; const int pixels_per_thread = active_policy.pixels_per_thread; + int dynamic_smem_bytes = 0; + if constexpr (use_dynamic_smem) + { + for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel) + { + dynamic_smem_bytes += (num_privatized_levels[channel] - 1) * static_cast(kernel_source.CounterSize()); + } + } + // Get SM count int sm_count; if (const auto error = CubDebug(launcher_factory.MultiProcessorCount(sm_count))) @@ -244,8 +284,8 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( // Get SM occupancy for sweep_kernel int histogram_sweep_sm_occupancy; - if (const auto error = - CubDebug(launcher_factory.MaxSmOccupancy(histogram_sweep_sm_occupancy, sweep_kernel, threads_per_block))) + if (const auto error = CubDebug(launcher_factory.MaxSmOccupancy( + histogram_sweep_sm_occupancy, sweep_kernel, threads_per_block, dynamic_smem_bytes))) { return error; } @@ -284,7 +324,8 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL) { allocation_sizes[CHANNEL] = - size_t(num_thread_blocks) * (num_privatized_levels[CHANNEL] - 1) * kernel_source.CounterSize(); + use_dynamic_smem ? 0 + : size_t(num_thread_blocks) * (num_privatized_levels[CHANNEL] - 1) * kernel_source.CounterSize(); } allocation_sizes[NUM_ALLOCATIONS - 1] = GridQueue::AllocationSize(); @@ -353,20 +394,24 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( // Log histogram_sweep_kernel configuration #ifdef CUB_DEBUG_LOG - _CubLog("Invoking histogram_sweep_kernel<<<{%d, %d, %d}, %d, 0, %lld>>>(), %d pixels " + _CubLog("Invoking histogram_sweep_kernel<<<{%d, %d, %d}, %d, %d, %lld>>>(), %d pixels " "per thread, %d SM occupancy\n", sweep_grid_dims.x, sweep_grid_dims.y, sweep_grid_dims.z, threads_per_block, + dynamic_smem_bytes, (long long) stream, pixels_per_thread, histogram_sweep_sm_occupancy); #endif // CUB_DEBUG_LOG if (const auto error = CubDebug( - launcher_factory( - sweep_grid_dims, threads_per_block, 0, stream, /* dependent_launch */ cc >= ::cuda::compute_capability{9, 0}) + launcher_factory(sweep_grid_dims, + threads_per_block, + dynamic_smem_bytes, + stream, + /* dependent_launch */ cc >= ::cuda::compute_capability{9, 0}) .doit(sweep_kernel, d_samples, num_output_bins_wrapper, @@ -891,6 +936,32 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( } int max_num_output_bins = max_levels - 1; + if constexpr (NUM_ACTIVE_CHANNELS == 1) + { + if (max_num_output_bins > max_privatized_smem_bins + && size_t(max_num_output_bins) * kernel_source.CounterSize() <= dynamic_smem_histogram_bytes) + { + constexpr int PRIVATIZED_SMEM_BINS = dynamic_smem_histogram_tag; + return detail::histogram::dispatch( + d_temp_storage, + temp_storage_bytes, + d_samples, + d_output_histograms, + num_output_levels, + num_output_levels, + output_decode_op, + privatized_decode_op, + max_num_output_bins, + num_row_pixels, + num_rows, + row_stride_samples, + stream, + policy_selector, + kernel_source, + launcher_factory); + } + } + // Dispatch if (max_num_output_bins > max_privatized_smem_bins) { @@ -1097,6 +1168,32 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t dispatch_even( } int max_num_output_bins = max_levels - 1; + if constexpr (NUM_ACTIVE_CHANNELS == 1) + { + if (max_num_output_bins > max_privatized_smem_bins + && size_t(max_num_output_bins) * kernel_source.CounterSize() <= dynamic_smem_histogram_bytes) + { + constexpr int PRIVATIZED_SMEM_BINS = dynamic_smem_histogram_tag; + return detail::histogram::dispatch( + d_temp_storage, + temp_storage_bytes, + d_samples, + d_output_histograms, + num_output_levels, + num_output_levels, + output_decode_op, + privatized_decode_op, + max_num_output_bins, + num_row_pixels, + num_rows, + row_stride_samples, + stream, + policy_selector, + kernel_source, + launcher_factory); + } + } + if (max_num_output_bins > max_privatized_smem_bins) { constexpr int PRIVATIZED_SMEM_BINS = 0; diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index cc64ddea1862..a55474191250 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -518,6 +518,81 @@ __launch_bounds__(int(current_policy().threads_per_block)) agent.StoreOutput(); } +//! Histogram sweep kernel with the privatized histogram in dynamic shared memory. +//! +//! The host supplies `num_privatized_bins * sizeof(CounterT)` bytes of dynamic +//! shared memory. Keeping the runtime-sized histogram outside `TempStorage` +//! allows one kernel instantiation to cover larger histograms without a ladder +//! of statically sized kernels. +template +#if _CCCL_HAS_CONCEPTS() + requires histogram_policy_selector +#endif // _CCCL_HAS_CONCEPTS() +__launch_bounds__(int(current_policy().threads_per_block)) + _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDynamicSmemKernel( + const SampleIteratorT d_samples, + const ::cuda::std::array num_output_bins_wrapper, + const ::cuda::std::array num_privatized_bins_wrapper, + ::cuda::std::array d_output_histograms_wrapper, + ::cuda::std::array d_privatized_histograms_wrapper, + const ::cuda::std::array output_decode_op_wrapper, + const ::cuda::std::array privatized_decode_op_wrapper, + const OffsetT num_row_pixels, + const OffsetT num_rows, + const OffsetT row_stride_samples, + const int tiles_per_row, + GridQueue tile_queue) +{ + static constexpr HistogramPolicy hp = current_policy(); + + using AgentHistogramPolicyT = agent_histogram_policy< + hp.threads_per_block, + hp.pixels_per_thread, + hp.load_algorithm, + hp.load_modifier, + hp.rle_compress, + hp.mem_preference, + hp.use_work_stealing, + hp.vec_size>; + using AgentHistogramT = + AgentHistogram; + + __shared__ typename AgentHistogramT::TempStorage temp_storage; + extern __shared__ unsigned char dynamic_smem[]; + + AgentHistogramT agent( + temp_storage, + d_samples, + num_output_bins_wrapper.data(), + num_privatized_bins_wrapper.data(), + d_output_histograms_wrapper.data(), + d_privatized_histograms_wrapper.data(), + output_decode_op_wrapper.data(), + privatized_decode_op_wrapper.data(), + reinterpret_cast(dynamic_smem)); + + agent.InitBinCounters(); + agent.ConsumeTiles(num_row_pixels, num_rows, row_stride_samples, tiles_per_row, tile_queue); + agent.StoreOutput(); +} + //! Histogram privatized sweep kernel entry point (multi-block) with device-side initialization. //! Computes privatized histograms, one per thread block. //! This kernel initializes decode operators from level arrays inside the kernel. diff --git a/cub/test/catch2_test_device_histogram.cu b/cub/test/catch2_test_device_histogram.cu index 6cdf13cb33a5..c345d2016e4a 100644 --- a/cub/test/catch2_test_device_histogram.cu +++ b/cub/test/catch2_test_device_histogram.cu @@ -574,6 +574,14 @@ CUB_TEST_LIST("DeviceHistogram::Histogram* channel configs", test_even_and_range(256, 256 + 1, 128, 32); } +C2H_TEST("DeviceHistogram::Histogram* dynamic shared-memory privatization", "[histogram][device]") +{ + using counter_t = unsigned long long; + const int num_levels = GENERATE(1025, 4097); + + test_even_and_range(num_levels - 1, num_levels, 4096, 4); +} + // Testing only HistogramEven is fine, because HistogramRange shares the loading logic and the different binning // implementations are not affected by the iterator. CUB_TEST("DeviceHistogram::HistogramEven sample iterator", "[histogram_even][device]", CUB_SMALL) From 6ae878e470e8524463bf11bac638d429cd48bd8d Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Thu, 30 Jul 2026 13:57:51 +0000 Subject: [PATCH 02/45] [cub] Complete SMEM-privatized histogram tuning --- cub/cub/agent/agent_histogram.cuh | 32 +- .../device/dispatch/dispatch_histogram.cuh | 78 ++- .../dispatch/kernels/kernel_histogram.cuh | 576 +++++++++++++++++- .../dispatch/tuning/tuning_histogram.cuh | 121 +++- cub/test/catch2_test_device_histogram_env.cu | 33 +- 5 files changed, 797 insertions(+), 43 deletions(-) diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index b4cc4bfd4cce..ac2cc13810a0 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -324,11 +324,35 @@ struct AgentHistogram // Bin pixels int bins[pixels_per_thread]; - _CCCL_PRAGMA_UNROLL_FULL() - for (int pixel = 0; pixel < pixels_per_thread; ++pixel) + if constexpr (UseDynamicSmemHistogram) + { + typename PrivatizedDecodeOpT::BracketCacheT mru; + _CCCL_PRAGMA_UNROLL_FULL() + for (int pixel = 0; pixel < pixels_per_thread; ++pixel) + { + bins[pixel] = -1; + privatized_decode_op[ch].template BinSelect( + samples[pixel][ch], bins[pixel], is_valid[pixel], mru); + } + } + else if constexpr (PrivatizedSmemBins > 0 && PrivatizedDecodeOpT::is_range_transform) + { + _CCCL_PRAGMA_UNROLL_FULL() + for (int pixel = 0; pixel < pixels_per_thread; ++pixel) + { + bins[pixel] = -1; + privatized_decode_op[ch].template BinSelectStaticLean( + samples[pixel][ch], bins[pixel], is_valid[pixel]); + } + } + else { - bins[pixel] = -1; - privatized_decode_op[ch].template BinSelect(samples[pixel][ch], bins[pixel], is_valid[pixel]); + _CCCL_PRAGMA_UNROLL_FULL() + for (int pixel = 0; pixel < pixels_per_thread; ++pixel) + { + bins[pixel] = -1; + privatized_decode_op[ch].template BinSelect(samples[pixel][ch], bins[pixel], is_valid[pixel]); + } } CounterT accumulator = 1; diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 32430c77628c..b7b398ee199c 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -56,10 +56,12 @@ namespace detail::histogram // Maximum number of bins per channel for which we will use a privatized smem strategy static constexpr int max_privatized_smem_bins = 256; -// Use one runtime-sized shared-memory histogram above the static tier. Keep the -// initial budget conservative so the path is portable across supported devices. -static constexpr int dynamic_smem_histogram_bytes = 32 * 1024; -static constexpr int dynamic_smem_histogram_tag = dynamic_smem_histogram_bytes / sizeof(unsigned int); +// Compile-time tag selecting the runtime-sized shared-memory kernel. The tag +// does not size storage; the actual byte budget comes from HistogramPolicy. +static constexpr int dynamic_smem_histogram_tag = 16384; + +static constexpr int multi_channel_dynamic_smem_bins_range = 2048; +static constexpr int multi_channel_dynamic_smem_bins_even = 8192; template 0 && !use_dynamic_smem; + const int threads_per_block = use_static_smem ? active_policy.static_smem_threads() : active_policy.threads_per_block; + const int pixels_per_thread = use_static_smem ? active_policy.static_smem_items() : active_policy.pixels_per_thread; int dynamic_smem_bytes = 0; if constexpr (use_dynamic_smem) @@ -273,6 +276,13 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( { dynamic_smem_bytes += (num_privatized_levels[channel] - 1) * static_cast(kernel_source.CounterSize()); } + NV_IF_TARGET(NV_IS_HOST, ({ + if (const auto error = + CubDebug(launcher_factory.set_max_dynamic_smem_size_for(sweep_kernel, dynamic_smem_bytes))) + { + return error; + } + })) } // Get SM count @@ -779,6 +789,20 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_pdl_trigger(long) return 0; } +// TODO(bgruber): drop in CCCL 4.0 +template +_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_bytes(int) -> decltype(ActivePolicy::dynamic_smem_bytes) +{ + return ActivePolicy::dynamic_smem_bytes; +} + +// TODO(bgruber): drop in CCCL 4.0 +template +_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_bytes(long) +{ + return 0; +} + // TODO(bgruber): drop in CCCL 4.0 template _CCCL_HOST_DEVICE_API constexpr auto convert_policy() -> HistogramPolicy @@ -793,7 +817,8 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_policy() -> HistogramPolicy ap::IS_RLE_COMPRESS, ap::MEM_PREFERENCE, ap::IS_WORK_STEALING, - convert_pdl_trigger(0)}; + convert_pdl_trigger(0), + convert_dynamic_smem_bytes(0)}; } // TODO(bgruber): drop in CCCL 4.0 @@ -936,10 +961,22 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( } int max_num_output_bins = max_levels - 1; - if constexpr (NUM_ACTIVE_CHANNELS == 1) + ::cuda::compute_capability cc{}; + if (const auto error = CubDebug(launcher_factory.PtxComputeCap(cc))) + { + return error; + } + const HistogramPolicy active_policy = policy_selector(cc); + + if constexpr (NUM_ACTIVE_CHANNELS >= 1) { - if (max_num_output_bins > max_privatized_smem_bins - && size_t(max_num_output_bins) * kernel_source.CounterSize() <= dynamic_smem_histogram_bytes) + const bool within_tuned_channel_cap = + NUM_ACTIVE_CHANNELS == 1 || max_num_output_bins <= multi_channel_dynamic_smem_bins_range; + const size_t dynamic_smem_bytes = size_t(max_num_output_bins) * NUM_ACTIVE_CHANNELS * kernel_source.CounterSize(); + const bool prefer_dynamic_smem = + kernel_source.CounterSize() > sizeof(unsigned int) || max_num_output_bins > max_privatized_smem_bins; + if (prefer_dynamic_smem && within_tuned_channel_cap + && dynamic_smem_bytes <= static_cast(active_policy.dynamic_smem_bytes)) { constexpr int PRIVATIZED_SMEM_BINS = dynamic_smem_histogram_tag; return detail::histogram::dispatch( @@ -1168,10 +1205,25 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t dispatch_even( } int max_num_output_bins = max_levels - 1; - if constexpr (NUM_ACTIVE_CHANNELS == 1) + ::cuda::compute_capability cc{}; + if (const auto error = CubDebug(launcher_factory.PtxComputeCap(cc))) + { + return error; + } + const HistogramPolicy active_policy = policy_selector(cc); + + if constexpr (NUM_ACTIVE_CHANNELS >= 1) { - if (max_num_output_bins > max_privatized_smem_bins - && size_t(max_num_output_bins) * kernel_source.CounterSize() <= dynamic_smem_histogram_bytes) + const bool within_tuned_channel_cap = + NUM_ACTIVE_CHANNELS == 1 || max_num_output_bins <= multi_channel_dynamic_smem_bins_even + || (NUM_ACTIVE_CHANNELS <= 3 + && size_t(max_num_output_bins) * NUM_ACTIVE_CHANNELS * kernel_source.CounterSize() + <= static_cast(active_policy.dynamic_smem_bytes)); + const size_t dynamic_smem_bytes = size_t(max_num_output_bins) * NUM_ACTIVE_CHANNELS * kernel_source.CounterSize(); + const bool prefer_dynamic_smem = + kernel_source.CounterSize() > sizeof(unsigned int) || max_num_output_bins > max_privatized_smem_bins; + if (prefer_dynamic_smem && within_tuned_channel_cap + && dynamic_smem_bytes <= static_cast(active_policy.dynamic_smem_bytes)) { constexpr int PRIVATIZED_SMEM_BINS = dynamic_smem_histogram_tag; return detail::histogram::dispatch( diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index a55474191250..73855cf1dedb 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -35,9 +35,78 @@ struct Transforms template struct SearchTransform { + // Compile-time RANGE marker, resolved at instantiation (no runtime branch). + // The direct-atomic kernels read this to specialize behavior that only helps + // the RANGE (SearchTransform) classify, e.g. the per-thread bracket cache. + static constexpr bool is_range_transform = true; + + // Below this bin count, BinSelect skips the interpolated-first-guess machinery + // (and PrecomputeOnDevice stays disabled) and uses the lean UpperBound binary + // search instead. The interpolation path carries ~7 extra per-thread registers + // (endpoints + slopes + 3-point split state) plus a device precompute prologue; + // on the small-bin tiers -- especially the static 256-bin privatized-SMEM kernel, + // which is occupancy/register-bound -- that overhead is a net LOSS. Measured on + // B200: at 256 bins interpolation runs ~0.67x of plain binary search, but by + // 1024 bins it already wins ~2.3x and keeps growing, so the cutoff sits between. + // (Was hard-coded `< 4`, which only skipped the degenerate tiny case and let the + // 256-bin RANGE/I32 tier regress vs upstream.) + static constexpr int kInterpolationMinBins = 512; + + //! @brief Per-thread most-recently-used (MRU) bin-bracket cache. + //! + //! Carries the last successfully-resolved bin and its two boundary level + //! values across consecutive `BinSelect` calls so a new sample that falls in + //! the same `[lo, hi)` bracket is classified with ZERO level-array loads (a + //! handful of register compares), skipping the interpolated first-guess, the + //! clamp, and -- crucially -- both verify loads on the dependent + //! `IMAD.WIDE -> LDG` level-load chain that binds the latency-bound RANGE + //! classify. Low-entropy inputs (constant or heavily-skewed samples) have high + //! consecutive-sample locality, so the bracket hits dominate. A `bin < 0` + //! sentinel marks the cache empty. + //! This is per-thread mutable state, so it is only sound on a per-thread + //! `SearchTransform` copy (the direct-atomic cuckoo/single-probe kernels' + //! `decode_op[ch]`), never on the shared `__grid_constant__` decode op that + //! the SMEM-privatized agent path reads through a const pointer. + struct BracketCacheT + { + LevelT lo; // cached d_levels[bin] + LevelT hi; // cached d_levels[bin + 1] + int bin = -1; // cached bin; < 0 means empty + }; + LevelIteratorT d_levels; // Pointer to levels array int num_output_levels; // Number of levels in array + // Precomputed (loop-invariant) interpolation state, populated by + // `PrecomputeOnDevice()`. The interpolation slope `num_bins / (last - + // first)` and the boundary levels are uniform across all samples a thread + // classifies, but the original `BinSelect` recomputed them per sample + // (two cache loads for the endpoints plus a `__fdividef` MUFU.RCP on the + // critical dependency chain). Hoisting them out turns the per-sample + // first-guess into a single `(float)delta * m_inv_scale` FMA and removes + // the two endpoint loads, which is the dominant cost on the ALU/XU-bound + // RANGE classify. `m_have_precompute == false` keeps the original + // per-sample path so host-only initialization (no device pointer to + // dereference) and tiny bin counts remain correct. + float m_inv_scale; // num_bins / (float)(last - first); valid iff m_have_precompute + LevelT m_first; // cached d_levels[0] + LevelT m_last; // cached d_levels[num_bins] + bool m_have_precompute; // whether the fields above are valid + + // Three-point (piecewise-linear) interpolation state, populated by + // PrecomputeOnDevice alongside the single-secant fields above. Splitting + // the [first,last] range at the midpoint level d_levels[mid] and + // interpolating on whichever half the sample falls in (a) halves the + // magnitude of `delta` fed to the lossy 32-bit float guess -- so the + // first-guess lands closer to the true bin and the verify-or-1-step ladder + // converges without reaching UpperBound -- and (b) captures large-scale + // non-uniformity (a slope change between the two halves) that a single + // first->last secant cannot. `m_mid_bin` is the bin index at the split. + LevelT m_mid; // cached d_levels[mid_bin] + float m_inv_scale_lo; // mid_bin / (float)(mid - first) + float m_inv_scale_hi; // (num_bins - mid_bin) / (float)(last - mid) + int m_mid_bin; // split bin index (num_bins / 2) + //! @brief Initializer //! //! @param d_levels_ Pointer to levels array @@ -46,6 +115,67 @@ struct Transforms { this->d_levels = d_levels_; this->num_output_levels = num_output_levels_; + this->m_have_precompute = false; + this->m_inv_scale = 0.0f; + } + + //! @brief Hoist the loop-invariant interpolation state out of `BinSelect`. + //! + //! Must be called on the device (it dereferences the device level array) + //! once per thread before the sweep loop. Reads the first and last level, + //! validates strict monotonicity of the endpoints and a usable bin count, + //! and on success precomputes the float reciprocal slope so the hot path + //! avoids a per-sample `__fdividef` and two endpoint loads. On any + //! degenerate input it leaves `m_have_precompute == false`, so `BinSelect` + //! transparently falls back to the original (fully general) path. + _CCCL_DEVICE _CCCL_FORCEINLINE void PrecomputeOnDevice() + { + const int num_bins = num_output_levels - 1; + if (num_bins < kInterpolationMinBins) + { + m_have_precompute = false; + return; + } + + using WrappedLevelIteratorT = + ::cuda::std::_If<::cuda::std::is_pointer_v, + CacheModifiedInputIterator, + LevelIteratorT>; + WrappedLevelIteratorT wrapped_levels(d_levels); + + const LevelT first = wrapped_levels[0]; + const LevelT last = wrapped_levels[num_bins]; + if (!(first < last)) + { + m_have_precompute = false; + return; + } + + m_first = first; + m_last = last; + m_inv_scale = static_cast(num_bins) / static_cast(last - first); + m_have_precompute = true; + + // Three-point split at the midpoint bin. Read d_levels[mid] and derive + // the two half-slopes. If either half is degenerate (non-increasing), + // fall back to the single-secant guess by setting m_mid_bin = 0, which + // BinSelect treats as "no split". + m_mid_bin = 0; + m_inv_scale_lo = m_inv_scale; + m_inv_scale_hi = m_inv_scale; + m_mid = first; + const int mid_bin = num_bins >> 1; + if (mid_bin > 0 && mid_bin < num_bins) + { + const LevelT mid = wrapped_levels[mid_bin]; + if ((first < mid) && (mid < last)) + { + m_mid = mid; + m_mid_bin = mid_bin; + m_inv_scale_lo = static_cast(mid_bin) / static_cast(mid - first); + m_inv_scale_hi = static_cast(num_bins - mid_bin) / static_cast(last - mid); + } + } } // Method for converting samples to bin-ids @@ -62,6 +192,199 @@ struct Transforms WrappedLevelIteratorT wrapped_levels(d_levels); + const int num_bins = num_output_levels - 1; + if (!valid) + { + return; + } + + const LevelT s = static_cast(sample); + + // For very small bin counts, the interpolation overhead is not worth + // it; fall back to the original binary search. + if (num_bins < kInterpolationMinBins) + { + bin = UpperBound(wrapped_levels, num_output_levels, s) - 1; + if (bin >= num_bins) + { + bin = -1; + } + return; + } + + // Read first and last levels. When `PrecomputeOnDevice()` has run we use + // the cached endpoints (and the precomputed reciprocal slope below), + // removing two per-sample endpoint loads and the per-sample + // `__fdividef`. Otherwise (host-only init, or a degenerate level array + // that PrecomputeOnDevice rejected) we read them per sample as before. + // These are warp/CTA-uniform and land in L1 / texture cache after the + // first read, so even the fallback amortizes across samples. + const LevelT first_level = m_have_precompute ? m_first : wrapped_levels[0]; + const LevelT last_level = m_have_precompute ? m_last : wrapped_levels[num_bins]; + + // Defensive: if a user-supplied level array has non-monotonic endpoints + // (e.g. `last_level <= first_level`), the boundary check below would + // misclassify all samples as out-of-range. Fall back to UpperBound, + // which uses ordered comparisons only and produces correct results + // regardless of endpoint ordering. (PrecomputeOnDevice already enforces + // `first < last` before setting m_have_precompute, so this only fires on + // the non-precomputed path.) + if (!(first_level < last_level)) + { + bin = UpperBound(wrapped_levels, num_output_levels, s) - 1; + if (bin >= num_bins) + { + bin = -1; + } + return; + } + + // Out-of-range samples map to bin -1. + if (s < first_level || !(s < last_level)) + { + bin = -1; + return; + } + + // Interpolated first-guess index. We always use a fast 32-bit float + // divide (MUFU.RCP) for the slope: the divide does not have to be + // accurate, only close enough that the verify-or-1-step-correct path + // hits a handful of bins. The full UpperBound fallback catches any + // remaining mismatch from precision loss or non-uniform spacing. + // For wide-ranged 64-bit types we still compute (sample - first) in + // the level type to avoid float overflow on the difference itself. + // + // On the precomputed path the slope `num_bins / (last - first)` is a + // loop-invariant `m_inv_scale`, so the guess collapses to a single + // `(float)delta * m_inv_scale` FMA (no per-sample MUFU.RCP). The result + // is bit-identical in intent to `__fdividef(delta*num_bins, range)`: + // both are approximate first guesses validated by the bracket check + // below, so any rounding difference is absorbed by the same verify / + // 1-step / UpperBound correction ladder. + const auto delta = (s - first_level); + int guess; + if (m_have_precompute) + { + // Three-point piecewise-linear first guess: interpolate on whichever + // half of [first, last] the sample falls in (split at the cached + // midpoint level m_mid / m_mid_bin). Using a local slope and a smaller + // delta magnitude lands the guess closer to the true bin than a single + // first->last secant, so the verify-or-1-step ladder converges without + // reaching the UpperBound binary search. m_mid_bin == 0 means the split + // was degenerate, so we use the single-secant guess. + if (m_mid_bin > 0) + { + if (s < m_mid) + { + guess = static_cast(static_cast(delta) * m_inv_scale_lo); + } + else + { + const auto delta_hi = (s - m_mid); + guess = m_mid_bin + static_cast(static_cast(delta_hi) * m_inv_scale_hi); + } + } + else + { + guess = static_cast(static_cast(delta) * m_inv_scale); + } + } + else + { + const auto range = (last_level - first_level); + NV_IF_ELSE_TARGET( + NV_IS_DEVICE, + (guess = static_cast( + __fdividef(static_cast(delta) * static_cast(num_bins), static_cast(range)));), + (guess = static_cast( + (static_cast(delta) * static_cast(num_bins)) / static_cast(range));)); + } + if (guess < 0) + { + guess = 0; + } + else if (guess > num_bins - 1) + { + guess = num_bins - 1; + } + + // Verify the guess: d_levels[guess] <= s < d_levels[guess + 1]. We + // load both bracketing levels in parallel to expose memory-level + // parallelism and branch on the result. The level array has length + // num_bins + 1, so wrapped_levels[guess + 1] is always in-bounds for + // guess <= num_bins - 1. + const LevelT lvl_lo = wrapped_levels[guess]; + const LevelT lvl_hi = wrapped_levels[guess + 1]; + + if (!(s < lvl_lo) && (s < lvl_hi)) + { + bin = guess; + return; + } + + // One-step linear correction: try a single neighbor before falling + // back to a binary search. If the guess was high, try guess - 1; if + // low, try guess + 1. + if (s < lvl_lo) + { + // guess too high; check guess - 1. + const int g2 = guess - 1; + if (g2 >= 0) + { + const LevelT lvl2_lo = wrapped_levels[g2]; + // lvl2_hi is lvl_lo (loaded already). + if (!(s < lvl2_lo)) + { + bin = g2; + return; + } + } + } + else + { + // s >= lvl_hi: guess too low; check guess + 1. + const int g2 = guess + 1; + if (g2 <= num_bins - 1) + { + // lvl2_lo is lvl_hi (loaded already). + const LevelT lvl2_hi = wrapped_levels[g2 + 1]; + if (s < lvl2_hi) + { + bin = g2; + return; + } + } + } + + // Fall back to binary search for irregular level distributions. + bin = UpperBound(wrapped_levels, num_output_levels, s) - 1; + if (bin >= num_bins) + { + bin = -1; + } + } + + //! @brief Lean classify for the STATIC <=256-bin SMEM tier. + //! + //! Byte-identical to upstream `main`'s flat `BinSelect`: a single `UpperBound` + //! + clamp, with NONE of the interpolation machinery the 3-arg `BinSelect` above + //! carries (no `num_bins < kInterpolationMinBins` runtime branch, no reads of the + //! precompute fields `m_inv_scale`/`m_first`/...). At <=256 bins the interpolation + //! fast-path never activates (`m_have_precompute` stays false there), so that + //! machinery is pure dead weight: its extra branch + the wider codegen/register + //! footprint measurably slow the latency/occupancy-bound static kernel (~1-3% vs + //! main, confirmed by A/B). The dynamic-SMEM tier (bins >= 512) keeps the full + //! interpolated `BinSelect`/MRU path, where that machinery pays off. EVEN's + //! ScaleTransform is unaffected (it has its own cheap classify). + template + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelectStaticLean(_SampleT sample, int& bin, bool valid) const + { + using WrappedLevelIteratorT = + ::cuda::std::_If<::cuda::std::is_pointer_v, + CacheModifiedInputIterator, + LevelIteratorT>; + WrappedLevelIteratorT wrapped_levels(d_levels); + const int num_bins = num_output_levels - 1; if (valid) { @@ -72,11 +395,193 @@ struct Transforms } } } + + //! @brief MRU-bracket-cached `BinSelect`. + //! + //! Same contract and result as the plain `BinSelect` above, but threads a + //! per-thread `BracketCacheT` across calls to exploit consecutive-sample + //! temporal locality. The fast path tests the cached `[lo, hi)` bracket with + //! register compares only -- on a hit it returns the cached bin without ANY + //! level-array load, cutting the dependent `IMAD.WIDE -> LDG` chain that + //! binds the high-bin RANGE classify. On a miss it runs the identical + //! interpolated-guess / verify / 1-step / `UpperBound` ladder as the plain + //! path (so correctness, including the non-uniform-level fallback, is + //! unchanged) and then records the resolved bracket -- reusing the bracket + //! levels the ladder already loaded in the common verify/1-step cases, and + //! reloading only on the rare `UpperBound` fallback. + template + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid, BracketCacheT& mru) const + { + using WrappedLevelIteratorT = + ::cuda::std::_If<::cuda::std::is_pointer_v, + CacheModifiedInputIterator, + LevelIteratorT>; + + const int num_bins = num_output_levels - 1; + if (!valid) + { + return; + } + + const LevelT s = static_cast(sample); + + // Fast path: the cached bracket holds the answer with no level loads. + // `mru.bin >= 0` guarantees the bracket is populated and in-range. + if (mru.bin >= 0 && !(s < mru.lo) && (s < mru.hi)) + { + bin = mru.bin; + return; + } + + // Tiny bin counts: the interpolation/bracket machinery is not worth it. + if (num_bins < kInterpolationMinBins) + { + WrappedLevelIteratorT wrapped_levels(d_levels); + bin = UpperBound(wrapped_levels, num_output_levels, s) - 1; + if (bin >= num_bins) + { + bin = -1; + } + return; + } + + WrappedLevelIteratorT wrapped_levels(d_levels); + + const LevelT first_level = m_have_precompute ? m_first : wrapped_levels[0]; + const LevelT last_level = m_have_precompute ? m_last : wrapped_levels[num_bins]; + + if (!(first_level < last_level)) + { + bin = UpperBound(wrapped_levels, num_output_levels, s) - 1; + if (bin >= num_bins) + { + bin = -1; + } + return; + } + + // Out-of-range samples map to bin -1 (and do not update the cache). + if (s < first_level || !(s < last_level)) + { + bin = -1; + return; + } + + // Identical first-guess ladder to the plain BinSelect above: on a cache + // MISS we reproduce the same three-point piecewise-linear first guess so + // miss-heavy inputs (high-entropy samples, and the irregular-level + // fallback) converge exactly as the uncached path does. Only the hit fast + // path and the cache writebacks differ. + const auto delta = (s - first_level); + int guess; + if (m_have_precompute) + { + if (m_mid_bin > 0) + { + if (s < m_mid) + { + guess = static_cast(static_cast(delta) * m_inv_scale_lo); + } + else + { + const auto delta_hi = (s - m_mid); + guess = m_mid_bin + static_cast(static_cast(delta_hi) * m_inv_scale_hi); + } + } + else + { + guess = static_cast(static_cast(delta) * m_inv_scale); + } + } + else + { + const auto range = (last_level - first_level); + NV_IF_ELSE_TARGET( + NV_IS_DEVICE, + (guess = static_cast( + __fdividef(static_cast(delta) * static_cast(num_bins), static_cast(range)));), + (guess = static_cast( + (static_cast(delta) * static_cast(num_bins)) / static_cast(range));)); + } + if (guess < 0) + { + guess = 0; + } + else if (guess > num_bins - 1) + { + guess = num_bins - 1; + } + + const LevelT lvl_lo = wrapped_levels[guess]; + const LevelT lvl_hi = wrapped_levels[guess + 1]; + + if (!(s < lvl_lo) && (s < lvl_hi)) + { + bin = guess; + mru.lo = lvl_lo; + mru.hi = lvl_hi; + mru.bin = guess; + return; + } + + // One-step linear correction. + if (s < lvl_lo) + { + const int g2 = guess - 1; + if (g2 >= 0) + { + const LevelT lvl2_lo = wrapped_levels[g2]; + if (!(s < lvl2_lo)) + { + bin = g2; + mru.lo = lvl2_lo; + mru.hi = lvl_lo; // lvl2_hi == lvl_lo (already loaded) + mru.bin = g2; + return; + } + } + } + else + { + const int g2 = guess + 1; + if (g2 <= num_bins - 1) + { + const LevelT lvl2_hi = wrapped_levels[g2 + 1]; + if (s < lvl2_hi) + { + bin = g2; + mru.lo = lvl_hi; // lvl2_lo == lvl_hi (already loaded) + mru.hi = lvl2_hi; + mru.bin = g2; + return; + } + } + } + + // Fall back to binary search for irregular level distributions. This is + // the rare path, so the two extra bracket loads needed to refresh the MRU + // cache are amortized; they keep subsequent in-bracket samples on the + // zero-load fast path. + bin = UpperBound(wrapped_levels, num_output_levels, s) - 1; + if (bin >= num_bins) + { + bin = -1; + return; + } + if (bin >= 0) + { + mru.lo = wrapped_levels[bin]; + mru.hi = wrapped_levels[bin + 1]; + mru.bin = bin; + } + } }; // Scales samples to evenly-spaced bins struct ScaleTransform { + static constexpr bool is_range_transform = false; + using CommonT = ::cuda::std::common_type_t; static_assert(::cuda::std::is_convertible_v, "The common type of `LevelT` and `SampleT` must be " @@ -265,6 +770,11 @@ struct Transforms m_scale = this->ComputeScale(num_levels, m_max, m_min); } + _CCCL_DEVICE _CCCL_FORCEINLINE void PrecomputeOnDevice() {} + + struct BracketCacheT + {}; + // Method for converting samples to bin-ids template _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(SampleT sample, int& bin, bool valid) const @@ -276,11 +786,19 @@ struct Transforms bin = this->ComputeBin(common_sample, m_min, m_scale); } } + + template + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(SampleT sample, int& bin, bool valid, BracketCacheT&) const + { + this->template BinSelect(sample, bin, valid); + } }; // Pass-through bin transform operator struct PassThruTransform { + static constexpr bool is_range_transform = false; + // GCC 14 rightfully warns that when a value-initialized array of this struct is copied using memcpy, uninitialized // bytes may be accessed. To avoid this, we add a dummy member, so value initialization actually initializes the memory. #if _CCCL_COMPILER(GCC, >=, 13) @@ -297,6 +815,11 @@ struct Transforms _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void Init(T, int) {} + _CCCL_DEVICE _CCCL_FORCEINLINE void PrecomputeOnDevice() {} + + struct BracketCacheT + {}; + // Method for converting samples to bin-ids template _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid) const @@ -306,6 +829,12 @@ struct Transforms bin = static_cast(sample); } } + + template + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid, BracketCacheT&) const + { + this->template BinSelect(sample, bin, valid); + } }; }; @@ -457,7 +986,17 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(current_policy().threads_per_block)) +__launch_bounds__( + int(PrivatizedSmemBins > 0 ? current_policy().static_smem_threads() + : current_policy().threads_per_block), + (PrivatizedSmemBins > 0 && PrivatizedDecodeOpT::is_range_transform + && current_policy().static_smem_threads() < 512) + ? 3 + : (((PrivatizedSmemBins > 0 ? current_policy().static_smem_threads() + : current_policy().threads_per_block) + >= 512) + ? 2 + : 0)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepKernel( const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, @@ -475,15 +1014,17 @@ __launch_bounds__(int(current_policy().threads_per_block)) static constexpr HistogramPolicy hp = current_policy(); // Thread block type for compositing input tiles - using AgentHistogramPolicyT = agent_histogram_policy< - hp.threads_per_block, - hp.pixels_per_thread, - hp.load_algorithm, - hp.load_modifier, - hp.rle_compress, - hp.mem_preference, - hp.use_work_stealing, - hp.vec_size>; + static constexpr int sweep_threads = PrivatizedSmemBins > 0 ? hp.static_smem_threads() : hp.threads_per_block; + static constexpr int sweep_items = PrivatizedSmemBins > 0 ? hp.static_smem_items() : hp.pixels_per_thread; + using AgentHistogramPolicyT = agent_histogram_policy< + sweep_threads, + sweep_items, + hp.load_algorithm, + hp.load_modifier, + hp.rle_compress, + hp.mem_preference, + hp.use_work_stealing, + hp.vec_size>; using AgentHistogramT = AgentHistogram().threads_per_block)) __shared__ typename AgentHistogramT::TempStorage temp_storage; extern __shared__ unsigned char dynamic_smem[]; + OutputDecodeOpT output_decode_op[NumActiveChannels]; + PrivatizedDecodeOpT privatized_decode_op[NumActiveChannels]; + _CCCL_PRAGMA_UNROLL_FULL() + for (int channel = 0; channel < NumActiveChannels; ++channel) + { + output_decode_op[channel] = output_decode_op_wrapper[channel]; + privatized_decode_op[channel] = privatized_decode_op_wrapper[channel]; + output_decode_op[channel].PrecomputeOnDevice(); + privatized_decode_op[channel].PrecomputeOnDevice(); + } + AgentHistogramT agent( temp_storage, d_samples, @@ -584,8 +1136,8 @@ __launch_bounds__(int(current_policy().threads_per_block)) num_privatized_bins_wrapper.data(), d_output_histograms_wrapper.data(), d_privatized_histograms_wrapper.data(), - output_decode_op_wrapper.data(), - privatized_decode_op_wrapper.data(), + output_decode_op, + privatized_decode_op, reinterpret_cast(dynamic_smem)); agent.InitBinCounters(); diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 2084666a5af3..16dfe0ea753a 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -39,6 +39,19 @@ struct HistogramPolicy bool use_work_stealing; //!< Whether to dequeue tiles from a global work queue int init_kernel_pdl_trigger_max_bins; //!< Maximum number of bins for the init kernel to trigger the histogram kernel //!< early using PDL + int dynamic_smem_bytes = 0; //!< Tuned byte budget for a runtime-sized privatized histogram; 0 disables it + int static_smem_threads_per_block = 0; //!< Static shared-memory tier threads; 0 inherits threads_per_block + int static_smem_items_per_thread = 0; //!< Static shared-memory tier items; 0 inherits pixels_per_thread + + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int static_smem_threads() const + { + return static_smem_threads_per_block != 0 ? static_smem_threads_per_block : threads_per_block; + } + + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int static_smem_items() const + { + return static_smem_items_per_thread != 0 ? static_smem_items_per_thread : pixels_per_thread; + } [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(const HistogramPolicy& lhs, const HistogramPolicy& rhs) noexcept @@ -47,7 +60,10 @@ struct HistogramPolicy && lhs.vec_size == rhs.vec_size && lhs.load_algorithm == rhs.load_algorithm && lhs.load_modifier == rhs.load_modifier && lhs.rle_compress == rhs.rle_compress && lhs.mem_preference == rhs.mem_preference && lhs.use_work_stealing == rhs.use_work_stealing - && lhs.init_kernel_pdl_trigger_max_bins == rhs.init_kernel_pdl_trigger_max_bins; + && lhs.init_kernel_pdl_trigger_max_bins == rhs.init_kernel_pdl_trigger_max_bins + && lhs.dynamic_smem_bytes == rhs.dynamic_smem_bytes + && lhs.static_smem_threads_per_block == rhs.static_smem_threads_per_block + && lhs.static_smem_items_per_thread == rhs.static_smem_items_per_thread; } [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool @@ -64,13 +80,20 @@ struct HistogramPolicy << p.pixels_per_thread << ", .vec_size = " << p.vec_size << ", .load_algorithm = " << p.load_algorithm << ", .load_modifier = " << p.load_modifier << ", .rle_compress = " << p.rle_compress << ", .mem_preference = " << p.mem_preference << ", .use_work_stealing = " << p.use_work_stealing - << ", .init_kernel_pdl_trigger_max_bins = " << p.init_kernel_pdl_trigger_max_bins << " }"; + << ", .init_kernel_pdl_trigger_max_bins = " << p.init_kernel_pdl_trigger_max_bins << ", .dynamic_smem_bytes = " + << p.dynamic_smem_bytes << ", .static_smem_threads_per_block = " << p.static_smem_threads_per_block + << ", .static_smem_items_per_thread = " << p.static_smem_items_per_thread << " }"; } #endif // _CCCL_HOSTED() }; namespace detail::histogram { +// B200 exposes 232448 bytes of opt-in shared memory per block. Autoresearch +// retained 4096 bytes for static kernel storage and driver bookkeeping, leaving +// 228352 bytes for the runtime-sized privatized histogram. +static constexpr int sm100_dynamic_smem_bytes = 232448 - 4096; + // TODO(bgruber): drop in CCCL 4.0 enum class primitive_sample { @@ -113,7 +136,12 @@ _CCCL_HOST_DEVICE_API constexpr counter_size classify_counter_size() template _CCCL_HOST_DEVICE_API constexpr sample_size classify_sample_size() { - return sizeof(SampleT) == 1 ? sample_size::_1 : sizeof(SampleT) == 2 ? sample_size::_2 : sample_size::unknown; + return sizeof(SampleT) == 1 ? sample_size::_1 + : sizeof(SampleT) == 2 ? sample_size::_2 + : sizeof(SampleT) == 4 ? sample_size::_4 + : sizeof(SampleT) == 8 + ? sample_size::_8 + : sample_size::unknown; } // TODO(bgruber): drop in CCCL 4.0 @@ -180,8 +208,6 @@ struct sm100_tuning struct sm100_tuning @@ -197,7 +223,33 @@ struct sm100_tuning +struct sm100_tuning +{ + static constexpr int items = 12; + static constexpr int threads = 768; + static constexpr bool rle_compress = true; + static constexpr bool use_work_stealing = false; + static constexpr BlockHistogramMemoryPreference mem_preference = SMEM; + static constexpr CacheLoadModifier load_modifier = LOAD_LDG; + static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_VECTORIZE; + static constexpr int vec_size = 1 << 2; +}; + +template +struct sm100_tuning +{ + static constexpr int items = 8; + static constexpr int threads = 512; + static constexpr bool rle_compress = true; + static constexpr bool use_work_stealing = false; + static constexpr BlockHistogramMemoryPreference mem_preference = SMEM; + static constexpr CacheLoadModifier load_modifier = LOAD_LDG; + static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_VECTORIZE; + static constexpr int vec_size = 1 << 2; +}; + +// sample_size 2 showed no benefit over SM90 during verification benchmarks // multi.even and multi.range: none of the found tunings surpassed the SM90 tuning during verification benchmarks @@ -268,6 +320,7 @@ struct policy_hub 0)); static constexpr int init_kernel_pdl_trigger_max_bins = 2048; + static constexpr int dynamic_smem_bytes = sm100_dynamic_smem_bytes; }; using MaxPolicy = Policy1000; @@ -295,6 +348,12 @@ private: return (::cuda::std::max) (nominal_items_per_thread / num_active_channels / sample_scale, 1); } + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto sm100_policy(HistogramPolicy policy) const -> HistogramPolicy + { + policy.dynamic_smem_bytes = sm100_dynamic_smem_bytes; + return policy; + } + public: [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy { @@ -305,17 +364,59 @@ public: if (is_even) { // ipt_12.tpb_928.rle_0.ws_0.mem_1.ld_2.laid_0.vec_2 1.033332 0.940517 1.031835 1.195876 - return HistogramPolicy{928, 12, 1 << 2, BLOCK_LOAD_DIRECT, LOAD_CA, false, SMEM, false, 2048}; + return sm100_policy(HistogramPolicy{928, 12, 1 << 2, BLOCK_LOAD_DIRECT, LOAD_CA, false, SMEM, false, 2048}); } else { // ipt_12.tpb_448.rle_0.ws_0.mem_1.ld_1.laid_0.vec_2 1.078987 0.985542 1.085118 1.175637 - return HistogramPolicy{448, 12, 1 << 2, BLOCK_LOAD_DIRECT, LOAD_LDG, false, SMEM, false, 2048}; + return sm100_policy(HistogramPolicy{448, 12, 1 << 2, BLOCK_LOAD_DIRECT, LOAD_LDG, false, SMEM, false, 2048}); + } + } + + if (num_channels == 1 && num_active_channels == 1 && counter_size == 4 && sample_is_primitive + && (sample_size == 4 || sample_size == 8)) + { + if (is_even) + { + return sm100_policy( + HistogramPolicy{768, t_scale(12), 1 << 2, BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false, 2048}); } + + const int static_threads = sample_size_bytes >= 8 ? 384 : 768; + const int static_items = sample_size_bytes >= 8 ? t_scale(16) : 0; + return sm100_policy(HistogramPolicy{ + 768, + t_scale(12), + 1 << 2, + BLOCK_LOAD_DIRECT, + LOAD_LDG, + true, + SMEM, + false, + 2048, + 0, + static_threads, + static_items}); + } + + if (num_channels >= 2 && counter_size == 4 && sample_is_primitive) + { + if (is_even) + { + return sm100_policy(HistogramPolicy{1024, t_scale(8), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false, 0}); + } + return sm100_policy( + HistogramPolicy{1024, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false, 0, 0, 384}); + } + + if (num_channels == 1 && num_active_channels == 1 && counter_size == 4 && sample_is_primitive && sample_size == 2) + { + return sm100_policy(HistogramPolicy{960, 10, 1 << 2, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, SMEM, false, 2048}); } - // sample_size 2/4/8 showed no benefit over SM90 during verification benchmarks - // multi.even and multi.range: none of the found tunings surpassed the SM90 tuning during verification benchmarks + // Even when no SM100 launch-shape specialization applies, retain the + // architecture's dynamic shared-memory budget on the inherited fallback. + return sm100_policy(HistogramPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false, 0}); } if (cc >= ::cuda::compute_capability{9, 0}) diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index 6712bc678055..7727b7815436 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1626,7 +1626,7 @@ struct histogram_tuning { _CCCL_HOST_DEVICE_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { - return {BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, cub::SMEM, false, 0}; + return {BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, cub::SMEM, false, 0, 0}; } }; @@ -1750,7 +1750,7 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) // aggregate init constexpr auto p1 = cub::HistogramPolicy{ - 128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, cub::SMEM, false, 2048}; + 128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, cub::SMEM, false, 2048, 12345, 96, 3}; # if _CCCL_STD_VER >= 2020 // designated init @@ -1763,7 +1763,10 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) .rle_compress = false, .mem_preference = cub::SMEM, .use_work_stealing = false, - .init_kernel_pdl_trigger_max_bins = 2048}; + .init_kernel_pdl_trigger_max_bins = 2048, + .dynamic_smem_bytes = 12345, + .static_smem_threads_per_block = 96, + .static_smem_items_per_thread = 3}; # else // _CCCL_STD_VER >= 2020 constexpr auto p2 = p1; # endif // _CCCL_STD_VER >= 2020 @@ -1780,6 +1783,28 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) REQUIRE(to_string(p1) == "HistogramPolicy { .threads_per_block = 128, .pixels_per_thread = 7, .vec_size = 4" ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" - ", .mem_preference = SMEM, .use_work_stealing = 0, .init_kernel_pdl_trigger_max_bins = 2048 }"); + ", .mem_preference = SMEM, .use_work_stealing = 0, .init_kernel_pdl_trigger_max_bins = 2048" + ", .dynamic_smem_bytes = 12345, .static_smem_threads_per_block = 96" + ", .static_smem_items_per_thread = 3 }"); +} + +C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget", "[histogram][device]") +{ + using selector_t = cub::detail::histogram::policy_selector_from_types; + + constexpr auto sm90_policy = selector_t{}(cuda::compute_capability{9, 0}); + constexpr auto sm100_policy = selector_t{}(cuda::compute_capability{10, 0}); + constexpr auto sm100_wide_counter_policy = + cub::detail::histogram::policy_selector_from_types{}( + cuda::compute_capability{10, 0}); + + STATIC_REQUIRE(sm90_policy.dynamic_smem_bytes == 0); + STATIC_REQUIRE(sm100_policy.dynamic_smem_bytes == 228352); + STATIC_REQUIRE(sm100_wide_counter_policy.dynamic_smem_bytes == 228352); + + using max_policy_t = typename cub::detail::histogram::policy_hub::MaxPolicy; + const auto legacy_sm100_policy = + cub::detail::histogram::policy_selector_from_max_policy{}(cuda::compute_capability{10, 0}); + REQUIRE(legacy_sm100_policy.dynamic_smem_bytes == 228352); } #endif // _CCCL_COMPILER(GCC, >=, 8) From dbe0bdc7a8f41ae98a1369b282202860b96bdc74 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Thu, 30 Jul 2026 15:19:28 +0000 Subject: [PATCH 03/45] [cub] Address histogram SMEM review feedback --- cub/cub/agent/agent_histogram.cuh | 54 ++- .../device/dispatch/dispatch_histogram.cuh | 308 +++++++++++----- .../dispatch/kernels/kernel_histogram.cuh | 347 +++++------------- .../dispatch/tuning/tuning_histogram.cuh | 40 +- cub/test/catch2_test_device_histogram.cu | 10 + cub/test/catch2_test_device_histogram_env.cu | 107 +++++- 6 files changed, 478 insertions(+), 388 deletions(-) diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index ac2cc13810a0..ec5e3e5dd0be 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -175,7 +175,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE auto NativePointer(IteratorT itr) //! Random-access input iterator type for reading samples //! //! @tparam CounterT -//! Integer type for counting sample occurrences per histogram bin +//! Integer type for per-block privatized histogram bins //! //! @tparam PrivatizedDecodeOpT //! The transform operator type for determining privatized counter indices from samples, one for @@ -190,6 +190,9 @@ _CCCL_DEVICE _CCCL_FORCEINLINE auto NativePointer(IteratorT itr) //! //! @tparam UseDynamicSmemHistogram //! Whether the privatized histogram is supplied separately in dynamic shared memory. +//! +//! @tparam OutputCounterT +//! Integer type for final output histogram bins. May be wider than `CounterT`. template + bool UseDynamicSmemHistogram = false, + typename OutputCounterT = CounterT> struct AgentHistogram { + static_assert(sizeof(CounterT) <= sizeof(OutputCounterT), + "The output histogram counter must be at least as wide as the local counter"); static constexpr int vec_size = AgentHistogramPolicyT::VEC_SIZE; static constexpr int threads_per_block = AgentHistogramPolicyT::BLOCK_THREADS; static constexpr int pixels_per_thread = AgentHistogramPolicyT::PIXELS_PER_THREAD; @@ -234,13 +240,9 @@ struct AgentHistogram BlockLoad; using BlockLoadVecT = BlockLoad; - using HistogramsStorageT = ::cuda::std:: - _If; - struct _TempStorage { - // Static histogram storage or pointers into the separately allocated dynamic storage. - HistogramsStorageT histograms; + CounterT histograms[NumActiveChannels][PrivatizedSmemBins + 1]; int tile_idx; union @@ -259,7 +261,8 @@ struct AgentHistogram const int* num_output_bins; // one for each channel const int* num_privatized_bins; // one for each channel CounterT* d_privatized_histograms[NumActiveChannels]; // one for each channel - CounterT** d_output_histograms; // in global memory + CounterT* smem_histograms[NumActiveChannels]; // dynamic shared-memory channel bases, when enabled + OutputCounterT** d_output_histograms; // final output, in global memory const OutputDecodeOpT* output_decode_op; // determines output bin-id from privatized counter index, one for each // channel const PrivatizedDecodeOpT* privatized_decode_op; // determines privatized counter index from sample, one for each @@ -304,7 +307,7 @@ struct AgentHistogram if (output_bin >= 0) { - atomicAdd(&d_output_histograms[ch][output_bin], count); + atomicAdd(&d_output_histograms[ch][output_bin], static_cast(count)); } } } @@ -505,7 +508,14 @@ struct AgentHistogram if (prefer_smem) { - AccumulatePixels(samples, is_valid, temp_storage.histograms, ::cuda::std::bool_constant{}); + if constexpr (UseDynamicSmemHistogram) + { + AccumulatePixels(samples, is_valid, smem_histograms, ::cuda::std::bool_constant{}); + } + else + { + AccumulatePixels(samples, is_valid, temp_storage.histograms, ::cuda::std::bool_constant{}); + } } else { @@ -645,7 +655,7 @@ struct AgentHistogram SampleIteratorT d_samples, const int* num_output_bins, const int* num_privatized_bins, - CounterT** d_output_histograms, + OutputCounterT** d_output_histograms, CounterT** d_privatized_histograms, const OutputDecodeOpT* output_decode_op, const PrivatizedDecodeOpT* privatized_decode_op) @@ -683,7 +693,7 @@ struct AgentHistogram SampleIteratorT d_samples, const int* num_output_bins, const int* num_privatized_bins, - CounterT** d_output_histograms, + OutputCounterT** d_output_histograms, CounterT** d_privatized_histograms, const OutputDecodeOpT* output_decode_op, const PrivatizedDecodeOpT* privatized_decode_op, @@ -710,7 +720,7 @@ struct AgentHistogram _CCCL_PRAGMA_UNROLL_FULL() for (int ch = 0; ch < NumActiveChannels; ++ch) { - this->temp_storage.histograms[ch] = p; + this->smem_histograms[ch] = p; p += num_privatized_bins[ch]; } } @@ -772,7 +782,14 @@ struct AgentHistogram { if (prefer_smem) { - ZeroBinCounters(temp_storage.histograms); + if constexpr (UseDynamicSmemHistogram) + { + ZeroBinCounters(smem_histograms); + } + else + { + ZeroBinCounters(temp_storage.histograms); + } } else { @@ -785,7 +802,14 @@ struct AgentHistogram { if (prefer_smem) { - StoreOutput(temp_storage.histograms); + if constexpr (UseDynamicSmemHistogram) + { + StoreOutput(smem_histograms); + } + else + { + StoreOutput(temp_storage.histograms); + } } else { diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index b7b398ee199c..c881722bb81f 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -53,15 +54,23 @@ CUB_NAMESPACE_BEGIN namespace detail::histogram { -// Maximum number of bins per channel for which we will use a privatized smem strategy -static constexpr int max_privatized_smem_bins = 256; +template +struct local_counter +{ + using type = OutputCounterT; +}; -// Compile-time tag selecting the runtime-sized shared-memory kernel. The tag -// does not size storage; the actual byte budget comes from HistogramPolicy. -static constexpr int dynamic_smem_histogram_tag = 16384; +template +struct local_counter> +{ + using type = typename PolicySelector::local_counter_type; +}; + +template +using local_counter_t = typename local_counter::type; -static constexpr int multi_channel_dynamic_smem_bins_range = 2048; -static constexpr int multi_channel_dynamic_smem_bins_even = 8192; +// Maximum number of bins per channel for which we will use a privatized smem strategy +static constexpr int max_privatized_smem_bins = 256; template + typename SampleT, + typename OutputCounterT = CounterT> struct DeviceHistogramKernelSource { + static_assert(sizeof(CounterT) <= sizeof(OutputCounterT), + "The output histogram counter must be at least as wide as the local counter"); + using TransformsT = detail::histogram::Transforms; template _CCCL_HIDE_FROM_ABI CUB_RUNTIME_FUNCTION static constexpr auto HistogramInitKernel() { - return &DeviceHistogramInitKernel; + return &DeviceHistogramInitKernel; } /// Returns the default histogram sweep kernel that receives pre-initialized decode operators from the host. @@ -93,22 +106,23 @@ struct DeviceHistogramKernelSource CounterT, PrivatizedDecodeOpT, OutputDecodeOpT, - OffsetT>; + OffsetT, + OutputCounterT>; } - template + template _CCCL_HIDE_FROM_ABI CUB_RUNTIME_FUNCTION static constexpr auto HistogramSweepDynamicSmemKernel() { return &DeviceHistogramSweepDynamicSmemKernel< PolicyT, - PRIVATIZED_SMEM_BINS, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, CounterT, PrivatizedDecodeOpT, OutputDecodeOpT, - OffsetT>; + OffsetT, + OutputCounterT>; } /// Returns the device-init histogram sweep kernel that initializes decode operators from level arrays in the kernel. @@ -149,7 +163,8 @@ struct DeviceHistogramKernelSource PrivatizedDecodeOpT, OutputDecodeOpT, OffsetT, - IsEven>; + IsEven, + OutputCounterT>; } CUB_RUNTIME_FUNCTION static constexpr size_t CounterSize() @@ -179,9 +194,39 @@ struct DeviceHistogramKernelSource } }; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool +should_use_dynamic_smem(const HistogramPolicy& policy, int num_bins, int counter_size, int num_active_channels) +{ + if (policy.dynamic_smem_bytes <= 0 || num_bins <= 0 || counter_size <= 0 || num_active_channels <= 0) + { + return false; + } + + const bool prefer_dynamic_smem = counter_size > int{sizeof(unsigned int)} || num_bins > max_privatized_smem_bins; + const size_t required_bytes = size_t(num_bins) * size_t(num_active_channels) * size_t(counter_size); + + int max_bins = num_bins; + if (num_active_channels > 1) + { + if constexpr (IsEven) + { + max_bins = num_active_channels <= 3 ? policy.dynamic_smem_even_3ch_max_bins : policy.dynamic_smem_even_max_bins; + } + else + { + max_bins = policy.dynamic_smem_range_max_bins; + } + } + + return prefer_dynamic_smem && max_bins > 0 && num_bins <= max_bins + && required_bytes <= static_cast(policy.dynamic_smem_bytes); +} + template ; + ::cuda::compute_capability cc{}; if (const auto error = CubDebug(launcher_factory.PtxComputeCap(cc))) { @@ -233,18 +280,15 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( })) #endif // _CCCL_HOSTED() && defined(CUB_DEBUG_LOG) - const auto init_kernel = kernel_source.template HistogramInitKernel(); - constexpr bool use_dynamic_smem = PRIVATIZED_SMEM_BINS == dynamic_smem_histogram_tag; - auto sweep_kernel = [&] { - if constexpr (use_dynamic_smem) + const auto init_kernel = kernel_source.template HistogramInitKernel(); + auto sweep_kernel = [&] { + if constexpr (UseDynamicSmem) { static_assert(!IsDeviceInit, "Dynamic shared-memory histograms require host-initialized transforms"); using output_decode_op_t = typename FirstLevelArrayT::value_type; using privatized_decode_op_t = typename SecondLevelArrayT::value_type; - return kernel_source.template HistogramSweepDynamicSmemKernel(); + return kernel_source + .template HistogramSweepDynamicSmemKernel(); } else if constexpr (IsDeviceInit) { @@ -265,12 +309,12 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( } }(); - constexpr bool use_static_smem = PRIVATIZED_SMEM_BINS > 0 && !use_dynamic_smem; + constexpr bool use_static_smem = PRIVATIZED_SMEM_BINS > 0 && !UseDynamicSmem; const int threads_per_block = use_static_smem ? active_policy.static_smem_threads() : active_policy.threads_per_block; const int pixels_per_thread = use_static_smem ? active_policy.static_smem_items() : active_policy.pixels_per_thread; int dynamic_smem_bytes = 0; - if constexpr (use_dynamic_smem) + if constexpr (UseDynamicSmem) { for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel) { @@ -334,8 +378,8 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL) { allocation_sizes[CHANNEL] = - use_dynamic_smem ? 0 - : size_t(num_thread_blocks) * (num_privatized_levels[CHANNEL] - 1) * kernel_source.CounterSize(); + UseDynamicSmem ? 0 + : size_t(num_thread_blocks) * (num_privatized_levels[CHANNEL] - 1) * kernel_source.CounterSize(); } allocation_sizes[NUM_ALLOCATIONS - 1] = GridQueue::AllocationSize(); @@ -358,11 +402,11 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( GridQueue tile_queue(allocations[NUM_ALLOCATIONS - 1]); // Wrap arrays so we can pass them by-value to the kernel - ::cuda::std::array d_privatized_histograms_wrapper; + ::cuda::std::array d_privatized_histograms_wrapper; ::cuda::std::array num_privatized_bins_wrapper; ::cuda::std::array num_output_bins_wrapper; - auto* typed_allocations = reinterpret_cast(allocations); + auto* typed_allocations = reinterpret_cast(allocations); ::cuda::std::copy(typed_allocations, typed_allocations + NUM_ACTIVE_CHANNELS, d_privatized_histograms_wrapper.begin()); auto minus_one = ::cuda::proclaim_return_type([](int levels) { @@ -576,6 +620,7 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device (detail::histogram::dispatch( @@ -608,6 +653,7 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device (detail::histogram::dispatch( @@ -748,6 +794,7 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device (detail::histogram::dispatch( @@ -803,6 +850,58 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_bytes(long) return 0; } +template +_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_range_max_bins(int) + -> decltype(ActivePolicy::dynamic_smem_range_max_bins) +{ + return ActivePolicy::dynamic_smem_range_max_bins; +} + +template +_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_range_max_bins(long) +{ + return 0; +} + +template +_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_max_bins(int) + -> decltype(ActivePolicy::dynamic_smem_even_max_bins) +{ + return ActivePolicy::dynamic_smem_even_max_bins; +} + +template +_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_max_bins(long) +{ + return 0; +} + +template +_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_3ch_max_bins(int) + -> decltype(ActivePolicy::dynamic_smem_even_3ch_max_bins) +{ + return ActivePolicy::dynamic_smem_even_3ch_max_bins; +} + +template +_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_3ch_max_bins(long) +{ + return 0; +} + +template +_CCCL_HOST_DEVICE_API constexpr auto convert_range_interpolation_min_bins(int) + -> decltype(ActivePolicy::range_interpolation_min_bins) +{ + return ActivePolicy::range_interpolation_min_bins; +} + +template +_CCCL_HOST_DEVICE_API constexpr auto convert_range_interpolation_min_bins(long) +{ + return 0; +} + // TODO(bgruber): drop in CCCL 4.0 template _CCCL_HOST_DEVICE_API constexpr auto convert_policy() -> HistogramPolicy @@ -818,7 +917,13 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_policy() -> HistogramPolicy ap::MEM_PREFERENCE, ap::IS_WORK_STEALING, convert_pdl_trigger(0), - convert_dynamic_smem_bytes(0)}; + convert_dynamic_smem_bytes(0), + 0, + 0, + convert_dynamic_smem_range_max_bins(0), + convert_dynamic_smem_even_max_bins(0), + convert_dynamic_smem_even_3ch_max_bins(0), + convert_range_interpolation_min_bins(0)}; } // TODO(bgruber): drop in CCCL 4.0 @@ -852,19 +957,25 @@ public: } }; -template < - int NUM_CHANNELS, - int NUM_ACTIVE_CHANNELS, - typename SampleIteratorT, - typename CounterT, - typename LevelT, - typename OffsetT, - bool IsByteSample, - typename PolicySelector, - typename SampleT = it_value_t, /// The sample value type of the input iterator - typename KernelSource = - DeviceHistogramKernelSource, - typename KernelLauncherFactory = CUB_DETAIL_DEFAULT_KERNEL_LAUNCHER_FACTORY> +template , /// The sample value type of the input iterator + typename KernelSource = DeviceHistogramKernelSource< + NUM_CHANNELS, + NUM_ACTIVE_CHANNELS, + SampleIteratorT, + local_counter_t, + LevelT, + OffsetT, + SampleT, + CounterT>, + typename KernelLauncherFactory = CUB_DETAIL_DEFAULT_KERNEL_LAUNCHER_FACTORY> CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( void* d_temp_storage, size_t& temp_storage_bytes, @@ -881,6 +992,15 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( KernelSource kernel_source = {}, KernelLauncherFactory launcher_factory = {}) { + using LocalCounterT = local_counter_t; + + ::cuda::compute_capability cc{}; + if (const auto error = CubDebug(launcher_factory.PtxComputeCap(cc))) + { + return error; + } + const HistogramPolicy active_policy = policy_selector(cc); + if constexpr (IsByteSample) { using TransformsT = Transforms; @@ -899,7 +1019,8 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel) { num_privatized_levels[channel] = 257; - output_decode_op[channel].Init(d_levels[channel], num_output_levels[channel]); + output_decode_op[channel].Init( + d_levels[channel], num_output_levels[channel], active_policy.range_interpolation_min_bins); if (num_output_levels[channel] > max_levels) { @@ -914,6 +1035,7 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( (detail::histogram::dispatch( @@ -953,7 +1075,8 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel) { - privatized_decode_op[channel].Init(d_levels[channel], num_output_levels[channel]); + privatized_decode_op[channel].Init( + d_levels[channel], num_output_levels[channel], active_policy.range_interpolation_min_bins); if (num_output_levels[channel] > max_levels) { max_levels = num_output_levels[channel]; @@ -961,25 +1084,17 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( } int max_num_output_bins = max_levels - 1; - ::cuda::compute_capability cc{}; - if (const auto error = CubDebug(launcher_factory.PtxComputeCap(cc))) - { - return error; - } - const HistogramPolicy active_policy = policy_selector(cc); - - if constexpr (NUM_ACTIVE_CHANNELS >= 1) + if (should_use_dynamic_smem( + active_policy, max_num_output_bins, int{kernel_source.CounterSize()}, NUM_ACTIVE_CHANNELS)) { - const bool within_tuned_channel_cap = - NUM_ACTIVE_CHANNELS == 1 || max_num_output_bins <= multi_channel_dynamic_smem_bins_range; - const size_t dynamic_smem_bytes = size_t(max_num_output_bins) * NUM_ACTIVE_CHANNELS * kernel_source.CounterSize(); - const bool prefer_dynamic_smem = - kernel_source.CounterSize() > sizeof(unsigned int) || max_num_output_bins > max_privatized_smem_bins; - if (prefer_dynamic_smem && within_tuned_channel_cap - && dynamic_smem_bytes <= static_cast(active_policy.dynamic_smem_bytes)) - { - constexpr int PRIVATIZED_SMEM_BINS = dynamic_smem_histogram_tag; - return detail::histogram::dispatch( + return CubDebug( + (detail::histogram::dispatch( d_temp_storage, temp_storage_bytes, d_samples, @@ -995,8 +1110,7 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( stream, policy_selector, kernel_source, - launcher_factory); - } + launcher_factory))); } // Dispatch @@ -1009,6 +1123,7 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( (detail::histogram::dispatch( @@ -1041,6 +1156,7 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( (detail::histogram::dispatch( @@ -1069,19 +1185,25 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( return cudaSuccess; } -template < - int NUM_CHANNELS, - int NUM_ACTIVE_CHANNELS, - typename SampleIteratorT, - typename CounterT, - typename LevelT, - typename OffsetT, - bool IsByteSample, - typename PolicySelector, - typename SampleT = it_value_t, /// The sample value type of the input iterator - typename KernelSource = - DeviceHistogramKernelSource, - typename KernelLauncherFactory = CUB_DETAIL_DEFAULT_KERNEL_LAUNCHER_FACTORY> +template , /// The sample value type of the input iterator + typename KernelSource = DeviceHistogramKernelSource< + NUM_CHANNELS, + NUM_ACTIVE_CHANNELS, + SampleIteratorT, + local_counter_t, + LevelT, + OffsetT, + SampleT, + CounterT>, + typename KernelLauncherFactory = CUB_DETAIL_DEFAULT_KERNEL_LAUNCHER_FACTORY> CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t dispatch_even( void* d_temp_storage, size_t& temp_storage_bytes, @@ -1099,6 +1221,8 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t dispatch_even( KernelSource kernel_source = {}, KernelLauncherFactory launcher_factory = {}) { + using LocalCounterT = local_counter_t; + if constexpr (IsByteSample) { using TransformsT = Transforms; @@ -1145,6 +1269,7 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t dispatch_even( (detail::histogram::dispatch( @@ -1212,21 +1337,17 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t dispatch_even( } const HistogramPolicy active_policy = policy_selector(cc); - if constexpr (NUM_ACTIVE_CHANNELS >= 1) + if (should_use_dynamic_smem( + active_policy, max_num_output_bins, int{kernel_source.CounterSize()}, NUM_ACTIVE_CHANNELS)) { - const bool within_tuned_channel_cap = - NUM_ACTIVE_CHANNELS == 1 || max_num_output_bins <= multi_channel_dynamic_smem_bins_even - || (NUM_ACTIVE_CHANNELS <= 3 - && size_t(max_num_output_bins) * NUM_ACTIVE_CHANNELS * kernel_source.CounterSize() - <= static_cast(active_policy.dynamic_smem_bytes)); - const size_t dynamic_smem_bytes = size_t(max_num_output_bins) * NUM_ACTIVE_CHANNELS * kernel_source.CounterSize(); - const bool prefer_dynamic_smem = - kernel_source.CounterSize() > sizeof(unsigned int) || max_num_output_bins > max_privatized_smem_bins; - if (prefer_dynamic_smem && within_tuned_channel_cap - && dynamic_smem_bytes <= static_cast(active_policy.dynamic_smem_bytes)) - { - constexpr int PRIVATIZED_SMEM_BINS = dynamic_smem_histogram_tag; - return detail::histogram::dispatch( + return CubDebug( + (detail::histogram::dispatch( d_temp_storage, temp_storage_bytes, d_samples, @@ -1242,8 +1363,7 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t dispatch_even( stream, policy_selector, kernel_source, - launcher_factory); - } + launcher_factory))); } if (max_num_output_bins > max_privatized_smem_bins) @@ -1254,6 +1374,7 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t dispatch_even( (detail::histogram::dispatch( @@ -1285,6 +1406,7 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t dispatch_even( (detail::histogram::dispatch( diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index 73855cf1dedb..ecb098692f49 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -35,23 +35,10 @@ struct Transforms template struct SearchTransform { - // Compile-time RANGE marker, resolved at instantiation (no runtime branch). - // The direct-atomic kernels read this to specialize behavior that only helps - // the RANGE (SearchTransform) classify, e.g. the per-thread bracket cache. + // Compile-time marker used by the privatized-SMEM sweep to select the RANGE + // low-bin classifier and its launch bound. static constexpr bool is_range_transform = true; - // Below this bin count, BinSelect skips the interpolated-first-guess machinery - // (and PrecomputeOnDevice stays disabled) and uses the lean UpperBound binary - // search instead. The interpolation path carries ~7 extra per-thread registers - // (endpoints + slopes + 3-point split state) plus a device precompute prologue; - // on the small-bin tiers -- especially the static 256-bin privatized-SMEM kernel, - // which is occupancy/register-bound -- that overhead is a net LOSS. Measured on - // B200: at 256 bins interpolation runs ~0.67x of plain binary search, but by - // 1024 bins it already wins ~2.3x and keeps growing, so the cutoff sits between. - // (Was hard-coded `< 4`, which only skipped the degenerate tiny case and let the - // 256-bin RANGE/I32 tier regress vs upstream.) - static constexpr int kInterpolationMinBins = 512; - //! @brief Per-thread most-recently-used (MRU) bin-bracket cache. //! //! Carries the last successfully-resolved bin and its two boundary level @@ -63,10 +50,9 @@ struct Transforms //! classify. Low-entropy inputs (constant or heavily-skewed samples) have high //! consecutive-sample locality, so the bracket hits dominate. A `bin < 0` //! sentinel marks the cache empty. - //! This is per-thread mutable state, so it is only sound on a per-thread - //! `SearchTransform` copy (the direct-atomic cuckoo/single-probe kernels' - //! `decode_op[ch]`), never on the shared `__grid_constant__` decode op that - //! the SMEM-privatized agent path reads through a const pointer. + //! This is per-thread mutable state, so the dynamic-SMEM sweep copies each + //! transform into thread-local storage before using the cache. The static + //! sweep reads the shared transform without a cache. struct BracketCacheT { LevelT lo; // cached d_levels[bin] @@ -76,6 +62,7 @@ struct Transforms LevelIteratorT d_levels; // Pointer to levels array int num_output_levels; // Number of levels in array + int interpolation_min_bins; // policy-selected minimum bin count for interpolation // Precomputed (loop-invariant) interpolation state, populated by // `PrecomputeOnDevice()`. The interpolation slope `num_bins / (last - @@ -111,12 +98,14 @@ struct Transforms //! //! @param d_levels_ Pointer to levels array //! @param num_output_levels_ Number of levels in array - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void Init(LevelIteratorT d_levels_, int num_output_levels_) + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void + Init(LevelIteratorT d_levels_, int num_output_levels_, int interpolation_min_bins_) { - this->d_levels = d_levels_; - this->num_output_levels = num_output_levels_; - this->m_have_precompute = false; - this->m_inv_scale = 0.0f; + this->d_levels = d_levels_; + this->num_output_levels = num_output_levels_; + this->interpolation_min_bins = interpolation_min_bins_; + this->m_have_precompute = false; + this->m_inv_scale = 0.0f; } //! @brief Hoist the loop-invariant interpolation state out of `BinSelect`. @@ -131,7 +120,7 @@ struct Transforms _CCCL_DEVICE _CCCL_FORCEINLINE void PrecomputeOnDevice() { const int num_bins = num_output_levels - 1; - if (num_bins < kInterpolationMinBins) + if (num_bins < interpolation_min_bins) { m_have_precompute = false; return; @@ -178,20 +167,20 @@ struct Transforms } } - // Method for converting samples to bin-ids - template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid) const + private: + struct NoBracketCacheT + {}; + + template + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void Classify(_SampleT sample, int& bin, bool valid, CacheT& cache) const { - /// Level iterator wrapper type - // Wrap the native input pointer with CacheModifiedInputIterator - // or Directly use the supplied input iterator type using WrappedLevelIteratorT = ::cuda::std::_If<::cuda::std::is_pointer_v, CacheModifiedInputIterator, LevelIteratorT>; + constexpr bool use_cache = ::cuda::std::is_same_v; WrappedLevelIteratorT wrapped_levels(d_levels); - const int num_bins = num_output_levels - 1; if (!valid) { @@ -200,9 +189,16 @@ struct Transforms const LevelT s = static_cast(sample); - // For very small bin counts, the interpolation overhead is not worth - // it; fall back to the original binary search. - if (num_bins < kInterpolationMinBins) + if constexpr (use_cache) + { + if (cache.bin >= 0 && !(s < cache.lo) && (s < cache.hi)) + { + bin = cache.bin; + return; + } + } + + if (interpolation_min_bins <= 0 || num_bins < interpolation_min_bins) { bin = UpperBound(wrapped_levels, num_output_levels, s) - 1; if (bin >= num_bins) @@ -212,23 +208,9 @@ struct Transforms return; } - // Read first and last levels. When `PrecomputeOnDevice()` has run we use - // the cached endpoints (and the precomputed reciprocal slope below), - // removing two per-sample endpoint loads and the per-sample - // `__fdividef`. Otherwise (host-only init, or a degenerate level array - // that PrecomputeOnDevice rejected) we read them per sample as before. - // These are warp/CTA-uniform and land in L1 / texture cache after the - // first read, so even the fallback amortizes across samples. const LevelT first_level = m_have_precompute ? m_first : wrapped_levels[0]; const LevelT last_level = m_have_precompute ? m_last : wrapped_levels[num_bins]; - // Defensive: if a user-supplied level array has non-monotonic endpoints - // (e.g. `last_level <= first_level`), the boundary check below would - // misclassify all samples as out-of-range. Fall back to UpperBound, - // which uses ordered comparisons only and produces correct results - // regardless of endpoint ordering. (PrecomputeOnDevice already enforces - // `first < last` before setting m_have_precompute, so this only fires on - // the non-precomputed path.) if (!(first_level < last_level)) { bin = UpperBound(wrapped_levels, num_output_levels, s) - 1; @@ -239,39 +221,16 @@ struct Transforms return; } - // Out-of-range samples map to bin -1. if (s < first_level || !(s < last_level)) { bin = -1; return; } - // Interpolated first-guess index. We always use a fast 32-bit float - // divide (MUFU.RCP) for the slope: the divide does not have to be - // accurate, only close enough that the verify-or-1-step-correct path - // hits a handful of bins. The full UpperBound fallback catches any - // remaining mismatch from precision loss or non-uniform spacing. - // For wide-ranged 64-bit types we still compute (sample - first) in - // the level type to avoid float overflow on the difference itself. - // - // On the precomputed path the slope `num_bins / (last - first)` is a - // loop-invariant `m_inv_scale`, so the guess collapses to a single - // `(float)delta * m_inv_scale` FMA (no per-sample MUFU.RCP). The result - // is bit-identical in intent to `__fdividef(delta*num_bins, range)`: - // both are approximate first guesses validated by the bracket check - // below, so any rounding difference is absorbed by the same verify / - // 1-step / UpperBound correction ladder. const auto delta = (s - first_level); int guess; if (m_have_precompute) { - // Three-point piecewise-linear first guess: interpolate on whichever - // half of [first, last] the sample falls in (split at the cached - // midpoint level m_mid / m_mid_bin). Using a local slope and a smaller - // delta magnitude lands the guess closer to the true bin than a single - // first->last secant, so the verify-or-1-step ladder converges without - // reaching the UpperBound binary search. m_mid_bin == 0 means the split - // was degenerate, so we use the single-secant guess. if (m_mid_bin > 0) { if (s < m_mid) @@ -308,67 +267,83 @@ struct Transforms guess = num_bins - 1; } - // Verify the guess: d_levels[guess] <= s < d_levels[guess + 1]. We - // load both bracketing levels in parallel to expose memory-level - // parallelism and branch on the result. The level array has length - // num_bins + 1, so wrapped_levels[guess + 1] is always in-bounds for - // guess <= num_bins - 1. const LevelT lvl_lo = wrapped_levels[guess]; const LevelT lvl_hi = wrapped_levels[guess + 1]; if (!(s < lvl_lo) && (s < lvl_hi)) { bin = guess; + if constexpr (use_cache) + { + cache = BracketCacheT{lvl_lo, lvl_hi, guess}; + } return; } - // One-step linear correction: try a single neighbor before falling - // back to a binary search. If the guess was high, try guess - 1; if - // low, try guess + 1. if (s < lvl_lo) { - // guess too high; check guess - 1. const int g2 = guess - 1; if (g2 >= 0) { const LevelT lvl2_lo = wrapped_levels[g2]; - // lvl2_hi is lvl_lo (loaded already). if (!(s < lvl2_lo)) { bin = g2; + if constexpr (use_cache) + { + cache = BracketCacheT{lvl2_lo, lvl_lo, g2}; + } return; } } } else { - // s >= lvl_hi: guess too low; check guess + 1. const int g2 = guess + 1; if (g2 <= num_bins - 1) { - // lvl2_lo is lvl_hi (loaded already). const LevelT lvl2_hi = wrapped_levels[g2 + 1]; if (s < lvl2_hi) { bin = g2; + if constexpr (use_cache) + { + cache = BracketCacheT{lvl_hi, lvl2_hi, g2}; + } return; } } } - // Fall back to binary search for irregular level distributions. bin = UpperBound(wrapped_levels, num_output_levels, s) - 1; if (bin >= num_bins) { bin = -1; + return; + } + if constexpr (use_cache) + { + if (bin >= 0) + { + cache = BracketCacheT{wrapped_levels[bin], wrapped_levels[bin + 1], bin}; + } } } + public: + // Method for converting samples to bin-ids + template + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid) const + { + NoBracketCacheT cache; + Classify(sample, bin, valid, cache); + } + //! @brief Lean classify for the STATIC <=256-bin SMEM tier. //! //! Byte-identical to upstream `main`'s flat `BinSelect`: a single `UpperBound` //! + clamp, with NONE of the interpolation machinery the 3-arg `BinSelect` above - //! carries (no `num_bins < kInterpolationMinBins` runtime branch, no reads of the + //! carries (no `num_bins < interpolation_min_bins` runtime branch, no reads of the //! precompute fields `m_inv_scale`/`m_first`/...). At <=256 bins the interpolation //! fast-path never activates (`m_have_precompute` stays false there), so that //! machinery is pure dead weight: its extra branch + the wider codegen/register @@ -412,168 +387,7 @@ struct Transforms template _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid, BracketCacheT& mru) const { - using WrappedLevelIteratorT = - ::cuda::std::_If<::cuda::std::is_pointer_v, - CacheModifiedInputIterator, - LevelIteratorT>; - - const int num_bins = num_output_levels - 1; - if (!valid) - { - return; - } - - const LevelT s = static_cast(sample); - - // Fast path: the cached bracket holds the answer with no level loads. - // `mru.bin >= 0` guarantees the bracket is populated and in-range. - if (mru.bin >= 0 && !(s < mru.lo) && (s < mru.hi)) - { - bin = mru.bin; - return; - } - - // Tiny bin counts: the interpolation/bracket machinery is not worth it. - if (num_bins < kInterpolationMinBins) - { - WrappedLevelIteratorT wrapped_levels(d_levels); - bin = UpperBound(wrapped_levels, num_output_levels, s) - 1; - if (bin >= num_bins) - { - bin = -1; - } - return; - } - - WrappedLevelIteratorT wrapped_levels(d_levels); - - const LevelT first_level = m_have_precompute ? m_first : wrapped_levels[0]; - const LevelT last_level = m_have_precompute ? m_last : wrapped_levels[num_bins]; - - if (!(first_level < last_level)) - { - bin = UpperBound(wrapped_levels, num_output_levels, s) - 1; - if (bin >= num_bins) - { - bin = -1; - } - return; - } - - // Out-of-range samples map to bin -1 (and do not update the cache). - if (s < first_level || !(s < last_level)) - { - bin = -1; - return; - } - - // Identical first-guess ladder to the plain BinSelect above: on a cache - // MISS we reproduce the same three-point piecewise-linear first guess so - // miss-heavy inputs (high-entropy samples, and the irregular-level - // fallback) converge exactly as the uncached path does. Only the hit fast - // path and the cache writebacks differ. - const auto delta = (s - first_level); - int guess; - if (m_have_precompute) - { - if (m_mid_bin > 0) - { - if (s < m_mid) - { - guess = static_cast(static_cast(delta) * m_inv_scale_lo); - } - else - { - const auto delta_hi = (s - m_mid); - guess = m_mid_bin + static_cast(static_cast(delta_hi) * m_inv_scale_hi); - } - } - else - { - guess = static_cast(static_cast(delta) * m_inv_scale); - } - } - else - { - const auto range = (last_level - first_level); - NV_IF_ELSE_TARGET( - NV_IS_DEVICE, - (guess = static_cast( - __fdividef(static_cast(delta) * static_cast(num_bins), static_cast(range)));), - (guess = static_cast( - (static_cast(delta) * static_cast(num_bins)) / static_cast(range));)); - } - if (guess < 0) - { - guess = 0; - } - else if (guess > num_bins - 1) - { - guess = num_bins - 1; - } - - const LevelT lvl_lo = wrapped_levels[guess]; - const LevelT lvl_hi = wrapped_levels[guess + 1]; - - if (!(s < lvl_lo) && (s < lvl_hi)) - { - bin = guess; - mru.lo = lvl_lo; - mru.hi = lvl_hi; - mru.bin = guess; - return; - } - - // One-step linear correction. - if (s < lvl_lo) - { - const int g2 = guess - 1; - if (g2 >= 0) - { - const LevelT lvl2_lo = wrapped_levels[g2]; - if (!(s < lvl2_lo)) - { - bin = g2; - mru.lo = lvl2_lo; - mru.hi = lvl_lo; // lvl2_hi == lvl_lo (already loaded) - mru.bin = g2; - return; - } - } - } - else - { - const int g2 = guess + 1; - if (g2 <= num_bins - 1) - { - const LevelT lvl2_hi = wrapped_levels[g2 + 1]; - if (s < lvl2_hi) - { - bin = g2; - mru.lo = lvl_hi; // lvl2_lo == lvl_hi (already loaded) - mru.hi = lvl2_hi; - mru.bin = g2; - return; - } - } - } - - // Fall back to binary search for irregular level distributions. This is - // the rare path, so the two extra bracket loads needed to refresh the MRU - // cache are amortized; they keep subsequent in-bracket samples on the - // zero-load fast path. - bin = UpperBound(wrapped_levels, num_output_levels, s) - 1; - if (bin >= num_bins) - { - bin = -1; - return; - } - if (bin >= 0) - { - mru.lo = wrapped_levels[bin]; - mru.hi = wrapped_levels[bin + 1]; - mru.bin = bin; - } + Classify(sample, bin, valid, mru); } }; @@ -982,7 +796,8 @@ template + typename OffsetT, + typename OutputCounterT = CounterT> #if _CCCL_HAS_CONCEPTS() requires histogram_policy_selector #endif // _CCCL_HAS_CONCEPTS() @@ -1001,7 +816,7 @@ __launch_bounds__( const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, const ::cuda::std::array num_privatized_bins_wrapper, - ::cuda::std::array d_output_histograms_wrapper, + ::cuda::std::array d_output_histograms_wrapper, ::cuda::std::array d_privatized_histograms_wrapper, const ::cuda::std::array output_decode_op_wrapper, const ::cuda::std::array privatized_decode_op_wrapper, @@ -1027,14 +842,16 @@ __launch_bounds__( hp.vec_size>; using AgentHistogramT = AgentHistogram; + OffsetT, + false, + OutputCounterT>; // Shared memory for AgentHistogram __shared__ typename AgentHistogramT::TempStorage temp_storage; @@ -1066,14 +883,14 @@ __launch_bounds__( //! allows one kernel instantiation to cover larger histograms without a ladder //! of statically sized kernels. template + typename OffsetT, + typename OutputCounterT = CounterT> #if _CCCL_HAS_CONCEPTS() requires histogram_policy_selector #endif // _CCCL_HAS_CONCEPTS() @@ -1082,7 +899,7 @@ __launch_bounds__(int(current_policy().threads_per_block)) const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, const ::cuda::std::array num_privatized_bins_wrapper, - ::cuda::std::array d_output_histograms_wrapper, + ::cuda::std::array d_output_histograms_wrapper, ::cuda::std::array d_privatized_histograms_wrapper, const ::cuda::std::array output_decode_op_wrapper, const ::cuda::std::array privatized_decode_op_wrapper, @@ -1105,7 +922,7 @@ __launch_bounds__(int(current_policy().threads_per_block)) hp.vec_size>; using AgentHistogramT = AgentHistogram().threads_per_block)) PrivatizedDecodeOpT, OutputDecodeOpT, OffsetT, - true>; + true, + OutputCounterT>; __shared__ typename AgentHistogramT::TempStorage temp_storage; - extern __shared__ unsigned char dynamic_smem[]; + extern __shared__ __align__(16) unsigned char dynamic_smem[]; OutputDecodeOpT output_decode_op[NumActiveChannels]; PrivatizedDecodeOpT privatized_decode_op[NumActiveChannels]; @@ -1240,7 +1058,8 @@ template + bool IsEven, + typename OutputCounterT = CounterT> #if _CCCL_HAS_CONCEPTS() requires histogram_policy_selector #endif // _CCCL_HAS_CONCEPTS() @@ -1249,7 +1068,7 @@ __launch_bounds__(int(current_policy().threads_per_block)) const SampleIteratorT d_samples, ::cuda::std::array num_output_bins_wrapper, ::cuda::std::array num_privatized_bins_wrapper, - ::cuda::std::array d_output_histograms_wrapper, + ::cuda::std::array d_output_histograms_wrapper, ::cuda::std::array d_privatized_histograms_wrapper, const FirstLevelArrayT first_level_array, const SecondLevelArrayT second_level_array, @@ -1282,8 +1101,8 @@ __launch_bounds__(int(current_policy().threads_per_block)) { const auto num_output_levels = first_level_array[channel]; const auto levels = second_level_array[channel]; - privatized_decode_op[channel].Init(levels, num_output_levels); - output_decode_op[channel].Init(levels, num_output_levels); + privatized_decode_op[channel].Init(levels, num_output_levels, hp.range_interpolation_min_bins); + output_decode_op[channel].Init(levels, num_output_levels, hp.range_interpolation_min_bins); } } @@ -1306,7 +1125,9 @@ __launch_bounds__(int(current_policy().threads_per_block)) CounterT, PrivatizedDecodeOpT, OutputDecodeOpT, - OffsetT>; + OffsetT, + false, + OutputCounterT>; // Shared memory for AgentHistogram __shared__ typename AgentHistogramT::TempStorage temp_storage; diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 16dfe0ea753a..ec4da1035911 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -39,9 +39,13 @@ struct HistogramPolicy bool use_work_stealing; //!< Whether to dequeue tiles from a global work queue int init_kernel_pdl_trigger_max_bins; //!< Maximum number of bins for the init kernel to trigger the histogram kernel //!< early using PDL - int dynamic_smem_bytes = 0; //!< Tuned byte budget for a runtime-sized privatized histogram; 0 disables it - int static_smem_threads_per_block = 0; //!< Static shared-memory tier threads; 0 inherits threads_per_block - int static_smem_items_per_thread = 0; //!< Static shared-memory tier items; 0 inherits pixels_per_thread + int dynamic_smem_bytes = 0; //!< Tuned byte budget for a runtime-sized privatized histogram; 0 disables it + int static_smem_threads_per_block = 0; //!< Static shared-memory tier threads; 0 inherits threads_per_block + int static_smem_items_per_thread = 0; //!< Static shared-memory tier items; 0 inherits pixels_per_thread + int dynamic_smem_range_max_bins = 0; //!< Multi-channel RANGE cap per channel; 0 disables the dynamic path + int dynamic_smem_even_max_bins = 0; //!< Multi-channel EVEN cap per channel; 0 disables the dynamic path + int dynamic_smem_even_3ch_max_bins = 0; //!< Extended EVEN cap when at most three channels are active + int range_interpolation_min_bins = 0; //!< Minimum RANGE bin count for interpolation; 0 disables interpolation [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int static_smem_threads() const { @@ -63,7 +67,11 @@ struct HistogramPolicy && lhs.init_kernel_pdl_trigger_max_bins == rhs.init_kernel_pdl_trigger_max_bins && lhs.dynamic_smem_bytes == rhs.dynamic_smem_bytes && lhs.static_smem_threads_per_block == rhs.static_smem_threads_per_block - && lhs.static_smem_items_per_thread == rhs.static_smem_items_per_thread; + && lhs.static_smem_items_per_thread == rhs.static_smem_items_per_thread + && lhs.dynamic_smem_range_max_bins == rhs.dynamic_smem_range_max_bins + && lhs.dynamic_smem_even_max_bins == rhs.dynamic_smem_even_max_bins + && lhs.dynamic_smem_even_3ch_max_bins == rhs.dynamic_smem_even_3ch_max_bins + && lhs.range_interpolation_min_bins == rhs.range_interpolation_min_bins; } [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool @@ -82,7 +90,10 @@ struct HistogramPolicy << ", .mem_preference = " << p.mem_preference << ", .use_work_stealing = " << p.use_work_stealing << ", .init_kernel_pdl_trigger_max_bins = " << p.init_kernel_pdl_trigger_max_bins << ", .dynamic_smem_bytes = " << p.dynamic_smem_bytes << ", .static_smem_threads_per_block = " << p.static_smem_threads_per_block - << ", .static_smem_items_per_thread = " << p.static_smem_items_per_thread << " }"; + << ", .static_smem_items_per_thread = " << p.static_smem_items_per_thread << ", .dynamic_smem_range_max_bins = " + << p.dynamic_smem_range_max_bins << ", .dynamic_smem_even_max_bins = " << p.dynamic_smem_even_max_bins + << ", .dynamic_smem_even_3ch_max_bins = " << p.dynamic_smem_even_3ch_max_bins + << ", .range_interpolation_min_bins = " << p.range_interpolation_min_bins << " }"; } #endif // _CCCL_HOSTED() }; @@ -92,7 +103,11 @@ namespace detail::histogram // B200 exposes 232448 bytes of opt-in shared memory per block. Autoresearch // retained 4096 bytes for static kernel storage and driver bookkeeping, leaving // 228352 bytes for the runtime-sized privatized histogram. -static constexpr int sm100_dynamic_smem_bytes = 232448 - 4096; +static constexpr int sm100_dynamic_smem_bytes = 232448 - 4096; +static constexpr int sm100_dynamic_smem_range_max_bins = 2048; +static constexpr int sm100_dynamic_smem_even_max_bins = 8192; +static constexpr int sm100_dynamic_smem_even_3ch_max_bins = sm100_dynamic_smem_bytes / int{sizeof(unsigned int)}; +static constexpr int sm100_range_interpolation_min_bins = 512; // TODO(bgruber): drop in CCCL 4.0 enum class primitive_sample @@ -321,6 +336,9 @@ struct policy_hub static constexpr int init_kernel_pdl_trigger_max_bins = 2048; static constexpr int dynamic_smem_bytes = sm100_dynamic_smem_bytes; + static constexpr int dynamic_smem_range_max_bins = sm100_dynamic_smem_range_max_bins; + static constexpr int dynamic_smem_even_max_bins = sm100_dynamic_smem_even_max_bins; + static constexpr int dynamic_smem_even_3ch_max_bins = sm100_dynamic_smem_even_3ch_max_bins; }; using MaxPolicy = Policy1000; @@ -350,7 +368,11 @@ private: [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto sm100_policy(HistogramPolicy policy) const -> HistogramPolicy { - policy.dynamic_smem_bytes = sm100_dynamic_smem_bytes; + policy.dynamic_smem_bytes = sm100_dynamic_smem_bytes; + policy.dynamic_smem_range_max_bins = sm100_dynamic_smem_range_max_bins; + policy.dynamic_smem_even_max_bins = sm100_dynamic_smem_even_max_bins; + policy.dynamic_smem_even_3ch_max_bins = sm100_dynamic_smem_even_3ch_max_bins; + policy.range_interpolation_min_bins = sm100_range_interpolation_min_bins; return policy; } @@ -382,8 +404,8 @@ public: HistogramPolicy{768, t_scale(12), 1 << 2, BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false, 2048}); } - const int static_threads = sample_size_bytes >= 8 ? 384 : 768; - const int static_items = sample_size_bytes >= 8 ? t_scale(16) : 0; + const int static_threads = sample_size == 8 ? 384 : 768; + const int static_items = sample_size == 8 ? t_scale(16) : 0; return sm100_policy(HistogramPolicy{ 768, t_scale(12), diff --git a/cub/test/catch2_test_device_histogram.cu b/cub/test/catch2_test_device_histogram.cu index c345d2016e4a..b55ba7203ab3 100644 --- a/cub/test/catch2_test_device_histogram.cu +++ b/cub/test/catch2_test_device_histogram.cu @@ -576,6 +576,16 @@ CUB_TEST_LIST("DeviceHistogram::Histogram* channel configs", C2H_TEST("DeviceHistogram::Histogram* dynamic shared-memory privatization", "[histogram][device]") { + int current_device{}; + REQUIRE(cudaSuccess == cudaGetDevice(¤t_device)); + + cuda::compute_capability cc{}; + REQUIRE(cudaSuccess == cub::detail::ptx_compute_cap(cc, current_device)); + if (cc < cuda::compute_capability{10, 0}) + { + SKIP("The runtime-sized shared-memory histogram policy is currently tuned for SM100"); + } + using counter_t = unsigned long long; const int num_levels = GENERATE(1025, 4097); diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index 7727b7815436..3705fea3970a 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1630,6 +1630,62 @@ struct histogram_tuning } }; +template +struct histogram_tuning_with_local_counter : histogram_tuning +{ + using local_counter_type = LocalCounterT; +}; + +struct mixed_counter_histogram_tuning +{ + using local_counter_type = unsigned int; + + _CCCL_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy + { + cub::HistogramPolicy policy{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, cub::SMEM, false, 0}; + policy.dynamic_smem_bytes = 228352; + policy.dynamic_smem_even_max_bins = 8192; + policy.range_interpolation_min_bins = 512; + return policy; + } +}; + +static_assert( + cuda::std::is_same_v< + cub::detail::histogram::local_counter_t, unsigned long long>, + unsigned int>); +static_assert(cuda::std::is_same_v, unsigned long long>, + unsigned long long>); + +C2H_TEST("DeviceHistogram supports narrower local counters than output counters", "[histogram][device]") +{ + int current_device{}; + REQUIRE(cudaSuccess == cudaGetDevice(¤t_device)); + + cuda::compute_capability cc{}; + REQUIRE(cudaSuccess == cub::detail::ptx_compute_cap(cc, current_device)); + if (cc < cuda::compute_capability{10, 0}) + { + SKIP("The runtime-sized shared-memory histogram policy is currently tuned for SM100"); + } + + constexpr int num_samples = 4096; + constexpr int num_levels = num_samples + 1; + auto d_histogram = c2h::device_vector(num_samples, 0); + auto env = cuda::execution::tune(mixed_counter_histogram_tuning{}); + + histogram_even( + cuda::counting_iterator(0), + thrust::raw_pointer_cast(d_histogram.data()), + num_levels, + 0u, + static_cast(num_samples), + num_samples, + env); + + REQUIRE(d_histogram == c2h::host_vector(num_samples, 1)); +} + using block_sizes = c2h::type_list, cuda::std::integral_constant>; @@ -1750,7 +1806,22 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) // aggregate init constexpr auto p1 = cub::HistogramPolicy{ - 128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, cub::SMEM, false, 2048, 12345, 96, 3}; + 128, + 7, + 4, + cub::BLOCK_LOAD_DIRECT, + cub::CacheLoadModifier::LOAD_LDG, + false, + cub::SMEM, + false, + 2048, + 12345, + 96, + 3, + 1024, + 4096, + 8192, + 512}; # if _CCCL_STD_VER >= 2020 // designated init @@ -1766,7 +1837,11 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) .init_kernel_pdl_trigger_max_bins = 2048, .dynamic_smem_bytes = 12345, .static_smem_threads_per_block = 96, - .static_smem_items_per_thread = 3}; + .static_smem_items_per_thread = 3, + .dynamic_smem_range_max_bins = 1024, + .dynamic_smem_even_max_bins = 4096, + .dynamic_smem_even_3ch_max_bins = 8192, + .range_interpolation_min_bins = 512}; # else // _CCCL_STD_VER >= 2020 constexpr auto p2 = p1; # endif // _CCCL_STD_VER >= 2020 @@ -1780,12 +1855,15 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) os << p; return os.str(); }; - REQUIRE(to_string(p1) - == "HistogramPolicy { .threads_per_block = 128, .pixels_per_thread = 7, .vec_size = 4" - ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" - ", .mem_preference = SMEM, .use_work_stealing = 0, .init_kernel_pdl_trigger_max_bins = 2048" - ", .dynamic_smem_bytes = 12345, .static_smem_threads_per_block = 96" - ", .static_smem_items_per_thread = 3 }"); + REQUIRE( + to_string(p1) + == "HistogramPolicy { .threads_per_block = 128, .pixels_per_thread = 7, .vec_size = 4" + ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" + ", .mem_preference = SMEM, .use_work_stealing = 0, .init_kernel_pdl_trigger_max_bins = 2048" + ", .dynamic_smem_bytes = 12345, .static_smem_threads_per_block = 96" + ", .static_smem_items_per_thread = 3, .dynamic_smem_range_max_bins = 1024" + ", .dynamic_smem_even_max_bins = 4096, .dynamic_smem_even_3ch_max_bins = 8192" + ", .range_interpolation_min_bins = 512 }"); } C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget", "[histogram][device]") @@ -1801,6 +1879,19 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE(sm90_policy.dynamic_smem_bytes == 0); STATIC_REQUIRE(sm100_policy.dynamic_smem_bytes == 228352); STATIC_REQUIRE(sm100_wide_counter_policy.dynamic_smem_bytes == 228352); + STATIC_REQUIRE(sm100_policy.dynamic_smem_range_max_bins == 2048); + STATIC_REQUIRE(sm100_policy.dynamic_smem_even_max_bins == 8192); + STATIC_REQUIRE(sm100_policy.dynamic_smem_even_3ch_max_bins == 57088); + STATIC_REQUIRE(sm100_policy.range_interpolation_min_bins == 512); + + STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57088, 4, 1)); + STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57089, 4, 1)); + STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 2048, 4, 3)); + STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 2049, 4, 3)); + STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 8192, 4, 4)); + STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 8193, 4, 4)); + STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 19029, 4, 3)); + STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 19030, 4, 3)); using max_policy_t = typename cub::detail::histogram::policy_hub::MaxPolicy; const auto legacy_sm100_policy = From 05377439ad437076fe80aa3ba2779032ec98dc1e Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Thu, 30 Jul 2026 15:31:08 +0000 Subject: [PATCH 04/45] [cub] Avoid signed overflow in RANGE interpolation --- .../dispatch/kernels/kernel_histogram.cuh | 29 +++++++++++---- cub/test/catch2_test_device_histogram.cu | 36 +++++++++++++++++++ 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index ecb098692f49..903e5e65317e 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -20,6 +20,8 @@ #include #include +#include +#include CUB_NAMESPACE_BEGIN namespace detail::histogram @@ -35,6 +37,20 @@ struct Transforms template struct SearchTransform { + template + [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto interpolation_difference(T lhs, T rhs) + { + if constexpr (::cuda::std::is_integral_v) + { + using unsigned_t = ::cuda::std::make_unsigned_t; + return static_cast(lhs) - static_cast(rhs); + } + else + { + return lhs - rhs; + } + } + // Compile-time marker used by the privatized-SMEM sweep to select the RANGE // low-bin classifier and its launch bound. static constexpr bool is_range_transform = true; @@ -142,7 +158,7 @@ struct Transforms m_first = first; m_last = last; - m_inv_scale = static_cast(num_bins) / static_cast(last - first); + m_inv_scale = static_cast(num_bins) / static_cast(interpolation_difference(last, first)); m_have_precompute = true; // Three-point split at the midpoint bin. Read d_levels[mid] and derive @@ -161,8 +177,9 @@ struct Transforms { m_mid = mid; m_mid_bin = mid_bin; - m_inv_scale_lo = static_cast(mid_bin) / static_cast(mid - first); - m_inv_scale_hi = static_cast(num_bins - mid_bin) / static_cast(last - mid); + m_inv_scale_lo = static_cast(mid_bin) / static_cast(interpolation_difference(mid, first)); + m_inv_scale_hi = + static_cast(num_bins - mid_bin) / static_cast(interpolation_difference(last, mid)); } } } @@ -227,7 +244,7 @@ struct Transforms return; } - const auto delta = (s - first_level); + const auto delta = interpolation_difference(s, first_level); int guess; if (m_have_precompute) { @@ -239,7 +256,7 @@ struct Transforms } else { - const auto delta_hi = (s - m_mid); + const auto delta_hi = interpolation_difference(s, m_mid); guess = m_mid_bin + static_cast(static_cast(delta_hi) * m_inv_scale_hi); } } @@ -250,7 +267,7 @@ struct Transforms } else { - const auto range = (last_level - first_level); + const auto range = interpolation_difference(last_level, first_level); NV_IF_ELSE_TARGET( NV_IS_DEVICE, (guess = static_cast( diff --git a/cub/test/catch2_test_device_histogram.cu b/cub/test/catch2_test_device_histogram.cu index b55ba7203ab3..851d0ab446e1 100644 --- a/cub/test/catch2_test_device_histogram.cu +++ b/cub/test/catch2_test_device_histogram.cu @@ -675,6 +675,42 @@ CUB_TEST("DeviceHistogram::HistogramRange levels/samples aliasing", "[histogram_ } } +C2H_TEST("DeviceHistogram::HistogramRange interpolation avoids signed overflow", "[histogram_range][device]") +{ + using sample_t = int; + constexpr int num_bins = 512; + + c2h::host_vector h_levels(num_bins + 1); + constexpr auto lo = static_cast(cs::numeric_limits::lowest()); + constexpr auto hi = static_cast(cs::numeric_limits::max()); + constexpr auto range = hi - lo; + for (int i = 0; i <= num_bins; ++i) + { + h_levels[i] = static_cast(lo + (range * i) / num_bins); + } + + const c2h::host_vector h_samples{ + cs::numeric_limits::lowest(), -1, 0, 1, cs::numeric_limits::max() - 1}; + c2h::device_vector d_levels = h_levels; + c2h::device_vector d_samples = h_samples; + c2h::device_vector d_histogram(num_bins, 0); + + histogram_range( + thrust::raw_pointer_cast(d_samples.data()), + thrust::raw_pointer_cast(d_histogram.data()), + num_bins + 1, + thrust::raw_pointer_cast(d_levels.data()), + static_cast(d_samples.size())); + + c2h::host_vector expected(num_bins, 0); + for (const sample_t sample : h_samples) + { + const auto upper = std::upper_bound(h_levels.begin(), h_levels.end(), sample); + ++expected[static_cast(std::distance(h_levels.begin(), upper) - 1)]; + } + REQUIRE(d_histogram == expected); +} + // Limit this large-memory reproducer to the host launch path. #if TEST_LAUNCH == 0 CUB_TEST("DeviceHistogram::MultiHistogramEven large privatized offsets", "[histogram_even][device]", CUB_LARGE) From f64b4d5eb1ac940576a0bac18c3ec1e86d3443d5 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Thu, 30 Jul 2026 15:53:48 +0000 Subject: [PATCH 05/45] [cub] Restore static histogram privatization --- cub/cub/agent/agent_histogram.cuh | 1 + cub/cub/device/dispatch/kernels/kernel_histogram.cuh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index ec5e3e5dd0be..7eea502eb085 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -208,6 +208,7 @@ struct AgentHistogram { static_assert(sizeof(CounterT) <= sizeof(OutputCounterT), "The output histogram counter must be at least as wide as the local counter"); + static constexpr int privatized_smem_bins = PrivatizedSmemBins; static constexpr int vec_size = AgentHistogramPolicyT::VEC_SIZE; static constexpr int threads_per_block = AgentHistogramPolicyT::BLOCK_THREADS; static constexpr int pixels_per_thread = AgentHistogramPolicyT::PIXELS_PER_THREAD; diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index 903e5e65317e..3533255c766e 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -859,7 +859,7 @@ __launch_bounds__( hp.vec_size>; using AgentHistogramT = AgentHistogram; + static_assert(AgentHistogramT::privatized_smem_bins == PrivatizedSmemBins); // Shared memory for AgentHistogram __shared__ typename AgentHistogramT::TempStorage temp_storage; From 21a5c007343e76704e6ac1f230cc2ab4e1fc415b Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Thu, 30 Jul 2026 16:24:36 +0000 Subject: [PATCH 06/45] [cub] Keep histogram policy paths synchronized --- .../device/dispatch/dispatch_histogram.cuh | 30 ++++++++++- .../dispatch/kernels/kernel_histogram.cuh | 6 +++ .../dispatch/tuning/tuning_histogram.cuh | 51 ++++++++++++++----- cub/test/catch2_test_device_histogram_env.cu | 18 ++++++- 4 files changed, 89 insertions(+), 16 deletions(-) diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index c881722bb81f..9d9661ba774e 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -850,6 +850,32 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_bytes(long) return 0; } +template +_CCCL_HOST_DEVICE_API constexpr auto convert_static_smem_threads_per_block(int) + -> decltype(ActivePolicy::static_smem_threads_per_block) +{ + return ActivePolicy::static_smem_threads_per_block; +} + +template +_CCCL_HOST_DEVICE_API constexpr auto convert_static_smem_threads_per_block(long) +{ + return 0; +} + +template +_CCCL_HOST_DEVICE_API constexpr auto convert_static_smem_items_per_thread(int) + -> decltype(ActivePolicy::static_smem_items_per_thread) +{ + return ActivePolicy::static_smem_items_per_thread; +} + +template +_CCCL_HOST_DEVICE_API constexpr auto convert_static_smem_items_per_thread(long) +{ + return 0; +} + template _CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_range_max_bins(int) -> decltype(ActivePolicy::dynamic_smem_range_max_bins) @@ -918,8 +944,8 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_policy() -> HistogramPolicy ap::IS_WORK_STEALING, convert_pdl_trigger(0), convert_dynamic_smem_bytes(0), - 0, - 0, + convert_static_smem_threads_per_block(0), + convert_static_smem_items_per_thread(0), convert_dynamic_smem_range_max_bins(0), convert_dynamic_smem_even_max_bins(0), convert_dynamic_smem_even_3ch_max_bins(0), diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index 3533255c766e..b8efd9eee0f1 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -122,6 +122,12 @@ struct Transforms this->interpolation_min_bins = interpolation_min_bins_; this->m_have_precompute = false; this->m_inv_scale = 0.0f; + this->m_first = LevelT{}; + this->m_last = LevelT{}; + this->m_mid = LevelT{}; + this->m_inv_scale_lo = 0.0f; + this->m_inv_scale_hi = 0.0f; + this->m_mid_bin = 0; } //! @brief Hoist the loop-invariant interpolation state out of `BinSelect`. diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index ec4da1035911..f88bc321ce4b 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -247,26 +247,24 @@ struct sm100_tuning struct sm100_tuning { - static constexpr int items = 8; - static constexpr int threads = 512; + static constexpr int items = 6; + static constexpr int threads = 768; static constexpr bool rle_compress = true; static constexpr bool use_work_stealing = false; static constexpr BlockHistogramMemoryPreference mem_preference = SMEM; static constexpr CacheLoadModifier load_modifier = LOAD_LDG; - static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_VECTORIZE; + static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_DIRECT; static constexpr int vec_size = 1 << 2; }; -// sample_size 2 showed no benefit over SM90 during verification benchmarks - -// multi.even and multi.range: none of the found tunings surpassed the SM90 tuning during verification benchmarks +// sample_size 2 retains the SM90 launch shape while using the SM100 shared-memory policy. // TODO(bgruber): drop in CCCL 4.0 template @@ -329,16 +327,43 @@ struct policy_hub template _CCCL_HOST_DEVICE_API static auto select_agent_policy(long) -> typename Policy900::AgentHistogramPolicyT; - using AgentHistogramPolicyT = + using SelectedAgentHistogramPolicyT = decltype(select_agent_policy< sm100_tuning()>>( 0)); - static constexpr int init_kernel_pdl_trigger_max_bins = 2048; - static constexpr int dynamic_smem_bytes = sm100_dynamic_smem_bytes; - static constexpr int dynamic_smem_range_max_bins = sm100_dynamic_smem_range_max_bins; - static constexpr int dynamic_smem_even_max_bins = sm100_dynamic_smem_even_max_bins; - static constexpr int dynamic_smem_even_3ch_max_bins = sm100_dynamic_smem_even_3ch_max_bins; + using MultiChannelAgentHistogramPolicyT = + agent_histogram_policy<1024, t_scale(IsEven ? 8 : 16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false, 4>; + + static constexpr bool use_sm100_multi_channel_policy = + NumChannels >= 2 && sizeof(CounterT) == 4 && is_primitive::value; + + using AgentHistogramPolicyT = + ::cuda::std::_If; + + static constexpr int init_kernel_pdl_trigger_max_bins = + NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value + && (sizeof(SampleT) == 1 || sizeof(SampleT) == 2 || sizeof(SampleT) == 4 || sizeof(SampleT) == 8) + ? 2048 + : 0; + static constexpr int dynamic_smem_bytes = sm100_dynamic_smem_bytes; + static constexpr int dynamic_smem_range_max_bins = sm100_dynamic_smem_range_max_bins; + static constexpr int dynamic_smem_even_max_bins = sm100_dynamic_smem_even_max_bins; + static constexpr int dynamic_smem_even_3ch_max_bins = sm100_dynamic_smem_even_3ch_max_bins; + static constexpr int range_interpolation_min_bins = sm100_range_interpolation_min_bins; + static constexpr int static_smem_threads_per_block = + !IsEven && sizeof(CounterT) == 4 && is_primitive::value + ? (NumChannels >= 2 + ? 384 + : (NumChannels == 1 && NumActiveChannels == 1 && sizeof(SampleT) == 4 + ? 768 + : (NumChannels == 1 && NumActiveChannels == 1 && sizeof(SampleT) == 8 ? 384 : 0))) + : 0; + static constexpr int static_smem_items_per_thread = + !IsEven && NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value + && sizeof(SampleT) == 8 + ? t_scale(16) + : 0; }; using MaxPolicy = Policy1000; diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index 3705fea3970a..b27f45d6cb0f 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1896,6 +1896,22 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" using max_policy_t = typename cub::detail::histogram::policy_hub::MaxPolicy; const auto legacy_sm100_policy = cub::detail::histogram::policy_selector_from_max_policy{}(cuda::compute_capability{10, 0}); - REQUIRE(legacy_sm100_policy.dynamic_smem_bytes == 228352); + REQUIRE(legacy_sm100_policy == sm100_policy); + + using range_max_policy_t = + typename cub::detail::histogram::policy_hub::MaxPolicy; + const auto legacy_range_policy = + cub::detail::histogram::policy_selector_from_max_policy{}(cuda::compute_capability{10, 0}); + constexpr auto range_policy = + cub::detail::histogram::policy_selector_from_types{}( + cuda::compute_capability{10, 0}); + REQUIRE(legacy_range_policy == range_policy); + + using multi_max_policy_t = typename cub::detail::histogram::policy_hub::MaxPolicy; + const auto legacy_multi_policy = + cub::detail::histogram::policy_selector_from_max_policy{}(cuda::compute_capability{10, 0}); + constexpr auto multi_policy = cub::detail::histogram::policy_selector_from_types{}( + cuda::compute_capability{10, 0}); + REQUIRE(legacy_multi_policy == multi_policy); } #endif // _CCCL_COMPILER(GCC, >=, 8) From e7da7b4c8a2c3b8c70ef7758ce27d9694c3764a0 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Thu, 30 Jul 2026 19:57:59 +0000 Subject: [PATCH 07/45] CUB: split RANGE histogram bin selection transforms --- cub/cub/agent/agent_histogram.cuh | 53 ++--- .../device/dispatch/dispatch_histogram.cuh | 67 ++++-- .../dispatch/kernels/kernel_histogram.cuh | 221 +++++++----------- .../dispatch/tuning/tuning_histogram.cuh | 49 ++-- cub/test/catch2_test_device_histogram_env.cu | 24 +- 5 files changed, 207 insertions(+), 207 deletions(-) diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index 7eea502eb085..a73718b187ec 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -188,7 +188,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE auto NativePointer(IteratorT itr) //! @tparam OffsetT //! Signed integer type for global offsets //! -//! @tparam UseDynamicSmemHistogram +//! @tparam UseDynamicSmem //! Whether the privatized histogram is supplied separately in dynamic shared memory. //! //! @tparam OutputCounterT @@ -202,8 +202,8 @@ template + bool UseDynamicSmem = false, + typename OutputCounterT = CounterT> struct AgentHistogram { static_assert(sizeof(CounterT) <= sizeof(OutputCounterT), @@ -328,35 +328,13 @@ struct AgentHistogram // Bin pixels int bins[pixels_per_thread]; - if constexpr (UseDynamicSmemHistogram) - { - typename PrivatizedDecodeOpT::BracketCacheT mru; - _CCCL_PRAGMA_UNROLL_FULL() - for (int pixel = 0; pixel < pixels_per_thread; ++pixel) - { - bins[pixel] = -1; - privatized_decode_op[ch].template BinSelect( - samples[pixel][ch], bins[pixel], is_valid[pixel], mru); - } - } - else if constexpr (PrivatizedSmemBins > 0 && PrivatizedDecodeOpT::is_range_transform) - { - _CCCL_PRAGMA_UNROLL_FULL() - for (int pixel = 0; pixel < pixels_per_thread; ++pixel) - { - bins[pixel] = -1; - privatized_decode_op[ch].template BinSelectStaticLean( - samples[pixel][ch], bins[pixel], is_valid[pixel]); - } - } - else + typename PrivatizedDecodeOpT::BinSelectState bin_select_state; + _CCCL_PRAGMA_UNROLL_FULL() + for (int pixel = 0; pixel < pixels_per_thread; ++pixel) { - _CCCL_PRAGMA_UNROLL_FULL() - for (int pixel = 0; pixel < pixels_per_thread; ++pixel) - { - bins[pixel] = -1; - privatized_decode_op[ch].template BinSelect(samples[pixel][ch], bins[pixel], is_valid[pixel]); - } + bins[pixel] = -1; + privatized_decode_op[ch].template BinSelect( + samples[pixel][ch], bins[pixel], is_valid[pixel], bin_select_state); } CounterT accumulator = 1; @@ -509,7 +487,7 @@ struct AgentHistogram if (prefer_smem) { - if constexpr (UseDynamicSmemHistogram) + if constexpr (UseDynamicSmem) { AccumulatePixels(samples, is_valid, smem_histograms, ::cuda::std::bool_constant{}); } @@ -673,8 +651,8 @@ struct AgentHistogram : // prefer gmem privatized histograms blockIdx.x & 1) // prefer blended privatized histograms { - static_assert(!UseDynamicSmemHistogram, - "AgentHistogram with UseDynamicSmemHistogram=true requires the dynamic-SMEM " + static_assert(!UseDynamicSmem, + "AgentHistogram with UseDynamicSmem=true requires the dynamic-SMEM " "constructor that takes an extern __shared__ base pointer."); const int blockId = static_cast((blockIdx.y * gridDim.x) + blockIdx.x); @@ -709,8 +687,7 @@ struct AgentHistogram , privatized_decode_op(privatized_decode_op) , prefer_smem(true) { - static_assert(UseDynamicSmemHistogram, - "Dynamic-SMEM AgentHistogram constructor requires UseDynamicSmemHistogram=true."); + static_assert(UseDynamicSmem, "Dynamic-SMEM AgentHistogram constructor requires UseDynamicSmem=true."); for (int ch = 0; ch < NumActiveChannels; ++ch) { @@ -783,7 +760,7 @@ struct AgentHistogram { if (prefer_smem) { - if constexpr (UseDynamicSmemHistogram) + if constexpr (UseDynamicSmem) { ZeroBinCounters(smem_histograms); } @@ -803,7 +780,7 @@ struct AgentHistogram { if (prefer_smem) { - if constexpr (UseDynamicSmemHistogram) + if constexpr (UseDynamicSmem) { StoreOutput(smem_histograms); } diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 9d9661ba774e..1a51c3af27e7 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -211,7 +211,10 @@ should_use_dynamic_smem(const HistogramPolicy& policy, int num_bins, int counter { if constexpr (IsEven) { - max_bins = num_active_channels <= 3 ? policy.dynamic_smem_even_3ch_max_bins : policy.dynamic_smem_even_max_bins; + max_bins = num_active_channels == 2 ? policy.dynamic_smem_even_2ch_max_bins + : num_active_channels == 3 + ? policy.dynamic_smem_even_3ch_max_bins + : policy.dynamic_smem_even_4ch_max_bins; } else { @@ -876,6 +879,19 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_static_smem_items_per_thread(long) return 0; } +template +_CCCL_HOST_DEVICE_API constexpr auto convert_static_smem_min_blocks_per_sm(int) + -> decltype(ActivePolicy::static_smem_min_blocks_per_sm) +{ + return ActivePolicy::static_smem_min_blocks_per_sm; +} + +template +_CCCL_HOST_DEVICE_API constexpr auto convert_static_smem_min_blocks_per_sm(long) +{ + return 0; +} + template _CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_range_max_bins(int) -> decltype(ActivePolicy::dynamic_smem_range_max_bins) @@ -890,14 +906,14 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_range_max_bins(long) } template -_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_max_bins(int) - -> decltype(ActivePolicy::dynamic_smem_even_max_bins) +_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_2ch_max_bins(int) + -> decltype(ActivePolicy::dynamic_smem_even_2ch_max_bins) { - return ActivePolicy::dynamic_smem_even_max_bins; + return ActivePolicy::dynamic_smem_even_2ch_max_bins; } template -_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_max_bins(long) +_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_2ch_max_bins(long) { return 0; } @@ -915,6 +931,19 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_3ch_max_bins(long return 0; } +template +_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_4ch_max_bins(int) + -> decltype(ActivePolicy::dynamic_smem_even_4ch_max_bins) +{ + return ActivePolicy::dynamic_smem_even_4ch_max_bins; +} + +template +_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_4ch_max_bins(long) +{ + return 0; +} + template _CCCL_HOST_DEVICE_API constexpr auto convert_range_interpolation_min_bins(int) -> decltype(ActivePolicy::range_interpolation_min_bins) @@ -946,9 +975,11 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_policy() -> HistogramPolicy convert_dynamic_smem_bytes(0), convert_static_smem_threads_per_block(0), convert_static_smem_items_per_thread(0), + convert_static_smem_min_blocks_per_sm(0), convert_dynamic_smem_range_max_bins(0), - convert_dynamic_smem_even_max_bins(0), + convert_dynamic_smem_even_2ch_max_bins(0), convert_dynamic_smem_even_3ch_max_bins(0), + convert_dynamic_smem_even_4ch_max_bins(0), convert_range_interpolation_min_bins(0)}; } @@ -1045,8 +1076,7 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel) { num_privatized_levels[channel] = 257; - output_decode_op[channel].Init( - d_levels[channel], num_output_levels[channel], active_policy.range_interpolation_min_bins); + output_decode_op[channel].Init(d_levels[channel], num_output_levels[channel]); if (num_output_levels[channel] > max_levels) { @@ -1089,20 +1119,14 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( { using TransformsT = Transforms; - // Use the search transform op for converting samples to privatized bins - using PrivatizedDecodeOpT = typename TransformsT::template SearchTransform; - // Use the pass-thru transform op for converting privatized bins to output bins using OutputDecodeOpT = typename TransformsT::PassThruTransform; - ::cuda::std::array privatized_decode_op{}; ::cuda::std::array output_decode_op{}; int max_levels = num_output_levels[0]; for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel) { - privatized_decode_op[channel].Init( - d_levels[channel], num_output_levels[channel], active_policy.range_interpolation_min_bins); if (num_output_levels[channel] > max_levels) { max_levels = num_output_levels[channel]; @@ -1113,6 +1137,14 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( if (should_use_dynamic_smem( active_policy, max_num_output_bins, int{kernel_source.CounterSize()}, NUM_ACTIVE_CHANNELS)) { + using PrivatizedDecodeOpT = typename TransformsT::template CachedSearchTransform; + ::cuda::std::array privatized_decode_op{}; + for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel) + { + privatized_decode_op[channel].Init( + d_levels[channel], num_output_levels[channel], active_policy.range_interpolation_min_bins); + } + return CubDebug( (detail::histogram::dispatch; + ::cuda::std::array privatized_decode_op{}; + for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel) + { + privatized_decode_op[channel].Init(d_levels[channel], num_output_levels[channel]); + } + // Dispatch if (max_num_output_bins > max_privatized_smem_bins) { diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index b8efd9eee0f1..acdd8d0a9d2f 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -33,10 +33,66 @@ struct Transforms // Transform functors for converting samples to bin-ids //--------------------------------------------------------------------- - // Searches for bin given a list of bin-boundary levels + //! @brief Finds a RANGE bin with binary search. + //! + //! Uses `UpperBound` without interpolation or per-thread state. template struct SearchTransform { + struct BinSelectState + {}; + + LevelIteratorT d_levels; // Pointer to levels array + int num_output_levels; // Number of levels in array + + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void Init(LevelIteratorT d_levels_, int num_output_levels_) + { + d_levels = d_levels_; + num_output_levels = num_output_levels_; + } + + _CCCL_DEVICE _CCCL_FORCEINLINE void PrecomputeOnDevice() {} + + template + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(SampleT2 sample, int& bin, bool valid) const + { + BinSelectState state; + BinSelect(sample, bin, valid, state); + } + + template + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(SampleT2 sample, int& bin, bool valid, BinSelectState&) const + { + using WrappedLevelIteratorT = + ::cuda::std::_If<::cuda::std::is_pointer_v, + CacheModifiedInputIterator, + LevelIteratorT>; + WrappedLevelIteratorT wrapped_levels(d_levels); + + const int num_bins = num_output_levels - 1; + if (valid) + { + bin = UpperBound(wrapped_levels, num_output_levels, static_cast(sample)) - 1; + if (bin >= num_bins) + { + bin = -1; + } + } + } + }; + + //! @brief Finds a RANGE bin with interpolation and a per-thread bracket cache. + //! + //! This transform is used by the runtime-sized shared-memory kernel. It + //! precomputes interpolation parameters once per thread, verifies each + //! interpolated guess against the level array, and falls back to binary + //! search for irregular levels. `BinSelectState` remembers the most recently + //! resolved bracket so consecutive samples in that bracket require no level + //! loads. + template + struct CachedSearchTransform + { + //! @brief Computes a non-negative interpolation distance without signed overflow. template [[nodiscard]] _CCCL_HOST_DEVICE_API static constexpr auto interpolation_difference(T lhs, T rhs) { @@ -51,25 +107,7 @@ struct Transforms } } - // Compile-time marker used by the privatized-SMEM sweep to select the RANGE - // low-bin classifier and its launch bound. - static constexpr bool is_range_transform = true; - - //! @brief Per-thread most-recently-used (MRU) bin-bracket cache. - //! - //! Carries the last successfully-resolved bin and its two boundary level - //! values across consecutive `BinSelect` calls so a new sample that falls in - //! the same `[lo, hi)` bracket is classified with ZERO level-array loads (a - //! handful of register compares), skipping the interpolated first-guess, the - //! clamp, and -- crucially -- both verify loads on the dependent - //! `IMAD.WIDE -> LDG` level-load chain that binds the latency-bound RANGE - //! classify. Low-entropy inputs (constant or heavily-skewed samples) have high - //! consecutive-sample locality, so the bracket hits dominate. A `bin < 0` - //! sentinel marks the cache empty. - //! This is per-thread mutable state, so the dynamic-SMEM sweep copies each - //! transform into thread-local storage before using the cache. The static - //! sweep reads the shared transform without a cache. - struct BracketCacheT + struct BinSelectState { LevelT lo; // cached d_levels[bin] LevelT hi; // cached d_levels[bin + 1] @@ -80,31 +118,13 @@ struct Transforms int num_output_levels; // Number of levels in array int interpolation_min_bins; // policy-selected minimum bin count for interpolation - // Precomputed (loop-invariant) interpolation state, populated by - // `PrecomputeOnDevice()`. The interpolation slope `num_bins / (last - - // first)` and the boundary levels are uniform across all samples a thread - // classifies, but the original `BinSelect` recomputed them per sample - // (two cache loads for the endpoints plus a `__fdividef` MUFU.RCP on the - // critical dependency chain). Hoisting them out turns the per-sample - // first-guess into a single `(float)delta * m_inv_scale` FMA and removes - // the two endpoint loads, which is the dominant cost on the ALU/XU-bound - // RANGE classify. `m_have_precompute == false` keeps the original - // per-sample path so host-only initialization (no device pointer to - // dereference) and tiny bin counts remain correct. + // Interpolation state shared by all samples processed by a thread. float m_inv_scale; // num_bins / (float)(last - first); valid iff m_have_precompute LevelT m_first; // cached d_levels[0] LevelT m_last; // cached d_levels[num_bins] bool m_have_precompute; // whether the fields above are valid - // Three-point (piecewise-linear) interpolation state, populated by - // PrecomputeOnDevice alongside the single-secant fields above. Splitting - // the [first,last] range at the midpoint level d_levels[mid] and - // interpolating on whichever half the sample falls in (a) halves the - // magnitude of `delta` fed to the lossy 32-bit float guess -- so the - // first-guess lands closer to the true bin and the verify-or-1-step ladder - // converges without reaching UpperBound -- and (b) captures large-scale - // non-uniformity (a slope change between the two halves) that a single - // first->last secant cannot. `m_mid_bin` is the bin index at the split. + // Piecewise-linear interpolation state split at the midpoint level. LevelT m_mid; // cached d_levels[mid_bin] float m_inv_scale_lo; // mid_bin / (float)(mid - first) float m_inv_scale_hi; // (num_bins - mid_bin) / (float)(last - mid) @@ -130,15 +150,7 @@ struct Transforms this->m_mid_bin = 0; } - //! @brief Hoist the loop-invariant interpolation state out of `BinSelect`. - //! - //! Must be called on the device (it dereferences the device level array) - //! once per thread before the sweep loop. Reads the first and last level, - //! validates strict monotonicity of the endpoints and a usable bin count, - //! and on success precomputes the float reciprocal slope so the hot path - //! avoids a per-sample `__fdividef` and two endpoint loads. On any - //! degenerate input it leaves `m_have_precompute == false`, so `BinSelect` - //! transparently falls back to the original (fully general) path. + //! @brief Precomputes interpolation slopes from the device level array. _CCCL_DEVICE _CCCL_FORCEINLINE void PrecomputeOnDevice() { const int num_bins = num_output_levels - 1; @@ -167,10 +179,7 @@ struct Transforms m_inv_scale = static_cast(num_bins) / static_cast(interpolation_difference(last, first)); m_have_precompute = true; - // Three-point split at the midpoint bin. Read d_levels[mid] and derive - // the two half-slopes. If either half is degenerate (non-increasing), - // fall back to the single-secant guess by setting m_mid_bin = 0, which - // BinSelect treats as "no split". + // Use a single secant if the midpoint does not split the level range. m_mid_bin = 0; m_inv_scale_lo = m_inv_scale; m_inv_scale_hi = m_inv_scale; @@ -191,17 +200,22 @@ struct Transforms } private: - struct NoBracketCacheT + struct NoBinSelectState {}; + //! @brief Implements cached/interpolated bin selection. + //! + //! A cached-bracket hit returns immediately. Otherwise, this computes and + //! verifies an interpolated guess, checks one adjacent bracket, and finally + //! falls back to `UpperBound` for arbitrary level distributions. template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void Classify(_SampleT sample, int& bin, bool valid, CacheT& cache) const + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelectImpl(_SampleT sample, int& bin, bool valid, CacheT& cache) const { using WrappedLevelIteratorT = ::cuda::std::_If<::cuda::std::is_pointer_v, CacheModifiedInputIterator, LevelIteratorT>; - constexpr bool use_cache = ::cuda::std::is_same_v; + constexpr bool use_cache = ::cuda::std::is_same_v; WrappedLevelIteratorT wrapped_levels(d_levels); const int num_bins = num_output_levels - 1; @@ -298,7 +312,7 @@ struct Transforms bin = guess; if constexpr (use_cache) { - cache = BracketCacheT{lvl_lo, lvl_hi, guess}; + cache = BinSelectState{lvl_lo, lvl_hi, guess}; } return; } @@ -314,7 +328,7 @@ struct Transforms bin = g2; if constexpr (use_cache) { - cache = BracketCacheT{lvl2_lo, lvl_lo, g2}; + cache = BinSelectState{lvl2_lo, lvl_lo, g2}; } return; } @@ -331,7 +345,7 @@ struct Transforms bin = g2; if constexpr (use_cache) { - cache = BracketCacheT{lvl_hi, lvl2_hi, g2}; + cache = BinSelectState{lvl_hi, lvl2_hi, g2}; } return; } @@ -348,77 +362,31 @@ struct Transforms { if (bin >= 0) { - cache = BracketCacheT{wrapped_levels[bin], wrapped_levels[bin + 1], bin}; + cache = BinSelectState{wrapped_levels[bin], wrapped_levels[bin + 1], bin}; } } } public: - // Method for converting samples to bin-ids + //! @brief Selects a bin without retaining bracket state. template _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid) const { - NoBracketCacheT cache; - Classify(sample, bin, valid, cache); - } - - //! @brief Lean classify for the STATIC <=256-bin SMEM tier. - //! - //! Byte-identical to upstream `main`'s flat `BinSelect`: a single `UpperBound` - //! + clamp, with NONE of the interpolation machinery the 3-arg `BinSelect` above - //! carries (no `num_bins < interpolation_min_bins` runtime branch, no reads of the - //! precompute fields `m_inv_scale`/`m_first`/...). At <=256 bins the interpolation - //! fast-path never activates (`m_have_precompute` stays false there), so that - //! machinery is pure dead weight: its extra branch + the wider codegen/register - //! footprint measurably slow the latency/occupancy-bound static kernel (~1-3% vs - //! main, confirmed by A/B). The dynamic-SMEM tier (bins >= 512) keeps the full - //! interpolated `BinSelect`/MRU path, where that machinery pays off. EVEN's - //! ScaleTransform is unaffected (it has its own cheap classify). - template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelectStaticLean(_SampleT sample, int& bin, bool valid) const - { - using WrappedLevelIteratorT = - ::cuda::std::_If<::cuda::std::is_pointer_v, - CacheModifiedInputIterator, - LevelIteratorT>; - WrappedLevelIteratorT wrapped_levels(d_levels); - - const int num_bins = num_output_levels - 1; - if (valid) - { - bin = UpperBound(wrapped_levels, num_output_levels, static_cast(sample)) - 1; - if (bin >= num_bins) - { - bin = -1; - } - } + NoBinSelectState cache; + BinSelectImpl(sample, bin, valid, cache); } - //! @brief MRU-bracket-cached `BinSelect`. - //! - //! Same contract and result as the plain `BinSelect` above, but threads a - //! per-thread `BracketCacheT` across calls to exploit consecutive-sample - //! temporal locality. The fast path tests the cached `[lo, hi)` bracket with - //! register compares only -- on a hit it returns the cached bin without ANY - //! level-array load, cutting the dependent `IMAD.WIDE -> LDG` chain that - //! binds the high-bin RANGE classify. On a miss it runs the identical - //! interpolated-guess / verify / 1-step / `UpperBound` ladder as the plain - //! path (so correctness, including the non-uniform-level fallback, is - //! unchanged) and then records the resolved bracket -- reusing the bracket - //! levels the ladder already loaded in the common verify/1-step cases, and - //! reloading only on the rare `UpperBound` fallback. + //! @brief Selects a bin and updates the per-thread bracket state. template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid, BracketCacheT& mru) const + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid, BinSelectState& mru) const { - Classify(sample, bin, valid, mru); + BinSelectImpl(sample, bin, valid, mru); } }; // Scales samples to evenly-spaced bins struct ScaleTransform { - static constexpr bool is_range_transform = false; - using CommonT = ::cuda::std::common_type_t; static_assert(::cuda::std::is_convertible_v, "The common type of `LevelT` and `SampleT` must be " @@ -609,7 +577,7 @@ struct Transforms _CCCL_DEVICE _CCCL_FORCEINLINE void PrecomputeOnDevice() {} - struct BracketCacheT + struct BinSelectState {}; // Method for converting samples to bin-ids @@ -625,7 +593,7 @@ struct Transforms } template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(SampleT sample, int& bin, bool valid, BracketCacheT&) const + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(SampleT sample, int& bin, bool valid, BinSelectState&) const { this->template BinSelect(sample, bin, valid); } @@ -634,8 +602,6 @@ struct Transforms // Pass-through bin transform operator struct PassThruTransform { - static constexpr bool is_range_transform = false; - // GCC 14 rightfully warns that when a value-initialized array of this struct is copied using memcpy, uninitialized // bytes may be accessed. To avoid this, we add a dummy member, so value initialization actually initializes the memory. #if _CCCL_COMPILER(GCC, >=, 13) @@ -654,7 +620,7 @@ struct Transforms _CCCL_DEVICE _CCCL_FORCEINLINE void PrecomputeOnDevice() {} - struct BracketCacheT + struct BinSelectState {}; // Method for converting samples to bin-ids @@ -668,7 +634,7 @@ struct Transforms } template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid, BracketCacheT&) const + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid, BinSelectState&) const { this->template BinSelect(sample, bin, valid); } @@ -824,17 +790,10 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__( - int(PrivatizedSmemBins > 0 ? current_policy().static_smem_threads() - : current_policy().threads_per_block), - (PrivatizedSmemBins > 0 && PrivatizedDecodeOpT::is_range_transform - && current_policy().static_smem_threads() < 512) - ? 3 - : (((PrivatizedSmemBins > 0 ? current_policy().static_smem_threads() - : current_policy().threads_per_block) - >= 512) - ? 2 - : 0)) +__launch_bounds__(int(PrivatizedSmemBins > 0 ? current_policy().static_smem_threads() + : current_policy().threads_per_block), + int(PrivatizedSmemBins > 0 ? current_policy().static_smem_min_blocks() + : (current_policy().threads_per_block >= 512 ? 2 : 0))) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepKernel( const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, @@ -1125,8 +1084,8 @@ __launch_bounds__(int(current_policy().threads_per_block)) { const auto num_output_levels = first_level_array[channel]; const auto levels = second_level_array[channel]; - privatized_decode_op[channel].Init(levels, num_output_levels, hp.range_interpolation_min_bins); - output_decode_op[channel].Init(levels, num_output_levels, hp.range_interpolation_min_bins); + privatized_decode_op[channel].Init(levels, num_output_levels); + output_decode_op[channel].Init(levels, num_output_levels); } } diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index f88bc321ce4b..58b56bd6cb50 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -42,9 +42,11 @@ struct HistogramPolicy int dynamic_smem_bytes = 0; //!< Tuned byte budget for a runtime-sized privatized histogram; 0 disables it int static_smem_threads_per_block = 0; //!< Static shared-memory tier threads; 0 inherits threads_per_block int static_smem_items_per_thread = 0; //!< Static shared-memory tier items; 0 inherits pixels_per_thread + int static_smem_min_blocks_per_sm = 0; //!< Static shared-memory launch bound; 0 derives it from the block size int dynamic_smem_range_max_bins = 0; //!< Multi-channel RANGE cap per channel; 0 disables the dynamic path - int dynamic_smem_even_max_bins = 0; //!< Multi-channel EVEN cap per channel; 0 disables the dynamic path - int dynamic_smem_even_3ch_max_bins = 0; //!< Extended EVEN cap when at most three channels are active + int dynamic_smem_even_2ch_max_bins = 0; //!< Two-channel EVEN cap per channel; 0 disables the dynamic path + int dynamic_smem_even_3ch_max_bins = 0; //!< Three-channel EVEN cap per channel; 0 disables the dynamic path + int dynamic_smem_even_4ch_max_bins = 0; //!< Four-channel EVEN cap per channel; 0 disables the dynamic path int range_interpolation_min_bins = 0; //!< Minimum RANGE bin count for interpolation; 0 disables interpolation [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int static_smem_threads() const @@ -57,6 +59,11 @@ struct HistogramPolicy return static_smem_items_per_thread != 0 ? static_smem_items_per_thread : pixels_per_thread; } + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int static_smem_min_blocks() const + { + return static_smem_min_blocks_per_sm != 0 ? static_smem_min_blocks_per_sm : (static_smem_threads() >= 512 ? 2 : 0); + } + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(const HistogramPolicy& lhs, const HistogramPolicy& rhs) noexcept { @@ -68,9 +75,11 @@ struct HistogramPolicy && lhs.dynamic_smem_bytes == rhs.dynamic_smem_bytes && lhs.static_smem_threads_per_block == rhs.static_smem_threads_per_block && lhs.static_smem_items_per_thread == rhs.static_smem_items_per_thread + && lhs.static_smem_min_blocks_per_sm == rhs.static_smem_min_blocks_per_sm && lhs.dynamic_smem_range_max_bins == rhs.dynamic_smem_range_max_bins - && lhs.dynamic_smem_even_max_bins == rhs.dynamic_smem_even_max_bins + && lhs.dynamic_smem_even_2ch_max_bins == rhs.dynamic_smem_even_2ch_max_bins && lhs.dynamic_smem_even_3ch_max_bins == rhs.dynamic_smem_even_3ch_max_bins + && lhs.dynamic_smem_even_4ch_max_bins == rhs.dynamic_smem_even_4ch_max_bins && lhs.range_interpolation_min_bins == rhs.range_interpolation_min_bins; } @@ -90,9 +99,12 @@ struct HistogramPolicy << ", .mem_preference = " << p.mem_preference << ", .use_work_stealing = " << p.use_work_stealing << ", .init_kernel_pdl_trigger_max_bins = " << p.init_kernel_pdl_trigger_max_bins << ", .dynamic_smem_bytes = " << p.dynamic_smem_bytes << ", .static_smem_threads_per_block = " << p.static_smem_threads_per_block - << ", .static_smem_items_per_thread = " << p.static_smem_items_per_thread << ", .dynamic_smem_range_max_bins = " - << p.dynamic_smem_range_max_bins << ", .dynamic_smem_even_max_bins = " << p.dynamic_smem_even_max_bins + << ", .static_smem_items_per_thread = " << p.static_smem_items_per_thread + << ", .static_smem_min_blocks_per_sm = " << p.static_smem_min_blocks_per_sm + << ", .dynamic_smem_range_max_bins = " << p.dynamic_smem_range_max_bins + << ", .dynamic_smem_even_2ch_max_bins = " << p.dynamic_smem_even_2ch_max_bins << ", .dynamic_smem_even_3ch_max_bins = " << p.dynamic_smem_even_3ch_max_bins + << ", .dynamic_smem_even_4ch_max_bins = " << p.dynamic_smem_even_4ch_max_bins << ", .range_interpolation_min_bins = " << p.range_interpolation_min_bins << " }"; } #endif // _CCCL_HOSTED() @@ -100,13 +112,12 @@ struct HistogramPolicy namespace detail::histogram { -// B200 exposes 232448 bytes of opt-in shared memory per block. Autoresearch -// retained 4096 bytes for static kernel storage and driver bookkeeping, leaving -// 228352 bytes for the runtime-sized privatized histogram. +// Leave 4096 bytes of the SM100 opt-in shared-memory limit available for static storage. static constexpr int sm100_dynamic_smem_bytes = 232448 - 4096; static constexpr int sm100_dynamic_smem_range_max_bins = 2048; -static constexpr int sm100_dynamic_smem_even_max_bins = 8192; -static constexpr int sm100_dynamic_smem_even_3ch_max_bins = sm100_dynamic_smem_bytes / int{sizeof(unsigned int)}; +static constexpr int sm100_dynamic_smem_even_2ch_max_bins = 28544; +static constexpr int sm100_dynamic_smem_even_3ch_max_bins = 19029; +static constexpr int sm100_dynamic_smem_even_4ch_max_bins = 8192; static constexpr int sm100_range_interpolation_min_bins = 512; // TODO(bgruber): drop in CCCL 4.0 @@ -348,8 +359,9 @@ struct policy_hub : 0; static constexpr int dynamic_smem_bytes = sm100_dynamic_smem_bytes; static constexpr int dynamic_smem_range_max_bins = sm100_dynamic_smem_range_max_bins; - static constexpr int dynamic_smem_even_max_bins = sm100_dynamic_smem_even_max_bins; + static constexpr int dynamic_smem_even_2ch_max_bins = sm100_dynamic_smem_even_2ch_max_bins; static constexpr int dynamic_smem_even_3ch_max_bins = sm100_dynamic_smem_even_3ch_max_bins; + static constexpr int dynamic_smem_even_4ch_max_bins = sm100_dynamic_smem_even_4ch_max_bins; static constexpr int range_interpolation_min_bins = sm100_range_interpolation_min_bins; static constexpr int static_smem_threads_per_block = !IsEven && sizeof(CounterT) == 4 && is_primitive::value @@ -364,6 +376,8 @@ struct policy_hub && sizeof(SampleT) == 8 ? t_scale(16) : 0; + static constexpr int static_smem_min_blocks_per_sm = + !IsEven && static_smem_threads_per_block > 0 && static_smem_threads_per_block < 512 ? 3 : 0; }; using MaxPolicy = Policy1000; @@ -395,8 +409,9 @@ private: { policy.dynamic_smem_bytes = sm100_dynamic_smem_bytes; policy.dynamic_smem_range_max_bins = sm100_dynamic_smem_range_max_bins; - policy.dynamic_smem_even_max_bins = sm100_dynamic_smem_even_max_bins; + policy.dynamic_smem_even_2ch_max_bins = sm100_dynamic_smem_even_2ch_max_bins; policy.dynamic_smem_even_3ch_max_bins = sm100_dynamic_smem_even_3ch_max_bins; + policy.dynamic_smem_even_4ch_max_bins = sm100_dynamic_smem_even_4ch_max_bins; policy.range_interpolation_min_bins = sm100_range_interpolation_min_bins; return policy; } @@ -429,8 +444,9 @@ public: HistogramPolicy{768, t_scale(12), 1 << 2, BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false, 2048}); } - const int static_threads = sample_size == 8 ? 384 : 768; - const int static_items = sample_size == 8 ? t_scale(16) : 0; + const int static_threads = sample_size == 8 ? 384 : 768; + const int static_items = sample_size == 8 ? t_scale(16) : 0; + const int static_min_blocks = static_threads < 512 ? 3 : 0; return sm100_policy(HistogramPolicy{ 768, t_scale(12), @@ -443,7 +459,8 @@ public: 2048, 0, static_threads, - static_items}); + static_items, + static_min_blocks}); } if (num_channels >= 2 && counter_size == 4 && sample_is_primitive) @@ -453,7 +470,7 @@ public: return sm100_policy(HistogramPolicy{1024, t_scale(8), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false, 0}); } return sm100_policy( - HistogramPolicy{1024, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false, 0, 0, 384}); + HistogramPolicy{1024, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false, 0, 0, 384, 0, 3}); } if (num_channels == 1 && num_active_channels == 1 && counter_size == 4 && sample_is_primitive && sample_size == 2) diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index b27f45d6cb0f..3762fa7ac7ba 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1643,9 +1643,9 @@ struct mixed_counter_histogram_tuning _CCCL_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { cub::HistogramPolicy policy{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, cub::SMEM, false, 0}; - policy.dynamic_smem_bytes = 228352; - policy.dynamic_smem_even_max_bins = 8192; - policy.range_interpolation_min_bins = 512; + policy.dynamic_smem_bytes = 228352; + policy.dynamic_smem_even_4ch_max_bins = 8192; + policy.range_interpolation_min_bins = 512; return policy; } }; @@ -1818,9 +1818,11 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) 12345, 96, 3, + 2, 1024, 4096, 8192, + 16384, 512}; # if _CCCL_STD_VER >= 2020 @@ -1838,9 +1840,11 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) .dynamic_smem_bytes = 12345, .static_smem_threads_per_block = 96, .static_smem_items_per_thread = 3, + .static_smem_min_blocks_per_sm = 2, .dynamic_smem_range_max_bins = 1024, - .dynamic_smem_even_max_bins = 4096, + .dynamic_smem_even_2ch_max_bins = 4096, .dynamic_smem_even_3ch_max_bins = 8192, + .dynamic_smem_even_4ch_max_bins = 16384, .range_interpolation_min_bins = 512}; # else // _CCCL_STD_VER >= 2020 constexpr auto p2 = p1; @@ -1861,8 +1865,9 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" ", .mem_preference = SMEM, .use_work_stealing = 0, .init_kernel_pdl_trigger_max_bins = 2048" ", .dynamic_smem_bytes = 12345, .static_smem_threads_per_block = 96" - ", .static_smem_items_per_thread = 3, .dynamic_smem_range_max_bins = 1024" - ", .dynamic_smem_even_max_bins = 4096, .dynamic_smem_even_3ch_max_bins = 8192" + ", .static_smem_items_per_thread = 3, .static_smem_min_blocks_per_sm = 2" + ", .dynamic_smem_range_max_bins = 1024, .dynamic_smem_even_2ch_max_bins = 4096" + ", .dynamic_smem_even_3ch_max_bins = 8192, .dynamic_smem_even_4ch_max_bins = 16384" ", .range_interpolation_min_bins = 512 }"); } @@ -1880,8 +1885,9 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE(sm100_policy.dynamic_smem_bytes == 228352); STATIC_REQUIRE(sm100_wide_counter_policy.dynamic_smem_bytes == 228352); STATIC_REQUIRE(sm100_policy.dynamic_smem_range_max_bins == 2048); - STATIC_REQUIRE(sm100_policy.dynamic_smem_even_max_bins == 8192); - STATIC_REQUIRE(sm100_policy.dynamic_smem_even_3ch_max_bins == 57088); + STATIC_REQUIRE(sm100_policy.dynamic_smem_even_2ch_max_bins == 28544); + STATIC_REQUIRE(sm100_policy.dynamic_smem_even_3ch_max_bins == 19029); + STATIC_REQUIRE(sm100_policy.dynamic_smem_even_4ch_max_bins == 8192); STATIC_REQUIRE(sm100_policy.range_interpolation_min_bins == 512); STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57088, 4, 1)); @@ -1890,6 +1896,8 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 2049, 4, 3)); STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 8192, 4, 4)); STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 8193, 4, 4)); + STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 28544, 4, 2)); + STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 28545, 4, 2)); STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 19029, 4, 3)); STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 19030, 4, 3)); From 586f68421002658fecab3ecd1617c986495855a3 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Fri, 31 Jul 2026 14:01:53 +0000 Subject: [PATCH 08/45] [cub] Address histogram privatization review feedback --- cub/cub/agent/agent_histogram.cuh | 4 +- .../device/dispatch/dispatch_histogram.cuh | 60 +------ .../dispatch/kernels/kernel_histogram.cuh | 152 +++++------------- .../dispatch/tuning/tuning_histogram.cuh | 65 +++++--- cub/test/catch2_test_device_histogram_env.cu | 11 +- 5 files changed, 94 insertions(+), 198 deletions(-) diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index a73718b187ec..b4cd057d5177 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -328,13 +328,11 @@ struct AgentHistogram // Bin pixels int bins[pixels_per_thread]; - typename PrivatizedDecodeOpT::BinSelectState bin_select_state; _CCCL_PRAGMA_UNROLL_FULL() for (int pixel = 0; pixel < pixels_per_thread; ++pixel) { bins[pixel] = -1; - privatized_decode_op[ch].template BinSelect( - samples[pixel][ch], bins[pixel], is_valid[pixel], bin_select_state); + privatized_decode_op[ch].template BinSelect(samples[pixel][ch], bins[pixel], is_valid[pixel]); } CounterT accumulator = 1; diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 1a51c3af27e7..696e850b5805 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -69,9 +69,6 @@ struct local_counter using local_counter_t = typename local_counter::type; -// Maximum number of bins per channel for which we will use a privatized smem strategy -static constexpr int max_privatized_smem_bins = 256; - template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool -should_use_dynamic_smem(const HistogramPolicy& policy, int num_bins, int counter_size, int num_active_channels) -{ - if (policy.dynamic_smem_bytes <= 0 || num_bins <= 0 || counter_size <= 0 || num_active_channels <= 0) - { - return false; - } - - const bool prefer_dynamic_smem = counter_size > int{sizeof(unsigned int)} || num_bins > max_privatized_smem_bins; - const size_t required_bytes = size_t(num_bins) * size_t(num_active_channels) * size_t(counter_size); - - int max_bins = num_bins; - if (num_active_channels > 1) - { - if constexpr (IsEven) - { - max_bins = num_active_channels == 2 ? policy.dynamic_smem_even_2ch_max_bins - : num_active_channels == 3 - ? policy.dynamic_smem_even_3ch_max_bins - : policy.dynamic_smem_even_4ch_max_bins; - } - else - { - max_bins = policy.dynamic_smem_range_max_bins; - } - } - - return prefer_dynamic_smem && max_bins > 0 && num_bins <= max_bins - && required_bytes <= static_cast(policy.dynamic_smem_bytes); -} - template 0 && !UseDynamicSmem; const int threads_per_block = use_static_smem ? active_policy.static_smem_threads() : active_policy.threads_per_block; - const int pixels_per_thread = use_static_smem ? active_policy.static_smem_items() : active_policy.pixels_per_thread; + const int items_per_thread = use_static_smem ? active_policy.static_smem_items() : active_policy.pixels_per_thread; int dynamic_smem_bytes = 0; if constexpr (UseDynamicSmem) @@ -359,7 +324,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( } // Get grid dimensions, trying to keep total blocks ~histogram_sweep_occupancy - int pixels_per_tile = threads_per_block * pixels_per_thread; + int pixels_per_tile = threads_per_block * items_per_thread; int tiles_per_row = static_cast(::cuda::ceil_div(num_row_pixels, pixels_per_tile)); int blocks_per_row = ::cuda::std::min(histogram_sweep_occupancy, tiles_per_row); int blocks_per_col = @@ -459,7 +424,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( threads_per_block, dynamic_smem_bytes, (long long) stream, - pixels_per_thread, + items_per_thread, histogram_sweep_sm_occupancy); #endif // CUB_DEBUG_LOG @@ -944,19 +909,6 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_4ch_max_bins(long return 0; } -template -_CCCL_HOST_DEVICE_API constexpr auto convert_range_interpolation_min_bins(int) - -> decltype(ActivePolicy::range_interpolation_min_bins) -{ - return ActivePolicy::range_interpolation_min_bins; -} - -template -_CCCL_HOST_DEVICE_API constexpr auto convert_range_interpolation_min_bins(long) -{ - return 0; -} - // TODO(bgruber): drop in CCCL 4.0 template _CCCL_HOST_DEVICE_API constexpr auto convert_policy() -> HistogramPolicy @@ -979,8 +931,7 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_policy() -> HistogramPolicy convert_dynamic_smem_range_max_bins(0), convert_dynamic_smem_even_2ch_max_bins(0), convert_dynamic_smem_even_3ch_max_bins(0), - convert_dynamic_smem_even_4ch_max_bins(0), - convert_range_interpolation_min_bins(0)}; + convert_dynamic_smem_even_4ch_max_bins(0)}; } // TODO(bgruber): drop in CCCL 4.0 @@ -1141,8 +1092,7 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( ::cuda::std::array privatized_decode_op{}; for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel) { - privatized_decode_op[channel].Init( - d_levels[channel], num_output_levels[channel], active_policy.range_interpolation_min_bins); + privatized_decode_op[channel].Init(d_levels[channel], num_output_levels[channel]); } return CubDebug( diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index acdd8d0a9d2f..cb204b09acee 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -39,9 +39,6 @@ struct Transforms template struct SearchTransform { - struct BinSelectState - {}; - LevelIteratorT d_levels; // Pointer to levels array int num_output_levels; // Number of levels in array @@ -51,17 +48,10 @@ struct Transforms num_output_levels = num_output_levels_; } - _CCCL_DEVICE _CCCL_FORCEINLINE void PrecomputeOnDevice() {} + _CCCL_DEVICE _CCCL_FORCEINLINE void Precompute() {} template _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(SampleT2 sample, int& bin, bool valid) const - { - BinSelectState state; - BinSelect(sample, bin, valid, state); - } - - template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(SampleT2 sample, int& bin, bool valid, BinSelectState&) const { using WrappedLevelIteratorT = ::cuda::std::_If<::cuda::std::is_pointer_v, @@ -81,7 +71,7 @@ struct Transforms } }; - //! @brief Finds a RANGE bin with interpolation and a per-thread bracket cache. + //! @brief Finds a RANGE bin with piecewise-linear interpolation and a per-thread bracket cache. //! //! This transform is used by the runtime-sized shared-memory kernel. It //! precomputes interpolation parameters once per thread, verifies each @@ -114,10 +104,10 @@ struct Transforms int bin = -1; // cached bin; < 0 means empty }; + mutable BinSelectState mru; + LevelIteratorT d_levels; // Pointer to levels array int num_output_levels; // Number of levels in array - int interpolation_min_bins; // policy-selected minimum bin count for interpolation - // Interpolation state shared by all samples processed by a thread. float m_inv_scale; // num_bins / (float)(last - first); valid iff m_have_precompute LevelT m_first; // cached d_levels[0] @@ -134,32 +124,25 @@ struct Transforms //! //! @param d_levels_ Pointer to levels array //! @param num_output_levels_ Number of levels in array - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void - Init(LevelIteratorT d_levels_, int num_output_levels_, int interpolation_min_bins_) + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void Init(LevelIteratorT d_levels_, int num_output_levels_) { - this->d_levels = d_levels_; - this->num_output_levels = num_output_levels_; - this->interpolation_min_bins = interpolation_min_bins_; - this->m_have_precompute = false; - this->m_inv_scale = 0.0f; - this->m_first = LevelT{}; - this->m_last = LevelT{}; - this->m_mid = LevelT{}; - this->m_inv_scale_lo = 0.0f; - this->m_inv_scale_hi = 0.0f; - this->m_mid_bin = 0; + this->d_levels = d_levels_; + this->num_output_levels = num_output_levels_; + this->m_have_precompute = false; + this->m_inv_scale = 0.0f; + this->m_first = LevelT{}; + this->m_last = LevelT{}; + this->m_mid = LevelT{}; + this->m_inv_scale_lo = 0.0f; + this->m_inv_scale_hi = 0.0f; + this->m_mid_bin = 0; + this->mru = BinSelectState{}; } //! @brief Precomputes interpolation slopes from the device level array. - _CCCL_DEVICE _CCCL_FORCEINLINE void PrecomputeOnDevice() + _CCCL_DEVICE _CCCL_FORCEINLINE void Precompute() { const int num_bins = num_output_levels - 1; - if (num_bins < interpolation_min_bins) - { - m_have_precompute = false; - return; - } - using WrappedLevelIteratorT = ::cuda::std::_If<::cuda::std::is_pointer_v, CacheModifiedInputIterator, @@ -200,23 +183,19 @@ struct Transforms } private: - struct NoBinSelectState - {}; - //! @brief Implements cached/interpolated bin selection. //! //! A cached-bracket hit returns immediately. Otherwise, this computes and //! verifies an interpolated guess, checks one adjacent bracket, and finally //! falls back to `UpperBound` for arbitrary level distributions. - template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelectImpl(_SampleT sample, int& bin, bool valid, CacheT& cache) const + template + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void + BinSelectImpl(_SampleT sample, int& bin, bool valid, BinSelectState& cache) const { using WrappedLevelIteratorT = ::cuda::std::_If<::cuda::std::is_pointer_v, CacheModifiedInputIterator, LevelIteratorT>; - constexpr bool use_cache = ::cuda::std::is_same_v; - WrappedLevelIteratorT wrapped_levels(d_levels); const int num_bins = num_output_levels - 1; if (!valid) @@ -226,22 +205,9 @@ struct Transforms const LevelT s = static_cast(sample); - if constexpr (use_cache) + if (cache.bin >= 0 && !(s < cache.lo) && (s < cache.hi)) { - if (cache.bin >= 0 && !(s < cache.lo) && (s < cache.hi)) - { - bin = cache.bin; - return; - } - } - - if (interpolation_min_bins <= 0 || num_bins < interpolation_min_bins) - { - bin = UpperBound(wrapped_levels, num_output_levels, s) - 1; - if (bin >= num_bins) - { - bin = -1; - } + bin = cache.bin; return; } @@ -288,12 +254,8 @@ struct Transforms else { const auto range = interpolation_difference(last_level, first_level); - NV_IF_ELSE_TARGET( - NV_IS_DEVICE, - (guess = static_cast( - __fdividef(static_cast(delta) * static_cast(num_bins), static_cast(range)));), - (guess = static_cast( - (static_cast(delta) * static_cast(num_bins)) / static_cast(range));)); + guess = + static_cast((static_cast(delta) * static_cast(num_bins)) / static_cast(range)); } if (guess < 0) { @@ -309,11 +271,8 @@ struct Transforms if (!(s < lvl_lo) && (s < lvl_hi)) { - bin = guess; - if constexpr (use_cache) - { - cache = BinSelectState{lvl_lo, lvl_hi, guess}; - } + bin = guess; + cache = BinSelectState{lvl_lo, lvl_hi, guess}; return; } @@ -325,11 +284,8 @@ struct Transforms const LevelT lvl2_lo = wrapped_levels[g2]; if (!(s < lvl2_lo)) { - bin = g2; - if constexpr (use_cache) - { - cache = BinSelectState{lvl2_lo, lvl_lo, g2}; - } + bin = g2; + cache = BinSelectState{lvl2_lo, lvl_lo, g2}; return; } } @@ -342,11 +298,8 @@ struct Transforms const LevelT lvl2_hi = wrapped_levels[g2 + 1]; if (s < lvl2_hi) { - bin = g2; - if constexpr (use_cache) - { - cache = BinSelectState{lvl_hi, lvl2_hi, g2}; - } + bin = g2; + cache = BinSelectState{lvl_hi, lvl2_hi, g2}; return; } } @@ -358,27 +311,16 @@ struct Transforms bin = -1; return; } - if constexpr (use_cache) + if (bin >= 0) { - if (bin >= 0) - { - cache = BinSelectState{wrapped_levels[bin], wrapped_levels[bin + 1], bin}; - } + cache = BinSelectState{wrapped_levels[bin], wrapped_levels[bin + 1], bin}; } } public: - //! @brief Selects a bin without retaining bracket state. - template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid) const - { - NoBinSelectState cache; - BinSelectImpl(sample, bin, valid, cache); - } - - //! @brief Selects a bin and updates the per-thread bracket state. + //! @brief Selects a bin and retains the most recently used bracket in this thread's transform copy. template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid, BinSelectState& mru) const + _CCCL_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid) const { BinSelectImpl(sample, bin, valid, mru); } @@ -575,10 +517,7 @@ struct Transforms m_scale = this->ComputeScale(num_levels, m_max, m_min); } - _CCCL_DEVICE _CCCL_FORCEINLINE void PrecomputeOnDevice() {} - - struct BinSelectState - {}; + _CCCL_DEVICE _CCCL_FORCEINLINE void Precompute() {} // Method for converting samples to bin-ids template @@ -591,12 +530,6 @@ struct Transforms bin = this->ComputeBin(common_sample, m_min, m_scale); } } - - template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(SampleT sample, int& bin, bool valid, BinSelectState&) const - { - this->template BinSelect(sample, bin, valid); - } }; // Pass-through bin transform operator @@ -618,10 +551,7 @@ struct Transforms _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void Init(T, int) {} - _CCCL_DEVICE _CCCL_FORCEINLINE void PrecomputeOnDevice() {} - - struct BinSelectState - {}; + _CCCL_DEVICE _CCCL_FORCEINLINE void Precompute() {} // Method for converting samples to bin-ids template @@ -632,12 +562,6 @@ struct Transforms bin = static_cast(sample); } } - - template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid, BinSelectState&) const - { - this->template BinSelect(sample, bin, valid); - } }; }; @@ -926,8 +850,8 @@ __launch_bounds__(int(current_policy().threads_per_block)) { output_decode_op[channel] = output_decode_op_wrapper[channel]; privatized_decode_op[channel] = privatized_decode_op_wrapper[channel]; - output_decode_op[channel].PrecomputeOnDevice(); - privatized_decode_op[channel].PrecomputeOnDevice(); + output_decode_op[channel].Precompute(); + privatized_decode_op[channel].Precompute(); } AgentHistogramT agent( diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 58b56bd6cb50..516e20802ebf 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -47,7 +47,6 @@ struct HistogramPolicy int dynamic_smem_even_2ch_max_bins = 0; //!< Two-channel EVEN cap per channel; 0 disables the dynamic path int dynamic_smem_even_3ch_max_bins = 0; //!< Three-channel EVEN cap per channel; 0 disables the dynamic path int dynamic_smem_even_4ch_max_bins = 0; //!< Four-channel EVEN cap per channel; 0 disables the dynamic path - int range_interpolation_min_bins = 0; //!< Minimum RANGE bin count for interpolation; 0 disables interpolation [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int static_smem_threads() const { @@ -79,8 +78,7 @@ struct HistogramPolicy && lhs.dynamic_smem_range_max_bins == rhs.dynamic_smem_range_max_bins && lhs.dynamic_smem_even_2ch_max_bins == rhs.dynamic_smem_even_2ch_max_bins && lhs.dynamic_smem_even_3ch_max_bins == rhs.dynamic_smem_even_3ch_max_bins - && lhs.dynamic_smem_even_4ch_max_bins == rhs.dynamic_smem_even_4ch_max_bins - && lhs.range_interpolation_min_bins == rhs.range_interpolation_min_bins; + && lhs.dynamic_smem_even_4ch_max_bins == rhs.dynamic_smem_even_4ch_max_bins; } [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool @@ -104,21 +102,54 @@ struct HistogramPolicy << ", .dynamic_smem_range_max_bins = " << p.dynamic_smem_range_max_bins << ", .dynamic_smem_even_2ch_max_bins = " << p.dynamic_smem_even_2ch_max_bins << ", .dynamic_smem_even_3ch_max_bins = " << p.dynamic_smem_even_3ch_max_bins - << ", .dynamic_smem_even_4ch_max_bins = " << p.dynamic_smem_even_4ch_max_bins - << ", .range_interpolation_min_bins = " << p.range_interpolation_min_bins << " }"; + << ", .dynamic_smem_even_4ch_max_bins = " << p.dynamic_smem_even_4ch_max_bins << " }"; } #endif // _CCCL_HOSTED() }; namespace detail::histogram { +// Maximum number of bins per channel for the compile-time-sized shared-memory tier. +static constexpr int max_privatized_smem_bins = 256; + // Leave 4096 bytes of the SM100 opt-in shared-memory limit available for static storage. static constexpr int sm100_dynamic_smem_bytes = 232448 - 4096; static constexpr int sm100_dynamic_smem_range_max_bins = 2048; static constexpr int sm100_dynamic_smem_even_2ch_max_bins = 28544; static constexpr int sm100_dynamic_smem_even_3ch_max_bins = 19029; static constexpr int sm100_dynamic_smem_even_4ch_max_bins = 8192; -static constexpr int sm100_range_interpolation_min_bins = 512; + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool +should_use_dynamic_smem(const HistogramPolicy& policy, int num_bins, int counter_size, int num_active_channels) +{ + if (policy.dynamic_smem_bytes <= 0 || num_bins <= 0 || counter_size <= 0 || num_active_channels <= 0) + { + return false; + } + + const bool prefer_dynamic_smem = counter_size > int{sizeof(unsigned int)} || num_bins > max_privatized_smem_bins; + const size_t required_bytes = size_t(num_bins) * size_t(num_active_channels) * size_t(counter_size); + + int max_bins = num_bins; + if (num_active_channels > 1) + { + if constexpr (IsEven) + { + max_bins = num_active_channels == 2 ? policy.dynamic_smem_even_2ch_max_bins + : num_active_channels == 3 + ? policy.dynamic_smem_even_3ch_max_bins + : policy.dynamic_smem_even_4ch_max_bins; + } + else + { + max_bins = policy.dynamic_smem_range_max_bins; + } + } + + return prefer_dynamic_smem && max_bins > 0 && num_bins <= max_bins + && required_bytes <= static_cast(policy.dynamic_smem_bytes); +} // TODO(bgruber): drop in CCCL 4.0 enum class primitive_sample @@ -190,8 +221,8 @@ struct sm90_tuning @@ -205,8 +236,8 @@ struct sm90_tuning; + Tuning::work_stealing>; template _CCCL_HOST_DEVICE_API static auto select_agent_policy(long) -> typename Policy500::AgentHistogramPolicyT; @@ -332,7 +363,7 @@ struct policy_hub Tuning::load_modifier, Tuning::rle_compress, Tuning::mem_preference, - Tuning::use_work_stealing, + Tuning::work_stealing, Tuning::vec_size>; template @@ -362,7 +393,6 @@ struct policy_hub static constexpr int dynamic_smem_even_2ch_max_bins = sm100_dynamic_smem_even_2ch_max_bins; static constexpr int dynamic_smem_even_3ch_max_bins = sm100_dynamic_smem_even_3ch_max_bins; static constexpr int dynamic_smem_even_4ch_max_bins = sm100_dynamic_smem_even_4ch_max_bins; - static constexpr int range_interpolation_min_bins = sm100_range_interpolation_min_bins; static constexpr int static_smem_threads_per_block = !IsEven && sizeof(CounterT) == 4 && is_primitive::value ? (NumChannels >= 2 @@ -412,7 +442,6 @@ private: policy.dynamic_smem_even_2ch_max_bins = sm100_dynamic_smem_even_2ch_max_bins; policy.dynamic_smem_even_3ch_max_bins = sm100_dynamic_smem_even_3ch_max_bins; policy.dynamic_smem_even_4ch_max_bins = sm100_dynamic_smem_even_4ch_max_bins; - policy.range_interpolation_min_bins = sm100_range_interpolation_min_bins; return policy; } diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index 3762fa7ac7ba..e199423a9652 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1645,7 +1645,6 @@ struct mixed_counter_histogram_tuning cub::HistogramPolicy policy{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, cub::SMEM, false, 0}; policy.dynamic_smem_bytes = 228352; policy.dynamic_smem_even_4ch_max_bins = 8192; - policy.range_interpolation_min_bins = 512; return policy; } }; @@ -1822,8 +1821,7 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) 1024, 4096, 8192, - 16384, - 512}; + 16384}; # if _CCCL_STD_VER >= 2020 // designated init @@ -1844,8 +1842,7 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) .dynamic_smem_range_max_bins = 1024, .dynamic_smem_even_2ch_max_bins = 4096, .dynamic_smem_even_3ch_max_bins = 8192, - .dynamic_smem_even_4ch_max_bins = 16384, - .range_interpolation_min_bins = 512}; + .dynamic_smem_even_4ch_max_bins = 16384}; # else // _CCCL_STD_VER >= 2020 constexpr auto p2 = p1; # endif // _CCCL_STD_VER >= 2020 @@ -1867,8 +1864,7 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) ", .dynamic_smem_bytes = 12345, .static_smem_threads_per_block = 96" ", .static_smem_items_per_thread = 3, .static_smem_min_blocks_per_sm = 2" ", .dynamic_smem_range_max_bins = 1024, .dynamic_smem_even_2ch_max_bins = 4096" - ", .dynamic_smem_even_3ch_max_bins = 8192, .dynamic_smem_even_4ch_max_bins = 16384" - ", .range_interpolation_min_bins = 512 }"); + ", .dynamic_smem_even_3ch_max_bins = 8192, .dynamic_smem_even_4ch_max_bins = 16384 }"); } C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget", "[histogram][device]") @@ -1888,7 +1884,6 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE(sm100_policy.dynamic_smem_even_2ch_max_bins == 28544); STATIC_REQUIRE(sm100_policy.dynamic_smem_even_3ch_max_bins == 19029); STATIC_REQUIRE(sm100_policy.dynamic_smem_even_4ch_max_bins == 8192); - STATIC_REQUIRE(sm100_policy.range_interpolation_min_bins == 512); STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57088, 4, 1)); STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57089, 4, 1)); From c81eee367af60835ac941a59180d08492be83325 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Fri, 31 Jul 2026 14:10:44 +0000 Subject: [PATCH 09/45] [cub] Normalize histogram tuning names --- .../dispatch/tuning/tuning_histogram.cuh | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 516e20802ebf..90af9132b68d 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -213,8 +213,8 @@ struct sm90_tuning; template struct sm90_tuning { - static constexpr int threads = 768; - static constexpr int items = 12; + static constexpr int threads_per_block = 768; + static constexpr int items_per_thread = 12; static constexpr CacheLoadModifier load_modifier = LOAD_LDG; static constexpr BlockHistogramMemoryPreference mem_preference = SMEM; @@ -228,8 +228,8 @@ struct sm90_tuning struct sm90_tuning { - static constexpr int threads = 960; - static constexpr int items = 10; + static constexpr int threads_per_block = 960; + static constexpr int items_per_thread = 10; static constexpr CacheLoadModifier load_modifier = LOAD_DEFAULT; static constexpr BlockHistogramMemoryPreference mem_preference = SMEM; @@ -255,8 +255,8 @@ template struct sm100_tuning { // ipt_12.tpb_928.rle_0.ws_0.mem_1.ld_2.laid_0.vec_2 1.033332 0.940517 1.031835 1.195876 - static constexpr int items = 12; - static constexpr int threads = 928; + static constexpr int items_per_thread = 12; + static constexpr int threads_per_block = 928; static constexpr bool rle_compress = false; static constexpr bool work_stealing = false; static constexpr BlockHistogramMemoryPreference mem_preference = SMEM; @@ -270,8 +270,8 @@ template struct sm100_tuning { // ipt_12.tpb_448.rle_0.ws_0.mem_1.ld_1.laid_0.vec_2 1.078987 0.985542 1.085118 1.175637 - static constexpr int items = 12; - static constexpr int threads = 448; + static constexpr int items_per_thread = 12; + static constexpr int threads_per_block = 448; static constexpr bool rle_compress = false; static constexpr bool work_stealing = false; static constexpr BlockHistogramMemoryPreference mem_preference = SMEM; @@ -283,8 +283,8 @@ struct sm100_tuning struct sm100_tuning { - static constexpr int items = 12; - static constexpr int threads = 768; + static constexpr int items_per_thread = 12; + static constexpr int threads_per_block = 768; static constexpr bool rle_compress = true; static constexpr bool work_stealing = false; static constexpr BlockHistogramMemoryPreference mem_preference = SMEM; @@ -296,8 +296,8 @@ struct sm100_tuning struct sm100_tuning { - static constexpr int items = 6; - static constexpr int threads = 768; + static constexpr int items_per_thread = 6; + static constexpr int threads_per_block = 768; static constexpr bool rle_compress = true; static constexpr bool work_stealing = false; static constexpr BlockHistogramMemoryPreference mem_preference = SMEM; @@ -334,8 +334,8 @@ struct policy_hub // Use values from tuning if a specialization exists, otherwise pick Policy500 template _CCCL_HOST_DEVICE_API static auto select_agent_policy(int) - -> agent_histogram_policy agent_histogram_policy _CCCL_HOST_DEVICE_API static auto select_agent_policy(int) -> agent_histogram_policy< - Tuning::threads, - Tuning::items, + Tuning::threads_per_block, + Tuning::items_per_thread, Tuning::load_algorithm, Tuning::load_modifier, Tuning::rle_compress, From 798e0f8c2f877777adbe2afdbef5213c4b7d9dab Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Fri, 31 Jul 2026 15:19:17 +0000 Subject: [PATCH 10/45] [cub] Raise static histogram SMEM tier to 512 bins --- cub/cub/device/dispatch/tuning/tuning_histogram.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 90af9132b68d..9b1628c22ba3 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -110,7 +110,7 @@ struct HistogramPolicy namespace detail::histogram { // Maximum number of bins per channel for the compile-time-sized shared-memory tier. -static constexpr int max_privatized_smem_bins = 256; +static constexpr int max_privatized_smem_bins = 512; // Leave 4096 bytes of the SM100 opt-in shared-memory limit available for static storage. static constexpr int sm100_dynamic_smem_bytes = 232448 - 4096; From a474901a2fba5f048a97d4eb425c54a447c5b2d2 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Fri, 31 Jul 2026 15:47:23 +0000 Subject: [PATCH 11/45] [cub] Address histogram CodeRabbit review --- .../device/dispatch/dispatch_histogram.cuh | 85 ++++++++++--------- .../dispatch/kernels/kernel_histogram.cuh | 5 +- .../dispatch/tuning/tuning_histogram.cuh | 37 ++++++-- cub/test/catch2_test_device_histogram.cu | 14 ++- cub/test/catch2_test_device_histogram_env.cu | 2 +- 5 files changed, 90 insertions(+), 53 deletions(-) diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 696e850b5805..7fb9bee8b912 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -289,8 +289,8 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( dynamic_smem_bytes += (num_privatized_levels[channel] - 1) * static_cast(kernel_source.CounterSize()); } NV_IF_TARGET(NV_IS_HOST, ({ - if (const auto error = - CubDebug(launcher_factory.set_max_dynamic_smem_size_for(sweep_kernel, dynamic_smem_bytes))) + if (const auto error = CubDebug(launcher_factory.set_max_dynamic_smem_size_for( + sweep_kernel, active_policy.dynamic_smem_bytes))) { return error; } @@ -1002,13 +1002,6 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( { using LocalCounterT = local_counter_t; - ::cuda::compute_capability cc{}; - if (const auto error = CubDebug(launcher_factory.PtxComputeCap(cc))) - { - return error; - } - const HistogramPolicy active_policy = policy_selector(cc); - if constexpr (IsByteSample) { using TransformsT = Transforms; @@ -1068,6 +1061,13 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( } else { + ::cuda::compute_capability cc{}; + if (const auto error = CubDebug(launcher_factory.PtxComputeCap(cc))) + { + return error; + } + const HistogramPolicy active_policy = policy_selector(cc); + using TransformsT = Transforms; // Use the pass-thru transform op for converting privatized bins to output bins @@ -1085,40 +1085,45 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( } int max_num_output_bins = max_levels - 1; - if (should_use_dynamic_smem( - active_policy, max_num_output_bins, int{kernel_source.CounterSize()}, NUM_ACTIVE_CHANNELS)) + constexpr bool supports_cached_search = + ::cuda::std::is_integral_v || ::cuda::std::is_floating_point_v; + if constexpr (supports_cached_search) { - using PrivatizedDecodeOpT = typename TransformsT::template CachedSearchTransform; - ::cuda::std::array privatized_decode_op{}; - for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel) + if (should_use_dynamic_smem( + active_policy, max_num_output_bins, int{kernel_source.CounterSize()}, NUM_ACTIVE_CHANNELS)) { - privatized_decode_op[channel].Init(d_levels[channel], num_output_levels[channel]); - } + using PrivatizedDecodeOpT = typename TransformsT::template CachedSearchTransform; + ::cuda::std::array privatized_decode_op{}; + for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel) + { + privatized_decode_op[channel].Init(d_levels[channel], num_output_levels[channel]); + } - return CubDebug( - (detail::histogram::dispatch( - d_temp_storage, - temp_storage_bytes, - d_samples, - d_output_histograms, - num_output_levels, - num_output_levels, - output_decode_op, - privatized_decode_op, - max_num_output_bins, - num_row_pixels, - num_rows, - row_stride_samples, - stream, - policy_selector, - kernel_source, - launcher_factory))); + return CubDebug( + (detail::histogram::dispatch( + d_temp_storage, + temp_storage_bytes, + d_samples, + d_output_histograms, + num_output_levels, + num_output_levels, + output_decode_op, + privatized_decode_op, + max_num_output_bins, + num_row_pixels, + num_rows, + row_stride_samples, + stream, + policy_selector, + kernel_source, + launcher_factory))); + } } using PrivatizedDecodeOpT = typename TransformsT::template SearchTransform; diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index cb204b09acee..4d8a73d690a2 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -785,8 +785,9 @@ __launch_bounds__(int(PrivatizedSmemBins > 0 ? current_policy(). //! Histogram sweep kernel with the privatized histogram in dynamic shared memory. //! -//! The host supplies `num_privatized_bins * sizeof(CounterT)` bytes of dynamic -//! shared memory. Keeping the runtime-sized histogram outside `TempStorage` +//! The host supplies `sum(num_privatized_bins[ch]) * sizeof(CounterT)` bytes of +//! dynamic shared memory, which the agent partitions per channel. Keeping the +//! runtime-sized histogram outside `TempStorage` //! allows one kernel instantiation to cover larger histograms without a ladder //! of statically sized kernels. template [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool should_use_dynamic_smem(const HistogramPolicy& policy, int num_bins, int counter_size, int num_active_channels) { + // Single-channel limits are intentionally byte-derived: the B200 tuning + // characterized that path through the full opt-in shared-memory budget. + // Multi-channel paths use explicit per-channel caps in addition to the byte + // budget because their channel-interleaved launch shapes have distinct + // measured crossover points. if (policy.dynamic_smem_bytes <= 0 || num_bins <= 0 || counter_size <= 0 || num_active_channels <= 0) { return false; @@ -383,16 +388,25 @@ struct policy_hub using AgentHistogramPolicyT = ::cuda::std::_If; + static constexpr bool has_dynamic_smem_tuning = + sizeof(CounterT) == 4 && is_primitive::value + && ((NumChannels == 1 && NumActiveChannels == 1 + && (sizeof(SampleT) == 1 || sizeof(SampleT) == 4 || sizeof(SampleT) == 8)) + || NumChannels >= 2); + static constexpr int init_kernel_pdl_trigger_max_bins = NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value && (sizeof(SampleT) == 1 || sizeof(SampleT) == 2 || sizeof(SampleT) == 4 || sizeof(SampleT) == 8) ? 2048 : 0; - static constexpr int dynamic_smem_bytes = sm100_dynamic_smem_bytes; - static constexpr int dynamic_smem_range_max_bins = sm100_dynamic_smem_range_max_bins; - static constexpr int dynamic_smem_even_2ch_max_bins = sm100_dynamic_smem_even_2ch_max_bins; - static constexpr int dynamic_smem_even_3ch_max_bins = sm100_dynamic_smem_even_3ch_max_bins; - static constexpr int dynamic_smem_even_4ch_max_bins = sm100_dynamic_smem_even_4ch_max_bins; + static constexpr int dynamic_smem_bytes = has_dynamic_smem_tuning ? sm100_dynamic_smem_bytes : 0; + static constexpr int dynamic_smem_range_max_bins = has_dynamic_smem_tuning ? sm100_dynamic_smem_range_max_bins : 0; + static constexpr int dynamic_smem_even_2ch_max_bins = + has_dynamic_smem_tuning ? sm100_dynamic_smem_even_2ch_max_bins : 0; + static constexpr int dynamic_smem_even_3ch_max_bins = + has_dynamic_smem_tuning ? sm100_dynamic_smem_even_3ch_max_bins : 0; + static constexpr int dynamic_smem_even_4ch_max_bins = + has_dynamic_smem_tuning ? sm100_dynamic_smem_even_4ch_max_bins : 0; static constexpr int static_smem_threads_per_block = !IsEven && sizeof(CounterT) == 4 && is_primitive::value ? (NumChannels >= 2 @@ -445,6 +459,14 @@ private: return policy; } + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool has_sm100_dynamic_smem_tuning() const + { + return sample_is_primitive && counter_size == 4 + && ((num_channels == 1 && num_active_channels == 1 + && (sample_size == 1 || sample_size == 4 || sample_size == 8)) + || num_channels >= 2); + } + public: [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy { @@ -507,9 +529,8 @@ public: return sm100_policy(HistogramPolicy{960, 10, 1 << 2, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, SMEM, false, 2048}); } - // Even when no SM100 launch-shape specialization applies, retain the - // architecture's dynamic shared-memory budget on the inherited fallback. - return sm100_policy(HistogramPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false, 0}); + auto fallback = HistogramPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false, 0}; + return has_sm100_dynamic_smem_tuning() ? sm100_policy(fallback) : fallback; } if (cc >= ::cuda::compute_capability{9, 0}) diff --git a/cub/test/catch2_test_device_histogram.cu b/cub/test/catch2_test_device_histogram.cu index 851d0ab446e1..0fcf83c8f16c 100644 --- a/cub/test/catch2_test_device_histogram.cu +++ b/cub/test/catch2_test_device_histogram.cu @@ -586,7 +586,7 @@ C2H_TEST("DeviceHistogram::Histogram* dynamic shared-memory privatization", "[hi SKIP("The runtime-sized shared-memory histogram policy is currently tuned for SM100"); } - using counter_t = unsigned long long; + using counter_t = unsigned int; const int num_levels = GENERATE(1025, 4097); test_even_and_range(num_levels - 1, num_levels, 4096, 4); @@ -677,8 +677,18 @@ CUB_TEST("DeviceHistogram::HistogramRange levels/samples aliasing", "[histogram_ C2H_TEST("DeviceHistogram::HistogramRange interpolation avoids signed overflow", "[histogram_range][device]") { + int current_device{}; + REQUIRE(cudaSuccess == cudaGetDevice(¤t_device)); + + cuda::compute_capability cc{}; + REQUIRE(cudaSuccess == cub::detail::ptx_compute_cap(cc, current_device)); + if (cc < cuda::compute_capability{10, 0}) + { + SKIP("The runtime-sized shared-memory histogram policy is currently tuned for SM100"); + } + using sample_t = int; - constexpr int num_bins = 512; + constexpr int num_bins = 1024; c2h::host_vector h_levels(num_bins + 1); constexpr auto lo = static_cast(cs::numeric_limits::lowest()); diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index e199423a9652..a6f3c6ad64fb 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1879,7 +1879,7 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE(sm90_policy.dynamic_smem_bytes == 0); STATIC_REQUIRE(sm100_policy.dynamic_smem_bytes == 228352); - STATIC_REQUIRE(sm100_wide_counter_policy.dynamic_smem_bytes == 228352); + STATIC_REQUIRE(sm100_wide_counter_policy.dynamic_smem_bytes == 0); STATIC_REQUIRE(sm100_policy.dynamic_smem_range_max_bins == 2048); STATIC_REQUIRE(sm100_policy.dynamic_smem_even_2ch_max_bins == 28544); STATIC_REQUIRE(sm100_policy.dynamic_smem_even_3ch_max_bins == 19029); From c2e39313278b1724b095c269933e131a6af0786d Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Fri, 31 Jul 2026 16:00:46 +0000 Subject: [PATCH 12/45] [cub] Align histogram policy and simplify accumulation --- cub/cub/agent/agent_histogram.cuh | 10 +++++++--- cub/cub/device/dispatch/tuning/tuning_histogram.cuh | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index b4cd057d5177..4d0bf6880afc 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -483,20 +483,24 @@ struct AgentHistogram LoadTile(block_offset, valid_samples, samples); MarkValid(is_valid, valid_samples); + auto accumulate_pixels = [&](auto& privatized_histograms) { + AccumulatePixels(samples, is_valid, privatized_histograms, ::cuda::std::bool_constant{}); + }; + if (prefer_smem) { if constexpr (UseDynamicSmem) { - AccumulatePixels(samples, is_valid, smem_histograms, ::cuda::std::bool_constant{}); + accumulate_pixels(smem_histograms); } else { - AccumulatePixels(samples, is_valid, temp_storage.histograms, ::cuda::std::bool_constant{}); + accumulate_pixels(temp_storage.histograms); } } else { - AccumulatePixels(samples, is_valid, d_privatized_histograms, ::cuda::std::bool_constant{}); + accumulate_pixels(d_privatized_histograms); } } diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 0d23a509f8ed..058696615fbc 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -526,7 +526,7 @@ public: if (num_channels == 1 && num_active_channels == 1 && counter_size == 4 && sample_is_primitive && sample_size == 2) { - return sm100_policy(HistogramPolicy{960, 10, 1 << 2, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, SMEM, false, 2048}); + return HistogramPolicy{960, 10, 1 << 2, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, SMEM, false, 2048}; } auto fallback = HistogramPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false, 0}; From 915d124111cd26d89dd05026793d030195e57b2d Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Fri, 31 Jul 2026 19:16:48 +0000 Subject: [PATCH 13/45] [cub] Address histogram review feedback --- cub/cub/agent/agent_histogram.cuh | 80 +++++++------------ .../device/dispatch/dispatch_histogram.cuh | 6 +- .../dispatch/kernels/kernel_histogram.cuh | 24 +++--- .../dispatch/tuning/tuning_histogram.cuh | 38 ++++----- cub/test/catch2_test_device_histogram_env.cu | 6 +- .../catch2_test_device_histogram_env_api.cu | 4 +- 6 files changed, 70 insertions(+), 88 deletions(-) diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index 4d0bf6880afc..14624ae55f88 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -262,7 +262,7 @@ struct AgentHistogram const int* num_output_bins; // one for each channel const int* num_privatized_bins; // one for each channel CounterT* d_privatized_histograms[NumActiveChannels]; // one for each channel - CounterT* smem_histograms[NumActiveChannels]; // dynamic shared-memory channel bases, when enabled + CounterT* dyn_smem_histograms[NumActiveChannels]; // dynamic shared-memory channel bases, when enabled OutputCounterT** d_output_histograms; // final output, in global memory const OutputDecodeOpT* output_decode_op; // determines output bin-id from privatized counter index, one for each // channel @@ -270,6 +270,26 @@ struct AgentHistogram // channel bool prefer_smem; // for privatized counterss + template + _CCCL_DEVICE _CCCL_FORCEINLINE void WithPrivatizedHistograms(F&& f) + { + if (prefer_smem) + { + if constexpr (UseDynamicSmem) + { + f(dyn_smem_histograms); + } + else + { + f(temp_storage.histograms); + } + } + else + { + f(d_privatized_histograms); + } + } + template _CCCL_DEVICE _CCCL_FORCEINLINE void ZeroBinCounters(TwoDimSubscriptableCounterT& privatized_histograms) { @@ -483,25 +503,9 @@ struct AgentHistogram LoadTile(block_offset, valid_samples, samples); MarkValid(is_valid, valid_samples); - auto accumulate_pixels = [&](auto& privatized_histograms) { + WithPrivatizedHistograms([&](auto& privatized_histograms) { AccumulatePixels(samples, is_valid, privatized_histograms, ::cuda::std::bool_constant{}); - }; - - if (prefer_smem) - { - if constexpr (UseDynamicSmem) - { - accumulate_pixels(smem_histograms); - } - else - { - accumulate_pixels(temp_storage.histograms); - } - } - else - { - accumulate_pixels(d_privatized_histograms); - } + }); } //! @brief Consume row tiles. Specialized for work-stealing from queue @@ -700,7 +704,7 @@ struct AgentHistogram _CCCL_PRAGMA_UNROLL_FULL() for (int ch = 0; ch < NumActiveChannels; ++ch) { - this->smem_histograms[ch] = p; + this->dyn_smem_histograms[ch] = p; p += num_privatized_bins[ch]; } } @@ -760,41 +764,17 @@ struct AgentHistogram //! Initialize privatized bin counters. Specialized for privatized shared-memory counters _CCCL_DEVICE _CCCL_FORCEINLINE void InitBinCounters() { - if (prefer_smem) - { - if constexpr (UseDynamicSmem) - { - ZeroBinCounters(smem_histograms); - } - else - { - ZeroBinCounters(temp_storage.histograms); - } - } - else - { - ZeroBinCounters(d_privatized_histograms); - } + WithPrivatizedHistograms([&](auto& privatized_histograms) { + ZeroBinCounters(privatized_histograms); + }); } //! Store privatized histogram to device-accessible memory. Specialized for privatized shared-memory counters _CCCL_DEVICE _CCCL_FORCEINLINE void StoreOutput() { - if (prefer_smem) - { - if constexpr (UseDynamicSmem) - { - StoreOutput(smem_histograms); - } - else - { - StoreOutput(temp_storage.histograms); - } - } - else - { - StoreOutput(d_privatized_histograms); - } + WithPrivatizedHistograms([&](auto& privatized_histograms) { + StoreOutput(privatized_histograms); + }); } }; } // namespace detail::histogram diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 7fb9bee8b912..4e5ff3d2cdd3 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -278,8 +278,10 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( }(); constexpr bool use_static_smem = PRIVATIZED_SMEM_BINS > 0 && !UseDynamicSmem; - const int threads_per_block = use_static_smem ? active_policy.static_smem_threads() : active_policy.threads_per_block; - const int items_per_thread = use_static_smem ? active_policy.static_smem_items() : active_policy.pixels_per_thread; + const int threads_per_block = + use_static_smem ? active_policy.static_smem_threads() : active_policy.sweep_threads_per_block; + const int items_per_thread = + use_static_smem ? active_policy.static_smem_items() : active_policy.sweep_items_per_thread; int dynamic_smem_bytes = 0; if constexpr (UseDynamicSmem) diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index 4d8a73d690a2..a5d205fee61d 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -189,8 +189,7 @@ struct Transforms //! verifies an interpolated guess, checks one adjacent bracket, and finally //! falls back to `UpperBound` for arbitrary level distributions. template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void - BinSelectImpl(_SampleT sample, int& bin, bool valid, BinSelectState& cache) const + _CCCL_DEVICE _CCCL_FORCEINLINE void BinSelectImpl(_SampleT sample, int& bin, bool valid, BinSelectState& cache) const { using WrappedLevelIteratorT = ::cuda::std::_If<::cuda::std::is_pointer_v, @@ -715,9 +714,8 @@ template #endif // _CCCL_HAS_CONCEPTS() __launch_bounds__(int(PrivatizedSmemBins > 0 ? current_policy().static_smem_threads() - : current_policy().threads_per_block), - int(PrivatizedSmemBins > 0 ? current_policy().static_smem_min_blocks() - : (current_policy().threads_per_block >= 512 ? 2 : 0))) + : current_policy().sweep_threads_per_block), + int(PrivatizedSmemBins > 0 ? current_policy().static_smem_min_blocks() : 0)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepKernel( const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, @@ -735,8 +733,8 @@ __launch_bounds__(int(PrivatizedSmemBins > 0 ? current_policy(). static constexpr HistogramPolicy hp = current_policy(); // Thread block type for compositing input tiles - static constexpr int sweep_threads = PrivatizedSmemBins > 0 ? hp.static_smem_threads() : hp.threads_per_block; - static constexpr int sweep_items = PrivatizedSmemBins > 0 ? hp.static_smem_items() : hp.pixels_per_thread; + static constexpr int sweep_threads = PrivatizedSmemBins > 0 ? hp.static_smem_threads() : hp.sweep_threads_per_block; + static constexpr int sweep_items = PrivatizedSmemBins > 0 ? hp.static_smem_items() : hp.sweep_items_per_thread; using AgentHistogramPolicyT = agent_histogram_policy< sweep_threads, sweep_items, @@ -802,7 +800,7 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(current_policy().threads_per_block)) +__launch_bounds__(int(current_policy().sweep_threads_per_block)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDynamicSmemKernel( const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, @@ -820,8 +818,8 @@ __launch_bounds__(int(current_policy().threads_per_block)) static constexpr HistogramPolicy hp = current_policy(); using AgentHistogramPolicyT = agent_histogram_policy< - hp.threads_per_block, - hp.pixels_per_thread, + hp.sweep_threads_per_block, + hp.sweep_items_per_thread, hp.load_algorithm, hp.load_modifier, hp.rle_compress, @@ -971,7 +969,7 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(current_policy().threads_per_block)) +__launch_bounds__(int(current_policy().sweep_threads_per_block)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDeviceInitKernel( const SampleIteratorT d_samples, ::cuda::std::array num_output_bins_wrapper, @@ -1016,8 +1014,8 @@ __launch_bounds__(int(current_policy().threads_per_block)) // Thread block type for compositing input tiles using AgentHistogramPolicyT = agent_histogram_policy< - hp.threads_per_block, - hp.pixels_per_thread, + hp.sweep_threads_per_block, + hp.sweep_items_per_thread, hp.load_algorithm, hp.load_modifier, hp.rle_compress, diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 058696615fbc..97e27639dd8b 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -28,8 +28,8 @@ CUB_NAMESPACE_BEGIN //! The tuning policy for all algorithms in @ref DeviceHistogram. struct HistogramPolicy { - int threads_per_block; //!< Number of threads in a CUDA block - int pixels_per_thread; //!< Number of pixels processed per thread + int sweep_threads_per_block; //!< Dynamic shared-memory and global-memory sweep threads + int sweep_items_per_thread; //!< Dynamic shared-memory and global-memory sweep items per thread int vec_size; //!< Vectorization size for loading samples BlockLoadAlgorithm load_algorithm; //!< The @ref BlockLoadAlgorithm used for loading samples from global memory CacheLoadModifier load_modifier; //!< The @ref CacheLoadModifier used for loading samples from global memory @@ -40,9 +40,9 @@ struct HistogramPolicy int init_kernel_pdl_trigger_max_bins; //!< Maximum number of bins for the init kernel to trigger the histogram kernel //!< early using PDL int dynamic_smem_bytes = 0; //!< Tuned byte budget for a runtime-sized privatized histogram; 0 disables it - int static_smem_threads_per_block = 0; //!< Static shared-memory tier threads; 0 inherits threads_per_block - int static_smem_items_per_thread = 0; //!< Static shared-memory tier items; 0 inherits pixels_per_thread - int static_smem_min_blocks_per_sm = 0; //!< Static shared-memory launch bound; 0 derives it from the block size + int static_smem_threads_per_block = 0; //!< Static shared-memory tier threads; 0 inherits the dynamic tier + int static_smem_items_per_thread = 0; //!< Static shared-memory tier items; 0 inherits the dynamic tier + int static_smem_min_blocks_per_sm = 0; //!< Static shared-memory minimum blocks per SM; 0 leaves it unspecified int dynamic_smem_range_max_bins = 0; //!< Multi-channel RANGE cap per channel; 0 disables the dynamic path int dynamic_smem_even_2ch_max_bins = 0; //!< Two-channel EVEN cap per channel; 0 disables the dynamic path int dynamic_smem_even_3ch_max_bins = 0; //!< Three-channel EVEN cap per channel; 0 disables the dynamic path @@ -50,26 +50,27 @@ struct HistogramPolicy [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int static_smem_threads() const { - return static_smem_threads_per_block != 0 ? static_smem_threads_per_block : threads_per_block; + return static_smem_threads_per_block != 0 ? static_smem_threads_per_block : sweep_threads_per_block; } [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int static_smem_items() const { - return static_smem_items_per_thread != 0 ? static_smem_items_per_thread : pixels_per_thread; + return static_smem_items_per_thread != 0 ? static_smem_items_per_thread : sweep_items_per_thread; } [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int static_smem_min_blocks() const { - return static_smem_min_blocks_per_sm != 0 ? static_smem_min_blocks_per_sm : (static_smem_threads() >= 512 ? 2 : 0); + return static_smem_min_blocks_per_sm; } [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(const HistogramPolicy& lhs, const HistogramPolicy& rhs) noexcept { - return lhs.threads_per_block == rhs.threads_per_block && lhs.pixels_per_thread == rhs.pixels_per_thread - && lhs.vec_size == rhs.vec_size && lhs.load_algorithm == rhs.load_algorithm - && lhs.load_modifier == rhs.load_modifier && lhs.rle_compress == rhs.rle_compress - && lhs.mem_preference == rhs.mem_preference && lhs.use_work_stealing == rhs.use_work_stealing + return lhs.sweep_threads_per_block == rhs.sweep_threads_per_block + && lhs.sweep_items_per_thread == rhs.sweep_items_per_thread && lhs.vec_size == rhs.vec_size + && lhs.load_algorithm == rhs.load_algorithm && lhs.load_modifier == rhs.load_modifier + && lhs.rle_compress == rhs.rle_compress && lhs.mem_preference == rhs.mem_preference + && lhs.use_work_stealing == rhs.use_work_stealing && lhs.init_kernel_pdl_trigger_max_bins == rhs.init_kernel_pdl_trigger_max_bins && lhs.dynamic_smem_bytes == rhs.dynamic_smem_bytes && lhs.static_smem_threads_per_block == rhs.static_smem_threads_per_block @@ -91,12 +92,13 @@ struct HistogramPolicy friend ::std::ostream& operator<<(::std::ostream& os, const HistogramPolicy& p) { return os - << "HistogramPolicy { .threads_per_block = " << p.threads_per_block << ", .pixels_per_thread = " - << p.pixels_per_thread << ", .vec_size = " << p.vec_size << ", .load_algorithm = " << p.load_algorithm - << ", .load_modifier = " << p.load_modifier << ", .rle_compress = " << p.rle_compress - << ", .mem_preference = " << p.mem_preference << ", .use_work_stealing = " << p.use_work_stealing - << ", .init_kernel_pdl_trigger_max_bins = " << p.init_kernel_pdl_trigger_max_bins << ", .dynamic_smem_bytes = " - << p.dynamic_smem_bytes << ", .static_smem_threads_per_block = " << p.static_smem_threads_per_block + << "HistogramPolicy { .sweep_threads_per_block = " << p.sweep_threads_per_block + << ", .sweep_items_per_thread = " << p.sweep_items_per_thread << ", .vec_size = " << p.vec_size + << ", .load_algorithm = " << p.load_algorithm << ", .load_modifier = " << p.load_modifier + << ", .rle_compress = " << p.rle_compress << ", .mem_preference = " << p.mem_preference + << ", .use_work_stealing = " << p.use_work_stealing << ", .init_kernel_pdl_trigger_max_bins = " + << p.init_kernel_pdl_trigger_max_bins << ", .dynamic_smem_bytes = " << p.dynamic_smem_bytes + << ", .static_smem_threads_per_block = " << p.static_smem_threads_per_block << ", .static_smem_items_per_thread = " << p.static_smem_items_per_thread << ", .static_smem_min_blocks_per_sm = " << p.static_smem_min_blocks_per_sm << ", .dynamic_smem_range_max_bins = " << p.dynamic_smem_range_max_bins diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index a6f3c6ad64fb..be4196c84279 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1826,8 +1826,8 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) # if _CCCL_STD_VER >= 2020 // designated init constexpr auto p2 = cub::HistogramPolicy{ - .threads_per_block = 128, - .pixels_per_thread = 7, + .sweep_threads_per_block = 128, + .sweep_items_per_thread = 7, .vec_size = 4, .load_algorithm = cub::BLOCK_LOAD_DIRECT, .load_modifier = cub::CacheLoadModifier::LOAD_LDG, @@ -1858,7 +1858,7 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) }; REQUIRE( to_string(p1) - == "HistogramPolicy { .threads_per_block = 128, .pixels_per_thread = 7, .vec_size = 4" + == "HistogramPolicy { .sweep_threads_per_block = 128, .sweep_items_per_thread = 7, .vec_size = 4" ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" ", .mem_preference = SMEM, .use_work_stealing = 0, .init_kernel_pdl_trigger_max_bins = 2048" ", .dynamic_smem_bytes = 12345, .static_smem_threads_per_block = 96" diff --git a/cub/test/catch2_test_device_histogram_env_api.cu b/cub/test/catch2_test_device_histogram_env_api.cu index c79fa6ce12a9..db34d542eb76 100644 --- a/cub/test/catch2_test_device_histogram_env_api.cu +++ b/cub/test/catch2_test_device_histogram_env_api.cu @@ -418,8 +418,8 @@ struct HistogramPolicySelector { __host__ __device__ constexpr auto operator()(cuda::compute_capability cc) const -> cub::HistogramPolicy { - return {.threads_per_block = 128, - .pixels_per_thread = cc > cuda::compute_capability{9, 0} ? 16 : 7, + return {.sweep_threads_per_block = 128, + .sweep_items_per_thread = cc > cuda::compute_capability{9, 0} ? 16 : 7, .vec_size = 4, .load_algorithm = cub::BLOCK_LOAD_DIRECT, .load_modifier = cub::LOAD_LDG, From 322db8ba9f1a20bdbf00b837f41ad55d144ec717 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sat, 1 Aug 2026 11:10:18 +0000 Subject: [PATCH 14/45] [cub] Simplify privatized histogram storage selection --- cub/cub/agent/agent_histogram.cuh | 78 ++++++++----------- .../device/dispatch/dispatch_histogram.cuh | 8 +- 2 files changed, 37 insertions(+), 49 deletions(-) diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index 14624ae55f88..ac45a19ffaa2 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -270,35 +270,32 @@ struct AgentHistogram // channel bool prefer_smem; // for privatized counterss - template - _CCCL_DEVICE _CCCL_FORCEINLINE void WithPrivatizedHistograms(F&& f) + _CCCL_DEVICE _CCCL_FORCEINLINE CounterT* PrivatizedHistogram(int channel) { if (prefer_smem) { if constexpr (UseDynamicSmem) { - f(dyn_smem_histograms); + return dyn_smem_histograms[channel]; } else { - f(temp_storage.histograms); + return temp_storage.histograms[channel]; } } - else - { - f(d_privatized_histograms); - } + + return d_privatized_histograms[channel]; } - template - _CCCL_DEVICE _CCCL_FORCEINLINE void ZeroBinCounters(TwoDimSubscriptableCounterT& privatized_histograms) + _CCCL_DEVICE _CCCL_FORCEINLINE void ZeroBinCounters() { _CCCL_PRAGMA_UNROLL_FULL() for (int ch = 0; ch < NumActiveChannels; ++ch) { + CounterT* privatized_histogram = PrivatizedHistogram(ch); for (int bin = static_cast(threadIdx.x); bin < num_privatized_bins[ch]; bin += threads_per_block) { - privatized_histograms[ch][bin] = 0; + privatized_histogram[bin] = 0; } } @@ -308,8 +305,7 @@ struct AgentHistogram } // Update final output histograms from privatized histograms - template - _CCCL_DEVICE _CCCL_FORCEINLINE void StoreOutput(TwoDimSubscriptableCounterT& privatized_histograms) + _CCCL_DEVICE _CCCL_FORCEINLINE void StoreOutputImpl() { // Barrier to make sure all threads are done updating counters __syncthreads(); @@ -318,11 +314,12 @@ struct AgentHistogram _CCCL_PRAGMA_UNROLL_FULL() for (int ch = 0; ch < NumActiveChannels; ++ch) { - const int channel_bins = num_privatized_bins[ch]; + CounterT* privatized_histogram = PrivatizedHistogram(ch); + const int channel_bins = num_privatized_bins[ch]; for (int bin = static_cast(threadIdx.x); bin < channel_bins; bin += threads_per_block) { int output_bin = -1; - const CounterT count = privatized_histograms[ch][bin]; + const CounterT count = privatized_histogram[bin]; const bool is_valid = count > 0; output_decode_op[ch].template BinSelect(static_cast(bin), output_bin, is_valid); @@ -335,16 +332,15 @@ struct AgentHistogram } // Accumulate pixels. Specialized for RLE compression. - template _CCCL_DEVICE _CCCL_FORCEINLINE void AccumulatePixels( SampleT samples[pixels_per_thread][NumChannels], bool is_valid[pixels_per_thread], - TwoDimSubscriptableCounterT& privatized_histograms, ::cuda::std::true_type is_rle_compress) { _CCCL_PRAGMA_UNROLL_FULL() for (int ch = 0; ch < NumActiveChannels; ++ch) { + CounterT* privatized_histogram = PrivatizedHistogram(ch); // Bin pixels int bins[pixels_per_thread]; @@ -365,8 +361,8 @@ struct AgentHistogram if (bins[pixel] >= 0) { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_60, - (atomicAdd_block(privatized_histograms[ch] + bins[pixel], accumulator);), - (atomicAdd(privatized_histograms[ch] + bins[pixel], accumulator);)); + (atomicAdd_block(privatized_histogram + bins[pixel], accumulator);), + (atomicAdd(privatized_histogram + bins[pixel], accumulator);)); } accumulator = 0; @@ -378,18 +374,16 @@ struct AgentHistogram if (bins[pixels_per_thread - 1] >= 0) { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_60, - (atomicAdd_block(privatized_histograms[ch] + bins[pixels_per_thread - 1], accumulator);), - (atomicAdd(privatized_histograms[ch] + bins[pixels_per_thread - 1], accumulator);)); + (atomicAdd_block(privatized_histogram + bins[pixels_per_thread - 1], accumulator);), + (atomicAdd(privatized_histogram + bins[pixels_per_thread - 1], accumulator);)); } } } // Accumulate pixels. Specialized for individual accumulation of each pixel. - template _CCCL_DEVICE _CCCL_FORCEINLINE void AccumulatePixels( SampleT samples[pixels_per_thread][NumChannels], bool is_valid[pixels_per_thread], - TwoDimSubscriptableCounterT& privatized_histograms, ::cuda::std::false_type is_rle_compress) { _CCCL_PRAGMA_UNROLL_FULL() @@ -398,13 +392,14 @@ struct AgentHistogram _CCCL_PRAGMA_UNROLL_FULL() for (int ch = 0; ch < NumActiveChannels; ++ch) { - int bin = -1; + CounterT* privatized_histogram = PrivatizedHistogram(ch); + int bin = -1; privatized_decode_op[ch].template BinSelect(samples[pixel][ch], bin, is_valid[pixel]); if (bin >= 0) { NV_IF_ELSE_TARGET(NV_PROVIDES_SM_60, - (atomicAdd_block(privatized_histograms[ch] + bin, 1);), - (atomicAdd(privatized_histograms[ch] + bin, 1);)); + (atomicAdd_block(privatized_histogram + bin, 1);), + (atomicAdd(privatized_histogram + bin, 1);)); } } } @@ -503,9 +498,7 @@ struct AgentHistogram LoadTile(block_offset, valid_samples, samples); MarkValid(is_valid, valid_samples); - WithPrivatizedHistograms([&](auto& privatized_histograms) { - AccumulatePixels(samples, is_valid, privatized_histograms, ::cuda::std::bool_constant{}); - }); + AccumulatePixels(samples, is_valid, ::cuda::std::bool_constant{}); } //! @brief Consume row tiles. Specialized for work-stealing from queue @@ -661,14 +654,14 @@ struct AgentHistogram "AgentHistogram with UseDynamicSmem=true requires the dynamic-SMEM " "constructor that takes an extern __shared__ base pointer."); - const int blockId = static_cast((blockIdx.y * gridDim.x) + blockIdx.x); - - // TODO(bgruber): d_privatized_histograms seems only used when !prefer_smem, can we skip it if prefer_smem? - // Initialize the locations of this block's privatized histograms - for (int ch = 0; ch < NumActiveChannels; ++ch) + if (!prefer_smem) { - const auto offset = static_cast<::cuda::std::int64_t>(blockId) * num_privatized_bins[ch]; - this->d_privatized_histograms[ch] = d_privatized_histograms[ch] + offset; + const int block_id = static_cast((blockIdx.y * gridDim.x) + blockIdx.x); + for (int ch = 0; ch < NumActiveChannels; ++ch) + { + const auto offset = static_cast<::cuda::std::int64_t>(block_id) * num_privatized_bins[ch]; + this->d_privatized_histograms[ch] = d_privatized_histograms[ch] + offset; + } } } @@ -695,11 +688,6 @@ struct AgentHistogram { static_assert(UseDynamicSmem, "Dynamic-SMEM AgentHistogram constructor requires UseDynamicSmem=true."); - for (int ch = 0; ch < NumActiveChannels; ++ch) - { - this->d_privatized_histograms[ch] = d_privatized_histograms[ch]; - } - CounterT* p = dyn_smem_histogram_base; _CCCL_PRAGMA_UNROLL_FULL() for (int ch = 0; ch < NumActiveChannels; ++ch) @@ -764,17 +752,13 @@ struct AgentHistogram //! Initialize privatized bin counters. Specialized for privatized shared-memory counters _CCCL_DEVICE _CCCL_FORCEINLINE void InitBinCounters() { - WithPrivatizedHistograms([&](auto& privatized_histograms) { - ZeroBinCounters(privatized_histograms); - }); + ZeroBinCounters(); } //! Store privatized histogram to device-accessible memory. Specialized for privatized shared-memory counters _CCCL_DEVICE _CCCL_FORCEINLINE void StoreOutput() { - WithPrivatizedHistograms([&](auto& privatized_histograms) { - StoreOutput(privatized_histograms); - }); + StoreOutputImpl(); } }; } // namespace detail::histogram diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 4e5ff3d2cdd3..e2833aeebd27 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -345,11 +345,15 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( void* allocations[NUM_ALLOCATIONS] = {}; size_t allocation_sizes[NUM_ALLOCATIONS]; + const bool requires_global_privatization = + !UseDynamicSmem + && (PRIVATIZED_SMEM_BINS == 0 || active_policy.mem_preference != BlockHistogramMemoryPreference::SMEM); for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL) { allocation_sizes[CHANNEL] = - UseDynamicSmem ? 0 - : size_t(num_thread_blocks) * (num_privatized_levels[CHANNEL] - 1) * kernel_source.CounterSize(); + requires_global_privatization + ? size_t(num_thread_blocks) * (num_privatized_levels[CHANNEL] - 1) * kernel_source.CounterSize() + : 0; } allocation_sizes[NUM_ALLOCATIONS - 1] = GridQueue::AllocationSize(); From 5adb7bce4f57e08d4e9cdfb023b76254f8831cc0 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sat, 1 Aug 2026 11:12:05 +0000 Subject: [PATCH 15/45] [cub] Clarify histogram dispatch comment --- cub/cub/device/dispatch/dispatch_histogram.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index e2833aeebd27..70f95593e656 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -587,7 +587,7 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device if (max_num_output_bins > detail::histogram::max_privatized_smem_bins) { - // Dispatch shared-privatized approach + // Dispatch global-memory-privatized approach constexpr int PRIVATIZED_SMEM_BINS = 0; if (const auto error = CubDebug( From f552d68536a8e9b25bc07c1153049303f077871c Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sat, 1 Aug 2026 14:52:34 +0000 Subject: [PATCH 16/45] [cub] Align histogram work stealing policy name --- .../device/dispatch/kernels/kernel_histogram.cuh | 6 +++--- cub/cub/device/dispatch/tuning/tuning_histogram.cuh | 13 ++++++------- cub/test/catch2_test_device_histogram_env.cu | 4 ++-- cub/test/catch2_test_device_histogram_env_api.cu | 2 +- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index a5d205fee61d..0eb5c12e8030 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -742,7 +742,7 @@ __launch_bounds__(int(PrivatizedSmemBins > 0 ? current_policy(). hp.load_modifier, hp.rle_compress, hp.mem_preference, - hp.use_work_stealing, + hp.work_stealing, hp.vec_size>; using AgentHistogramT = AgentHistogram().sweep_threads_per_block)) hp.load_modifier, hp.rle_compress, hp.mem_preference, - hp.use_work_stealing, + hp.work_stealing, hp.vec_size>; using AgentHistogramT = AgentHistogram().sweep_threads_per_block)) hp.load_modifier, hp.rle_compress, hp.mem_preference, - hp.use_work_stealing, + hp.work_stealing, hp.vec_size>; using AgentHistogramT = AgentHistogram Date: Sat, 1 Aug 2026 15:28:39 +0000 Subject: [PATCH 17/45] [cub] Remove histogram memory preference --- cub/benchmarks/bench/histogram/even.cu | 1 - .../bench/histogram/histogram_common.cuh | 9 -- cub/benchmarks/bench/histogram/multi/even.cu | 1 - cub/benchmarks/bench/histogram/multi/range.cu | 1 - cub/benchmarks/bench/histogram/range.cu | 1 - cub/cub/agent/agent_histogram.cuh | 54 ++------ .../device/dispatch/dispatch_histogram.cuh | 5 +- .../dispatch/kernels/kernel_histogram.cuh | 51 ++++---- .../dispatch/tuning/tuning_histogram.cuh | 123 ++++++++---------- cub/test/catch2_test_device_histogram_env.cu | 8 +- .../catch2_test_device_histogram_env_api.cu | 1 - cub/test/catch2_test_enum_formatting.cu | 1 - 12 files changed, 97 insertions(+), 159 deletions(-) diff --git a/cub/benchmarks/bench/histogram/even.cu b/cub/benchmarks/bench/histogram/even.cu index a175139b1b99..28465245171a 100644 --- a/cub/benchmarks/bench/histogram/even.cu +++ b/cub/benchmarks/bench/histogram/even.cu @@ -9,7 +9,6 @@ // %RANGE% TUNE_THREADS tpb 128:1024:32 // %RANGE% TUNE_RLE_COMPRESS rle 0:1:1 // %RANGE% TUNE_WORK_STEALING ws 0:1:1 -// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1 // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 diff --git a/cub/benchmarks/bench/histogram/histogram_common.cuh b/cub/benchmarks/bench/histogram/histogram_common.cuh index c87565264042..ab08b84beee4 100644 --- a/cub/benchmarks/bench/histogram/histogram_common.cuh +++ b/cub/benchmarks/bench/histogram/histogram_common.cuh @@ -19,14 +19,6 @@ # define TUNE_VEC_SIZE (1 << TUNE_VEC_SIZE_POW) -# if TUNE_MEM_PREFERENCE == 0 -constexpr cub::BlockHistogramMemoryPreference MEM_PREFERENCE = cub::GMEM; -# elif TUNE_MEM_PREFERENCE == 1 -constexpr cub::BlockHistogramMemoryPreference MEM_PREFERENCE = cub::SMEM; -# else // TUNE_MEM_PREFERENCE == 2 -constexpr cub::BlockHistogramMemoryPreference MEM_PREFERENCE = cub::BLEND; -# endif // TUNE_MEM_PREFERENCE - # if TUNE_LOAD_ALGORITHM_ID == 0 # define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_DIRECT # elif TUNE_LOAD_ALGORITHM_ID == 1 @@ -51,7 +43,6 @@ struct bench_policy_selector load_algorithm, TUNE_LOAD_MODIFIER, TUNE_RLE_COMPRESS, - MEM_PREFERENCE, TUNE_WORK_STEALING, 2048}; // TODO(bgruber): make tunable } diff --git a/cub/benchmarks/bench/histogram/multi/even.cu b/cub/benchmarks/bench/histogram/multi/even.cu index 4131206a8bd7..0d6b4c51e191 100644 --- a/cub/benchmarks/bench/histogram/multi/even.cu +++ b/cub/benchmarks/bench/histogram/multi/even.cu @@ -9,7 +9,6 @@ // %RANGE% TUNE_THREADS tpb 128:1024:32 // %RANGE% TUNE_RLE_COMPRESS rle 0:1:1 // %RANGE% TUNE_WORK_STEALING ws 0:1:1 -// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1 // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 diff --git a/cub/benchmarks/bench/histogram/multi/range.cu b/cub/benchmarks/bench/histogram/multi/range.cu index dcb402d7db20..2d786ce95cbb 100644 --- a/cub/benchmarks/bench/histogram/multi/range.cu +++ b/cub/benchmarks/bench/histogram/multi/range.cu @@ -11,7 +11,6 @@ // %RANGE% TUNE_THREADS tpb 128:1024:32 // %RANGE% TUNE_RLE_COMPRESS rle 0:1:1 // %RANGE% TUNE_WORK_STEALING ws 0:1:1 -// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1 // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 diff --git a/cub/benchmarks/bench/histogram/range.cu b/cub/benchmarks/bench/histogram/range.cu index e1e808466395..4c989ede589b 100644 --- a/cub/benchmarks/bench/histogram/range.cu +++ b/cub/benchmarks/bench/histogram/range.cu @@ -11,7 +11,6 @@ // %RANGE% TUNE_THREADS tpb 128:1024:32 // %RANGE% TUNE_RLE_COMPRESS rle 0:1:1 // %RANGE% TUNE_WORK_STEALING ws 0:1:1 -// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1 // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index ac45a19ffaa2..7c0195406b02 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -36,8 +36,7 @@ CUB_NAMESPACE_BEGIN enum BlockHistogramMemoryPreference { GMEM, - SMEM, - BLEND + SMEM }; #if _CCCL_HOSTED() @@ -51,8 +50,6 @@ namespace detail return "GMEM"; case SMEM: return "SMEM"; - case BLEND: - return "BLEND"; } return ""; } @@ -88,7 +85,6 @@ template struct agent_histogram_policy @@ -101,9 +97,6 @@ struct agent_histogram_policy /// Whether to perform localized RLE to compress samples before histogramming static constexpr bool IS_RLE_COMPRESS = RleCompress; - /// Whether to prefer privatized shared-memory bins (versus privatized global-memory bins) - static constexpr BlockHistogramMemoryPreference MEM_PREFERENCE = MemoryPreference; - /// Whether to dequeue tiles from a global work queue static constexpr bool IS_WORK_STEALING = WorkStealing; @@ -128,16 +121,8 @@ template -using AgentHistogramPolicy - CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") = detail::agent_histogram_policy< - ThreadsPerBlock, - PixelsPerThread, - LoadAlgorithm, - LoadModifier, - RleCompress, - MemoryPreference, - WorkStealing, - VecSize>; +using AgentHistogramPolicy CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") = detail:: + agent_histogram_policy; namespace detail::histogram { @@ -219,8 +204,6 @@ struct AgentHistogram static constexpr bool is_rle_compress = AgentHistogramPolicyT::IS_RLE_COMPRESS; static constexpr bool is_work_stealing = AgentHistogramPolicyT::IS_WORK_STEALING; static constexpr CacheLoadModifier load_modifier = AgentHistogramPolicyT::LOAD_MODIFIER; - static constexpr auto mem_preference = - (PrivatizedSmemBins > 0) ? BlockHistogramMemoryPreference{AgentHistogramPolicyT::MEM_PREFERENCE} : GMEM; using SampleT = it_value_t; using PixelT = typename CubVector::Type; @@ -268,23 +251,20 @@ struct AgentHistogram // channel const PrivatizedDecodeOpT* privatized_decode_op; // determines privatized counter index from sample, one for each // channel - bool prefer_smem; // for privatized counterss - _CCCL_DEVICE _CCCL_FORCEINLINE CounterT* PrivatizedHistogram(int channel) { - if (prefer_smem) + if constexpr (UseDynamicSmem) { - if constexpr (UseDynamicSmem) - { - return dyn_smem_histograms[channel]; - } - else - { - return temp_storage.histograms[channel]; - } + return dyn_smem_histograms[channel]; + } + else if constexpr (PrivatizedSmemBins > 0) + { + return temp_storage.histograms[channel]; + } + else + { + return d_privatized_histograms[channel]; } - - return d_privatized_histograms[channel]; } _CCCL_DEVICE _CCCL_FORCEINLINE void ZeroBinCounters() @@ -299,7 +279,6 @@ struct AgentHistogram } } - // TODO(bgruber): do we also need the __syncthreads() when prefer_smem is false? // Barrier to make sure all threads are done updating counters __syncthreads(); } @@ -645,16 +624,12 @@ struct AgentHistogram , d_output_histograms(d_output_histograms) , output_decode_op(output_decode_op) , privatized_decode_op(privatized_decode_op) - , prefer_smem((mem_preference == SMEM) ? true : // prefer smem privatized histograms - (mem_preference == GMEM) ? false - : // prefer gmem privatized histograms - blockIdx.x & 1) // prefer blended privatized histograms { static_assert(!UseDynamicSmem, "AgentHistogram with UseDynamicSmem=true requires the dynamic-SMEM " "constructor that takes an extern __shared__ base pointer."); - if (!prefer_smem) + if constexpr (PrivatizedSmemBins == 0) { const int block_id = static_cast((blockIdx.y * gridDim.x) + blockIdx.x); for (int ch = 0; ch < NumActiveChannels; ++ch) @@ -684,7 +659,6 @@ struct AgentHistogram , d_output_histograms(d_output_histograms) , output_decode_op(output_decode_op) , privatized_decode_op(privatized_decode_op) - , prefer_smem(true) { static_assert(UseDynamicSmem, "Dynamic-SMEM AgentHistogram constructor requires UseDynamicSmem=true."); diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 70f95593e656..e9605b178ec9 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -345,9 +345,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( void* allocations[NUM_ALLOCATIONS] = {}; size_t allocation_sizes[NUM_ALLOCATIONS]; - const bool requires_global_privatization = - !UseDynamicSmem - && (PRIVATIZED_SMEM_BINS == 0 || active_policy.mem_preference != BlockHistogramMemoryPreference::SMEM); + constexpr bool requires_global_privatization = !UseDynamicSmem && PRIVATIZED_SMEM_BINS == 0; for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL) { allocation_sizes[CHANNEL] = @@ -927,7 +925,6 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_policy() -> HistogramPolicy ap::LOAD_ALGORITHM, ap::LOAD_MODIFIER, ap::IS_RLE_COMPRESS, - ap::MEM_PREFERENCE, ap::IS_WORK_STEALING, convert_pdl_trigger(0), convert_dynamic_smem_bytes(0), diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index 0eb5c12e8030..a59533c01314 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -735,15 +735,14 @@ __launch_bounds__(int(PrivatizedSmemBins > 0 ? current_policy(). // Thread block type for compositing input tiles static constexpr int sweep_threads = PrivatizedSmemBins > 0 ? hp.static_smem_threads() : hp.sweep_threads_per_block; static constexpr int sweep_items = PrivatizedSmemBins > 0 ? hp.static_smem_items() : hp.sweep_items_per_thread; - using AgentHistogramPolicyT = agent_histogram_policy< - sweep_threads, - sweep_items, - hp.load_algorithm, - hp.load_modifier, - hp.rle_compress, - hp.mem_preference, - hp.work_stealing, - hp.vec_size>; + using AgentHistogramPolicyT = + agent_histogram_policy; using AgentHistogramT = AgentHistogram().sweep_threads_per_block)) { static constexpr HistogramPolicy hp = current_policy(); - using AgentHistogramPolicyT = agent_histogram_policy< - hp.sweep_threads_per_block, - hp.sweep_items_per_thread, - hp.load_algorithm, - hp.load_modifier, - hp.rle_compress, - hp.mem_preference, - hp.work_stealing, - hp.vec_size>; + using AgentHistogramPolicyT = + agent_histogram_policy; using AgentHistogramT = AgentHistogram().sweep_threads_per_block)) } // Thread block type for compositing input tiles - using AgentHistogramPolicyT = agent_histogram_policy< - hp.sweep_threads_per_block, - hp.sweep_items_per_thread, - hp.load_algorithm, - hp.load_modifier, - hp.rle_compress, - hp.mem_preference, - hp.work_stealing, - hp.vec_size>; + using AgentHistogramPolicyT = + agent_histogram_policy; using AgentHistogramT = AgentHistogram"); } From 2b84b74b45160d28d68a6eb7948d779c1fa2c9ca Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sat, 1 Aug 2026 15:37:21 +0000 Subject: [PATCH 18/45] [cub] Simplify histogram kernel configuration --- cub/cub/agent/agent_histogram.cuh | 19 ++---- .../device/dispatch/dispatch_histogram.cuh | 8 +-- .../dispatch/kernels/kernel_histogram.cuh | 46 ++++++--------- .../dispatch/tuning/tuning_histogram.cuh | 58 ++++++++++++++----- 4 files changed, 70 insertions(+), 61 deletions(-) diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index 7c0195406b02..db394f73ee6b 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -249,8 +249,7 @@ struct AgentHistogram OutputCounterT** d_output_histograms; // final output, in global memory const OutputDecodeOpT* output_decode_op; // determines output bin-id from privatized counter index, one for each // channel - const PrivatizedDecodeOpT* privatized_decode_op; // determines privatized counter index from sample, one for each - // channel + PrivatizedDecodeOpT* privatized_decode_op; // determines privatized counter index from sample, one for each channel _CCCL_DEVICE _CCCL_FORCEINLINE CounterT* PrivatizedHistogram(int channel) { if constexpr (UseDynamicSmem) @@ -339,9 +338,7 @@ struct AgentHistogram { if (bins[pixel] >= 0) { - NV_IF_ELSE_TARGET(NV_PROVIDES_SM_60, - (atomicAdd_block(privatized_histogram + bins[pixel], accumulator);), - (atomicAdd(privatized_histogram + bins[pixel], accumulator);)); + atomicAdd_block(privatized_histogram + bins[pixel], accumulator); } accumulator = 0; @@ -352,9 +349,7 @@ struct AgentHistogram // Last pixel if (bins[pixels_per_thread - 1] >= 0) { - NV_IF_ELSE_TARGET(NV_PROVIDES_SM_60, - (atomicAdd_block(privatized_histogram + bins[pixels_per_thread - 1], accumulator);), - (atomicAdd(privatized_histogram + bins[pixels_per_thread - 1], accumulator);)); + atomicAdd_block(privatized_histogram + bins[pixels_per_thread - 1], accumulator); } } } @@ -376,9 +371,7 @@ struct AgentHistogram privatized_decode_op[ch].template BinSelect(samples[pixel][ch], bin, is_valid[pixel]); if (bin >= 0) { - NV_IF_ELSE_TARGET(NV_PROVIDES_SM_60, - (atomicAdd_block(privatized_histogram + bin, 1);), - (atomicAdd(privatized_histogram + bin, 1);)); + atomicAdd_block(privatized_histogram + bin, 1); } } } @@ -615,7 +608,7 @@ struct AgentHistogram OutputCounterT** d_output_histograms, CounterT** d_privatized_histograms, const OutputDecodeOpT* output_decode_op, - const PrivatizedDecodeOpT* privatized_decode_op) + PrivatizedDecodeOpT* privatized_decode_op) : temp_storage(temp_storage.Alias()) , d_wrapped_samples(d_samples) , d_native_samples(NativePointer(d_wrapped_samples)) @@ -649,7 +642,7 @@ struct AgentHistogram OutputCounterT** d_output_histograms, CounterT** d_privatized_histograms, const OutputDecodeOpT* output_decode_op, - const PrivatizedDecodeOpT* privatized_decode_op, + PrivatizedDecodeOpT* privatized_decode_op, CounterT* dyn_smem_histogram_base) : temp_storage(temp_storage.Alias()) , d_wrapped_samples(d_samples) diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index e9605b178ec9..0c79274aaf7a 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -277,11 +277,9 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( } }(); - constexpr bool use_static_smem = PRIVATIZED_SMEM_BINS > 0 && !UseDynamicSmem; - const int threads_per_block = - use_static_smem ? active_policy.static_smem_threads() : active_policy.sweep_threads_per_block; - const int items_per_thread = - use_static_smem ? active_policy.static_smem_items() : active_policy.sweep_items_per_thread; + constexpr int privatized_smem_bins = UseDynamicSmem ? 0 : PRIVATIZED_SMEM_BINS; + const int threads_per_block = detail::histogram::threads_per_block(active_policy); + const int items_per_thread = detail::histogram::items_per_thread(active_policy); int dynamic_smem_bytes = 0; if constexpr (UseDynamicSmem) diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index a59533c01314..7b35eac724d8 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -104,7 +104,7 @@ struct Transforms int bin = -1; // cached bin; < 0 means empty }; - mutable BinSelectState mru; + BinSelectState mru; LevelIteratorT d_levels; // Pointer to levels array int num_output_levels; // Number of levels in array @@ -182,14 +182,13 @@ struct Transforms } } - private: //! @brief Implements cached/interpolated bin selection. //! //! A cached-bracket hit returns immediately. Otherwise, this computes and //! verifies an interpolated guess, checks one adjacent bracket, and finally //! falls back to `UpperBound` for arbitrary level distributions. template - _CCCL_DEVICE _CCCL_FORCEINLINE void BinSelectImpl(_SampleT sample, int& bin, bool valid, BinSelectState& cache) const + _CCCL_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid) { using WrappedLevelIteratorT = ::cuda::std::_If<::cuda::std::is_pointer_v, @@ -204,9 +203,9 @@ struct Transforms const LevelT s = static_cast(sample); - if (cache.bin >= 0 && !(s < cache.lo) && (s < cache.hi)) + if (mru.bin >= 0 && !(s < mru.lo) && (s < mru.hi)) { - bin = cache.bin; + bin = mru.bin; return; } @@ -270,8 +269,8 @@ struct Transforms if (!(s < lvl_lo) && (s < lvl_hi)) { - bin = guess; - cache = BinSelectState{lvl_lo, lvl_hi, guess}; + bin = guess; + mru = BinSelectState{lvl_lo, lvl_hi, guess}; return; } @@ -283,8 +282,8 @@ struct Transforms const LevelT lvl2_lo = wrapped_levels[g2]; if (!(s < lvl2_lo)) { - bin = g2; - cache = BinSelectState{lvl2_lo, lvl_lo, g2}; + bin = g2; + mru = BinSelectState{lvl2_lo, lvl_lo, g2}; return; } } @@ -297,8 +296,8 @@ struct Transforms const LevelT lvl2_hi = wrapped_levels[g2 + 1]; if (s < lvl2_hi) { - bin = g2; - cache = BinSelectState{lvl_hi, lvl2_hi, g2}; + bin = g2; + mru = BinSelectState{lvl_hi, lvl2_hi, g2}; return; } } @@ -312,17 +311,9 @@ struct Transforms } if (bin >= 0) { - cache = BinSelectState{wrapped_levels[bin], wrapped_levels[bin + 1], bin}; + mru = BinSelectState{wrapped_levels[bin], wrapped_levels[bin + 1], bin}; } } - - public: - //! @brief Selects a bin and retains the most recently used bracket in this thread's transform copy. - template - _CCCL_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid) const - { - BinSelectImpl(sample, bin, valid, mru); - } }; // Scales samples to evenly-spaced bins @@ -713,9 +704,8 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(PrivatizedSmemBins > 0 ? current_policy().static_smem_threads() - : current_policy().sweep_threads_per_block), - int(PrivatizedSmemBins > 0 ? current_policy().static_smem_min_blocks() : 0)) +__launch_bounds__(int(threads_per_block(current_policy())), + int(min_blocks_per_sm(current_policy()))) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepKernel( const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, @@ -723,7 +713,7 @@ __launch_bounds__(int(PrivatizedSmemBins > 0 ? current_policy(). ::cuda::std::array d_output_histograms_wrapper, ::cuda::std::array d_privatized_histograms_wrapper, const ::cuda::std::array output_decode_op_wrapper, - const ::cuda::std::array privatized_decode_op_wrapper, + ::cuda::std::array privatized_decode_op_wrapper, const OffsetT num_row_pixels, const OffsetT num_rows, const OffsetT row_stride_samples, @@ -733,11 +723,11 @@ __launch_bounds__(int(PrivatizedSmemBins > 0 ? current_policy(). static constexpr HistogramPolicy hp = current_policy(); // Thread block type for compositing input tiles - static constexpr int sweep_threads = PrivatizedSmemBins > 0 ? hp.static_smem_threads() : hp.sweep_threads_per_block; - static constexpr int sweep_items = PrivatizedSmemBins > 0 ? hp.static_smem_items() : hp.sweep_items_per_thread; + static constexpr int threads_per_block = histogram::threads_per_block(hp); + static constexpr int items_per_thread = histogram::items_per_thread(hp); using AgentHistogramPolicyT = - agent_histogram_policy +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int threads_per_block(const HistogramPolicy& policy) +{ + if constexpr (PrivatizedSmemBins > 0) + { + return policy.static_smem_threads_per_block != 0 + ? policy.static_smem_threads_per_block + : policy.sweep_threads_per_block; + } + else + { + return policy.sweep_threads_per_block; + } +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int items_per_thread(const HistogramPolicy& policy) +{ + if constexpr (PrivatizedSmemBins > 0) + { + return policy.static_smem_items_per_thread != 0 + ? policy.static_smem_items_per_thread + : policy.sweep_items_per_thread; + } + else + { + return policy.sweep_items_per_thread; + } +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int min_blocks_per_sm(const HistogramPolicy& policy) +{ + if constexpr (PrivatizedSmemBins > 0) + { + return policy.static_smem_min_blocks_per_sm; + } + else + { + return 0; + } +} + // Maximum number of bins per channel for the compile-time-sized shared-memory tier. static constexpr int max_privatized_smem_bins = 512; From 37847da1152a0a6879f52e49bac9389f6277c682 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sat, 1 Aug 2026 16:09:27 +0000 Subject: [PATCH 19/45] [cub] Structure histogram tuning by kernel tier --- .../device/dispatch/dispatch_histogram.cuh | 239 +++------ .../dispatch/kernels/kernel_histogram.cuh | 83 ++-- .../dispatch/tuning/tuning_histogram.cuh | 454 +++++++++--------- cub/test/catch2_test_device_histogram_env.cu | 116 +++-- .../catch2_test_device_histogram_env_api.cu | 12 +- 5 files changed, 386 insertions(+), 518 deletions(-) diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 0c79274aaf7a..39468fce6b82 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -91,12 +91,12 @@ struct DeviceHistogramKernelSource } /// Returns the default histogram sweep kernel that receives pre-initialized decode operators from the host. - template + template _CCCL_HIDE_FROM_ABI CUB_RUNTIME_FUNCTION static constexpr auto HistogramSweepKernel() { return &DeviceHistogramSweepKernel< PolicyT, - PRIVATIZED_SMEM_BINS, + UseStaticSmem, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, @@ -124,7 +124,7 @@ struct DeviceHistogramKernelSource /// Returns the device-init histogram sweep kernel that initializes decode operators from level arrays in the kernel. template (); + .template HistogramSweepKernel(); } }(); - constexpr int privatized_smem_bins = UseDynamicSmem ? 0 : PRIVATIZED_SMEM_BINS; - const int threads_per_block = detail::histogram::threads_per_block(active_policy); - const int items_per_thread = detail::histogram::items_per_thread(active_policy); + constexpr auto tier = + UseDynamicSmem ? privatization_tier::dynamic_smem + : UseStaticSmem ? privatization_tier::static_smem + : privatization_tier::gmem; + const HistogramSweepPolicy sweep = sweep_policy(active_policy); + const int threads_per_block = sweep.threads_per_block; + const int items_per_thread = sweep.items_per_thread; int dynamic_smem_bytes = 0; if constexpr (UseDynamicSmem) @@ -290,7 +294,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( } NV_IF_TARGET(NV_IS_HOST, ({ if (const auto error = CubDebug(launcher_factory.set_max_dynamic_smem_size_for( - sweep_kernel, active_policy.dynamic_smem_bytes))) + sweep_kernel, active_policy.dynamic_smem.max_bytes))) { return error; } @@ -343,7 +347,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( void* allocations[NUM_ALLOCATIONS] = {}; size_t allocation_sizes[NUM_ALLOCATIONS]; - constexpr bool requires_global_privatization = !UseDynamicSmem && PRIVATIZED_SMEM_BINS == 0; + constexpr bool requires_global_privatization = !UseDynamicSmem && !UseStaticSmem; for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL) { allocation_sizes[CHANNEL] = @@ -581,15 +585,20 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device } int max_num_output_bins = max_levels - 1; - if (max_num_output_bins > detail::histogram::max_privatized_smem_bins) + ::cuda::compute_capability cc{}; + if (const auto error = CubDebug(launcher_factory.PtxComputeCap(cc))) { - // Dispatch global-memory-privatized approach - constexpr int PRIVATIZED_SMEM_BINS = 0; + return error; + } + const HistogramPolicy active_policy = policy_selector(cc); + if (!should_use_static_smem(active_policy, max_num_output_bins)) + { + // Dispatch global-memory-privatized approach if (const auto error = CubDebug( (detail::histogram::dispatch -_CCCL_HOST_DEVICE_API constexpr auto convert_pdl_trigger(int) - -> decltype(ActivePolicy::init_kernel_pdl_trigger_max_bins) -{ - return ActivePolicy::init_kernel_pdl_trigger_max_bins; -} - -// TODO(bgruber): drop in CCCL 4.0 -template -_CCCL_HOST_DEVICE_API constexpr auto convert_pdl_trigger(long) +_CCCL_HOST_DEVICE_API constexpr auto convert_policy(int) + -> decltype(typename ActivePolicy::GmemPolicy{}, HistogramPolicy{}) { - return 0; + return convert_chained_policy(); } // TODO(bgruber): drop in CCCL 4.0 template -_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_bytes(int) -> decltype(ActivePolicy::dynamic_smem_bytes) +_CCCL_HOST_DEVICE_API constexpr auto convert_policy(long) -> HistogramPolicy { - return ActivePolicy::dynamic_smem_bytes; -} - -// TODO(bgruber): drop in CCCL 4.0 -template -_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_bytes(long) -{ - return 0; -} - -template -_CCCL_HOST_DEVICE_API constexpr auto convert_static_smem_threads_per_block(int) - -> decltype(ActivePolicy::static_smem_threads_per_block) -{ - return ActivePolicy::static_smem_threads_per_block; -} - -template -_CCCL_HOST_DEVICE_API constexpr auto convert_static_smem_threads_per_block(long) -{ - return 0; -} - -template -_CCCL_HOST_DEVICE_API constexpr auto convert_static_smem_items_per_thread(int) - -> decltype(ActivePolicy::static_smem_items_per_thread) -{ - return ActivePolicy::static_smem_items_per_thread; -} - -template -_CCCL_HOST_DEVICE_API constexpr auto convert_static_smem_items_per_thread(long) -{ - return 0; -} - -template -_CCCL_HOST_DEVICE_API constexpr auto convert_static_smem_min_blocks_per_sm(int) - -> decltype(ActivePolicy::static_smem_min_blocks_per_sm) -{ - return ActivePolicy::static_smem_min_blocks_per_sm; -} - -template -_CCCL_HOST_DEVICE_API constexpr auto convert_static_smem_min_blocks_per_sm(long) -{ - return 0; -} - -template -_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_range_max_bins(int) - -> decltype(ActivePolicy::dynamic_smem_range_max_bins) -{ - return ActivePolicy::dynamic_smem_range_max_bins; -} - -template -_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_range_max_bins(long) -{ - return 0; -} - -template -_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_2ch_max_bins(int) - -> decltype(ActivePolicy::dynamic_smem_even_2ch_max_bins) -{ - return ActivePolicy::dynamic_smem_even_2ch_max_bins; -} - -template -_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_2ch_max_bins(long) -{ - return 0; -} - -template -_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_3ch_max_bins(int) - -> decltype(ActivePolicy::dynamic_smem_even_3ch_max_bins) -{ - return ActivePolicy::dynamic_smem_even_3ch_max_bins; -} - -template -_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_3ch_max_bins(long) -{ - return 0; -} - -template -_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_4ch_max_bins(int) - -> decltype(ActivePolicy::dynamic_smem_even_4ch_max_bins) -{ - return ActivePolicy::dynamic_smem_even_4ch_max_bins; -} - -template -_CCCL_HOST_DEVICE_API constexpr auto convert_dynamic_smem_even_4ch_max_bins(long) -{ - return 0; -} - -// TODO(bgruber): drop in CCCL 4.0 -template -_CCCL_HOST_DEVICE_API constexpr auto convert_policy() -> HistogramPolicy -{ - using ap = typename ActivePolicy::AgentHistogramPolicyT; - return HistogramPolicy{ - ap::BLOCK_THREADS, - ap::PIXELS_PER_THREAD, - ap::VEC_SIZE, - ap::LOAD_ALGORITHM, - ap::LOAD_MODIFIER, - ap::IS_RLE_COMPRESS, - ap::IS_WORK_STEALING, - convert_pdl_trigger(0), - convert_dynamic_smem_bytes(0), - convert_static_smem_threads_per_block(0), - convert_static_smem_items_per_thread(0), - convert_static_smem_min_blocks_per_sm(0), - convert_dynamic_smem_range_max_bins(0), - convert_dynamic_smem_even_2ch_max_bins(0), - convert_dynamic_smem_even_3ch_max_bins(0), - convert_dynamic_smem_even_4ch_max_bins(0)}; + using sweep = typename ActivePolicy::AgentHistogramPolicyT; + const auto sweep_policy = HistogramSweepPolicy{ + sweep::BLOCK_THREADS, + sweep::PIXELS_PER_THREAD, + sweep::VEC_SIZE, + sweep::LOAD_ALGORITHM, + sweep::LOAD_MODIFIER, + sweep::IS_RLE_COMPRESS, + sweep::IS_WORK_STEALING}; + return {sweep_policy, {sweep_policy, 256, 0}, {sweep_policy, 0, 0, 0, 0, 0, 0}, 0}; } // TODO(bgruber): drop in CCCL 4.0 @@ -940,14 +825,14 @@ template struct policy_selector_from_max_policy { private: - struct extract_policy_dispatch_t + struct dispatch_t { HistogramPolicy& policy; - template + template _CCCL_HOST_DEVICE_API constexpr cudaError_t Invoke() { - policy = convert_policy(); + policy = convert_policy(0); return cudaSuccess; } }; @@ -958,11 +843,11 @@ public: NV_IF_ELSE_TARGET(NV_IS_HOST, ({ HistogramPolicy policy{}; - extract_policy_dispatch_t dispatch{policy}; + dispatch_t dispatch{policy}; _CCCL_VERIFY(MaxPolicy::Invoke(cc.get() * 10, dispatch) == cudaSuccess, ""); return policy; }), - ({ return convert_policy(); })); + ({ return convert_policy(0); })); } }; @@ -1030,12 +915,10 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( } int max_num_output_bins = max_levels - 1; - constexpr int PRIVATIZED_SMEM_BINS = 256; - if (const auto error = CubDebug( (detail::histogram::dispatch max_privatized_smem_bins) + if (!should_use_static_smem(active_policy, max_num_output_bins)) { // Too many bins to keep in shared memory. - constexpr int PRIVATIZED_SMEM_BINS = 0; - if (const auto error = CubDebug( (detail::histogram::dispatch max_privatized_smem_bins) + if (!should_use_static_smem(active_policy, max_num_output_bins)) { - constexpr int PRIVATIZED_SMEM_BINS = 0; - if (const auto error = CubDebug( (detail::histogram::dispatch #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(threads_per_block(current_policy())), - int(min_blocks_per_sm(current_policy()))) +__launch_bounds__(int(sweep_policy( + current_policy()) + .threads_per_block), + int(UseStaticSmem ? current_policy().static_smem.min_blocks_per_sm : 0)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepKernel( const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, @@ -721,21 +723,22 @@ __launch_bounds__(int(threads_per_block(current_policy tile_queue) { static constexpr HistogramPolicy hp = current_policy(); + static constexpr auto sweep = + sweep_policy(hp); + static constexpr int privatized_smem_bins = UseStaticSmem ? hp.static_smem.max_bins : 0; // Thread block type for compositing input tiles - static constexpr int threads_per_block = histogram::threads_per_block(hp); - static constexpr int items_per_thread = histogram::items_per_thread(hp); using AgentHistogramPolicyT = - agent_histogram_policy; + agent_histogram_policy; using AgentHistogramT = AgentHistogram(current_policy; - static_assert(AgentHistogramT::privatized_smem_bins == PrivatizedSmemBins); + static_assert(AgentHistogramT::privatized_smem_bins == privatized_smem_bins); // Shared memory for AgentHistogram __shared__ typename AgentHistogramT::TempStorage temp_storage; @@ -789,7 +792,7 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(current_policy().sweep_threads_per_block)) +__launch_bounds__(int(current_policy().dynamic_smem.sweep.threads_per_block)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDynamicSmemKernel( const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, @@ -804,16 +807,17 @@ __launch_bounds__(int(current_policy().sweep_threads_per_block)) const int tiles_per_row, GridQueue tile_queue) { - static constexpr HistogramPolicy hp = current_policy(); + static constexpr HistogramPolicy hp = current_policy(); + static constexpr HistogramSweepPolicy sweep = hp.dynamic_smem.sweep; using AgentHistogramPolicyT = - agent_histogram_policy; + agent_histogram_policy; using AgentHistogramT = AgentHistogram().sweep_threads_per_block)) //! @tparam PolicySelector //! Selects the tuning policy //! -//! @tparam PrivatizedSmemBins -//! Maximum number of histogram bins per channel (e.g., up to 256) +//! @tparam UseStaticSmem +//! Whether the privatized histogram is stored in compile-time-sized shared memory //! //! @tparam NumChannels //! Number of channels interleaved in the input data (may be greater than the number of channels @@ -941,7 +945,7 @@ __launch_bounds__(int(current_policy().sweep_threads_per_block)) //! @param tile_queue //! Drain queue descriptor for dynamically mapping tile data onto thread blocks template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(current_policy().sweep_threads_per_block)) +__launch_bounds__(int(sweep_policy( + current_policy()) + .threads_per_block)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDeviceInitKernel( const SampleIteratorT d_samples, ::cuda::std::array num_output_bins_wrapper, @@ -973,6 +979,9 @@ __launch_bounds__(int(current_policy().sweep_threads_per_block)) const GridQueue tile_queue) { static constexpr HistogramPolicy hp = current_policy(); + static constexpr auto sweep = + sweep_policy(hp); + static constexpr int privatized_smem_bins = UseStaticSmem ? hp.static_smem.max_bins : 0; OutputDecodeOpT output_decode_op[NumActiveChannels]; PrivatizedDecodeOpT privatized_decode_op[NumActiveChannels]; @@ -1002,16 +1011,16 @@ __launch_bounds__(int(current_policy().sweep_threads_per_block)) // Thread block type for compositing input tiles using AgentHistogramPolicyT = - agent_histogram_policy; + agent_histogram_policy; using AgentHistogramT = AgentHistogram ::std::ostream& { + return os + << "{ .threads_per_block = " << sweep.threads_per_block << ", .items_per_thread = " << sweep.items_per_thread + << ", .vec_size = " << sweep.vec_size << ", .load_algorithm = " << sweep.load_algorithm + << ", .load_modifier = " << sweep.load_modifier << ", .rle_compress = " << sweep.rle_compress + << ", .work_stealing = " << sweep.work_stealing << " }"; + }; + os << "HistogramPolicy { .gmem = "; + print_sweep(p.gmem); + os << ", .static_smem = { .sweep = "; + print_sweep(p.static_smem.sweep); + os << ", .max_bins = " << p.static_smem.max_bins << ", .min_blocks_per_sm = " << p.static_smem.min_blocks_per_sm + << " }, .dynamic_smem = { .sweep = "; + print_sweep(p.dynamic_smem.sweep); return os - << "HistogramPolicy { .sweep_threads_per_block = " << p.sweep_threads_per_block - << ", .sweep_items_per_thread = " << p.sweep_items_per_thread << ", .vec_size = " << p.vec_size - << ", .load_algorithm = " << p.load_algorithm << ", .load_modifier = " << p.load_modifier - << ", .rle_compress = " << p.rle_compress << ", .work_stealing = " << p.work_stealing - << ", .init_kernel_pdl_trigger_max_bins = " << p.init_kernel_pdl_trigger_max_bins << ", .dynamic_smem_bytes = " - << p.dynamic_smem_bytes << ", .static_smem_threads_per_block = " << p.static_smem_threads_per_block - << ", .static_smem_items_per_thread = " << p.static_smem_items_per_thread - << ", .static_smem_min_blocks_per_sm = " << p.static_smem_min_blocks_per_sm - << ", .dynamic_smem_range_max_bins = " << p.dynamic_smem_range_max_bins - << ", .dynamic_smem_even_2ch_max_bins = " << p.dynamic_smem_even_2ch_max_bins - << ", .dynamic_smem_even_3ch_max_bins = " << p.dynamic_smem_even_3ch_max_bins - << ", .dynamic_smem_even_4ch_max_bins = " << p.dynamic_smem_even_4ch_max_bins << " }"; + << ", .max_bins = " << p.dynamic_smem.max_bins << ", .max_bytes = " << p.dynamic_smem.max_bytes + << ", .range_max_bins = " << p.dynamic_smem.range_max_bins << ", .even_2ch_max_bins = " + << p.dynamic_smem.even_2ch_max_bins << ", .even_3ch_max_bins = " << p.dynamic_smem.even_3ch_max_bins + << ", .even_4ch_max_bins = " << p.dynamic_smem.even_4ch_max_bins + << " }, .init_kernel_pdl_trigger_max_bins = " << p.init_kernel_pdl_trigger_max_bins << " }"; } -#endif // _CCCL_HOSTED() +#endif }; namespace detail::histogram { -template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int threads_per_block(const HistogramPolicy& policy) +template +struct static_smem_policy { - if constexpr (PrivatizedSmemBins > 0) - { - return policy.static_smem_threads_per_block != 0 - ? policy.static_smem_threads_per_block - : policy.sweep_threads_per_block; - } - else - { - return policy.sweep_threads_per_block; - } -} + using SweepPolicyT = SweepPolicy; + static constexpr int MAX_BINS = MaxBins; + static constexpr int MIN_BLOCKS_PER_SM = MinBlocksPerSm; +}; -template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int items_per_thread(const HistogramPolicy& policy) +template +struct dynamic_smem_policy { - if constexpr (PrivatizedSmemBins > 0) - { - return policy.static_smem_items_per_thread != 0 - ? policy.static_smem_items_per_thread - : policy.sweep_items_per_thread; - } - else - { - return policy.sweep_items_per_thread; - } -} + using SweepPolicyT = SweepPolicy; + static constexpr int MAX_BINS = MaxBins; + static constexpr int MAX_BYTES = MaxBytes; + static constexpr int RANGE_MAX_BINS = RangeMaxBins; + static constexpr int EVEN_2CH_MAX_BINS = Even2chMaxBins; + static constexpr int EVEN_3CH_MAX_BINS = Even3chMaxBins; + static constexpr int EVEN_4CH_MAX_BINS = Even4chMaxBins; +}; + +enum class privatization_tier +{ + gmem, + static_smem, + dynamic_smem +}; -template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int min_blocks_per_sm(const HistogramPolicy& policy) +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const HistogramSweepPolicy& sweep_policy(const HistogramPolicy& policy) { - if constexpr (PrivatizedSmemBins > 0) + if constexpr (Tier == privatization_tier::static_smem) { - return policy.static_smem_min_blocks_per_sm; + return policy.static_smem.sweep; + } + else if constexpr (Tier == privatization_tier::dynamic_smem) + { + return policy.dynamic_smem.sweep; } else { - return 0; + return policy.gmem; } } -// Maximum number of bins per channel for the compile-time-sized shared-memory tier. -static constexpr int max_privatized_smem_bins = 512; - -// Leave 4096 bytes of the SM100 opt-in shared-memory limit available for static storage. -static constexpr int sm100_dynamic_smem_bytes = 232448 - 4096; -static constexpr int sm100_dynamic_smem_range_max_bins = 2048; -static constexpr int sm100_dynamic_smem_even_2ch_max_bins = 28544; -static constexpr int sm100_dynamic_smem_even_3ch_max_bins = 19029; -static constexpr int sm100_dynamic_smem_even_4ch_max_bins = 8192; +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool should_use_static_smem(const HistogramPolicy& policy, int num_bins) +{ + return num_bins > 0 && num_bins <= policy.static_smem.max_bins; +} template [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool @@ -154,32 +194,33 @@ should_use_dynamic_smem(const HistogramPolicy& policy, int num_bins, int counter // Multi-channel paths use explicit per-channel caps in addition to the byte // budget because their channel-interleaved launch shapes have distinct // measured crossover points. - if (policy.dynamic_smem_bytes <= 0 || num_bins <= 0 || counter_size <= 0 || num_active_channels <= 0) + if (policy.dynamic_smem.max_bytes <= 0 || num_bins <= 0 || counter_size <= 0 || num_active_channels <= 0) { return false; } - const bool prefer_dynamic_smem = counter_size > int{sizeof(unsigned int)} || num_bins > max_privatized_smem_bins; - const size_t required_bytes = size_t(num_bins) * size_t(num_active_channels) * size_t(counter_size); + const bool prefer_dynamic_smem = + counter_size > int{sizeof(unsigned int)} || !should_use_static_smem(policy, num_bins); + const size_t required_bytes = size_t(num_bins) * size_t(num_active_channels) * size_t(counter_size); - int max_bins = num_bins; + int max_bins = policy.dynamic_smem.max_bins; if (num_active_channels > 1) { if constexpr (IsEven) { - max_bins = num_active_channels == 2 ? policy.dynamic_smem_even_2ch_max_bins + max_bins = num_active_channels == 2 ? policy.dynamic_smem.even_2ch_max_bins : num_active_channels == 3 - ? policy.dynamic_smem_even_3ch_max_bins - : policy.dynamic_smem_even_4ch_max_bins; + ? policy.dynamic_smem.even_3ch_max_bins + : policy.dynamic_smem.even_4ch_max_bins; } else { - max_bins = policy.dynamic_smem_range_max_bins; + max_bins = policy.dynamic_smem.range_max_bins; } } return prefer_dynamic_smem && max_bins > 0 && num_bins <= max_bins - && required_bytes <= static_cast(policy.dynamic_smem_bytes); + && required_bytes <= static_cast(policy.dynamic_smem.max_bytes); } // TODO(bgruber): drop in CCCL 4.0 @@ -350,6 +391,11 @@ struct policy_hub { // TODO This might be worth it to separate usual histogram and the multi one using AgentHistogramPolicyT = agent_histogram_policy<384, t_scale(16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, false>; + using GmemPolicy = AgentHistogramPolicyT; + using StaticSmemPolicy = static_smem_policy; + using DynamicSmemPolicy = dynamic_smem_policy; + + static constexpr int init_kernel_pdl_trigger_max_bins = 0; }; // SM90 @@ -372,6 +418,10 @@ struct policy_hub decltype(select_agent_policy< sm90_tuning()>>(0)); + using GmemPolicy = AgentHistogramPolicyT; + using StaticSmemPolicy = static_smem_policy; + using DynamicSmemPolicy = dynamic_smem_policy; + static constexpr int init_kernel_pdl_trigger_max_bins = 2048; }; @@ -416,29 +466,46 @@ struct policy_hub && (sizeof(SampleT) == 1 || sizeof(SampleT) == 2 || sizeof(SampleT) == 4 || sizeof(SampleT) == 8) ? 2048 : 0; - static constexpr int dynamic_smem_bytes = has_dynamic_smem_tuning ? sm100_dynamic_smem_bytes : 0; - static constexpr int dynamic_smem_range_max_bins = has_dynamic_smem_tuning ? sm100_dynamic_smem_range_max_bins : 0; - static constexpr int dynamic_smem_even_2ch_max_bins = - has_dynamic_smem_tuning ? sm100_dynamic_smem_even_2ch_max_bins : 0; - static constexpr int dynamic_smem_even_3ch_max_bins = - has_dynamic_smem_tuning ? sm100_dynamic_smem_even_3ch_max_bins : 0; - static constexpr int dynamic_smem_even_4ch_max_bins = - has_dynamic_smem_tuning ? sm100_dynamic_smem_even_4ch_max_bins : 0; + static constexpr bool use_range_multi_static_smem_policy = + !IsEven && NumChannels >= 2 && sizeof(CounterT) == 4 && is_primitive::value; + static constexpr bool use_range_u32_static_smem_policy = + !IsEven && NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value + && sizeof(SampleT) == 4; + static constexpr bool use_range_u64_static_smem_policy = + !IsEven && NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value + && sizeof(SampleT) == 8; + static constexpr int static_smem_threads_per_block = - !IsEven && sizeof(CounterT) == 4 && is_primitive::value - ? (NumChannels >= 2 - ? 384 - : (NumChannels == 1 && NumActiveChannels == 1 && sizeof(SampleT) == 4 - ? 768 - : (NumChannels == 1 && NumActiveChannels == 1 && sizeof(SampleT) == 8 ? 384 : 0))) - : 0; + use_range_multi_static_smem_policy || use_range_u64_static_smem_policy ? 384 + : use_range_u32_static_smem_policy + ? 768 + : AgentHistogramPolicyT::BLOCK_THREADS; static constexpr int static_smem_items_per_thread = - !IsEven && NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value - && sizeof(SampleT) == 8 - ? t_scale(16) - : 0; + use_range_u64_static_smem_policy ? t_scale(16) : AgentHistogramPolicyT::PIXELS_PER_THREAD; static constexpr int static_smem_min_blocks_per_sm = - !IsEven && static_smem_threads_per_block > 0 && static_smem_threads_per_block < 512 ? 3 : 0; + use_range_multi_static_smem_policy || use_range_u64_static_smem_policy ? 3 : 0; + + using StaticSmemSweepPolicy = + agent_histogram_policy; + + using GmemPolicy = AgentHistogramPolicyT; + using StaticSmemPolicy = static_smem_policy; + + static constexpr int dynamic_smem_max_bytes = has_dynamic_smem_tuning ? 232448 - 4096 : 0; + using DynamicSmemPolicy = + dynamic_smem_policy; }; using MaxPolicy = Policy1000; @@ -449,144 +516,51 @@ template concept histogram_policy_selector = policy_selector; #endif // _CCCL_HAS_CONCEPTS() -struct policy_selector +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto convert_sweep_policy() -> HistogramSweepPolicy { - bool sample_is_primitive; - int sample_size; - int counter_size; - int sample_size_bytes; - int num_channels; - int num_active_channels; - bool is_even; - -private: - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int t_scale(int nominal_items_per_thread) const - { - const int sample_scale = (sample_size_bytes + int{sizeof(int)} - 1) / int{sizeof(int)}; - return (::cuda::std::max) (nominal_items_per_thread / num_active_channels / sample_scale, 1); - } - - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto sm100_policy(HistogramPolicy policy) const -> HistogramPolicy - { - policy.dynamic_smem_bytes = sm100_dynamic_smem_bytes; - policy.dynamic_smem_range_max_bins = sm100_dynamic_smem_range_max_bins; - policy.dynamic_smem_even_2ch_max_bins = sm100_dynamic_smem_even_2ch_max_bins; - policy.dynamic_smem_even_3ch_max_bins = sm100_dynamic_smem_even_3ch_max_bins; - policy.dynamic_smem_even_4ch_max_bins = sm100_dynamic_smem_even_4ch_max_bins; - return policy; - } + return {StaticSweepPolicy::BLOCK_THREADS, + StaticSweepPolicy::PIXELS_PER_THREAD, + StaticSweepPolicy::VEC_SIZE, + StaticSweepPolicy::LOAD_ALGORITHM, + StaticSweepPolicy::LOAD_MODIFIER, + StaticSweepPolicy::IS_RLE_COMPRESS, + StaticSweepPolicy::IS_WORK_STEALING}; +} - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool has_sm100_dynamic_smem_tuning() const - { - return sample_is_primitive && counter_size == 4 - && ((num_channels == 1 && num_active_channels == 1 - && (sample_size == 1 || sample_size == 4 || sample_size == 8)) - || num_channels >= 2); - } +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto convert_chained_policy() -> HistogramPolicy +{ + using static_smem = typename ActivePolicy::StaticSmemPolicy; + using dynamic_smem = typename ActivePolicy::DynamicSmemPolicy; + return { + convert_sweep_policy(), + {convert_sweep_policy(), static_smem::MAX_BINS, static_smem::MIN_BLOCKS_PER_SM}, + {convert_sweep_policy(), + dynamic_smem::MAX_BINS, + dynamic_smem::MAX_BYTES, + dynamic_smem::RANGE_MAX_BINS, + dynamic_smem::EVEN_2CH_MAX_BINS, + dynamic_smem::EVEN_3CH_MAX_BINS, + dynamic_smem::EVEN_4CH_MAX_BINS}, + ActivePolicy::init_kernel_pdl_trigger_max_bins}; +} -public: +template +struct policy_selector_from_types +{ [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy { + using hub = policy_hub; if (cc >= ::cuda::compute_capability{10, 0}) { - if (num_channels == 1 && num_active_channels == 1 && counter_size == 4 && sample_is_primitive && sample_size == 1) - { - if (is_even) - { - // ipt_12.tpb_928.rle_0.ws_0.mem_1.ld_2.laid_0.vec_2 1.033332 0.940517 1.031835 1.195876 - return sm100_policy(HistogramPolicy{928, 12, 1 << 2, BLOCK_LOAD_DIRECT, LOAD_CA, false, false, 2048}); - } - else - { - // ipt_12.tpb_448.rle_0.ws_0.mem_1.ld_1.laid_0.vec_2 1.078987 0.985542 1.085118 1.175637 - return sm100_policy(HistogramPolicy{448, 12, 1 << 2, BLOCK_LOAD_DIRECT, LOAD_LDG, false, false, 2048}); - } - } - - if (num_channels == 1 && num_active_channels == 1 && counter_size == 4 && sample_is_primitive - && (sample_size == 4 || sample_size == 8)) - { - if (is_even) - { - return sm100_policy( - HistogramPolicy{768, t_scale(12), 1 << 2, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false, 2048}); - } - - const int static_threads = sample_size == 8 ? 384 : 768; - const int static_items = sample_size == 8 ? t_scale(16) : 0; - const int static_min_blocks = static_threads < 512 ? 3 : 0; - return sm100_policy(HistogramPolicy{ - 768, - t_scale(12), - 1 << 2, - BLOCK_LOAD_DIRECT, - LOAD_LDG, - true, - false, - 2048, - 0, - static_threads, - static_items, - static_min_blocks}); - } - - if (num_channels >= 2 && counter_size == 4 && sample_is_primitive) - { - if (is_even) - { - return sm100_policy(HistogramPolicy{1024, t_scale(8), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false, 0}); - } - return sm100_policy( - HistogramPolicy{1024, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false, 0, 0, 384, 0, 3}); - } - - if (num_channels == 1 && num_active_channels == 1 && counter_size == 4 && sample_is_primitive && sample_size == 2) - { - return HistogramPolicy{960, 10, 1 << 2, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, false, 2048}; - } - - auto fallback = HistogramPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false, 0}; - return has_sm100_dynamic_smem_tuning() ? sm100_policy(fallback) : fallback; + return convert_chained_policy(); } - if (cc >= ::cuda::compute_capability{9, 0}) { - if (num_channels == 1 && num_active_channels == 1 && counter_size == 4 && sample_is_primitive) - { - if (sample_size == 1) - { - return HistogramPolicy{768, 12, 1 << 2, BLOCK_LOAD_DIRECT, LOAD_LDG, false, false, 2048}; - } - else if (sample_size == 2) - { - return HistogramPolicy{960, 10, 1 << 2, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, false, 2048}; - } - } + return convert_chained_policy(); } - - // fallback from SM50 - return HistogramPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false, 0}; - } -}; - -#if _CCCL_HAS_CONCEPTS() -static_assert(histogram_policy_selector); -#endif // _CCCL_HAS_CONCEPTS() - -template -struct policy_selector_from_types -{ - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy - { - constexpr auto policies = policy_selector{ - is_primitive_v, - int{sizeof(SampleT)}, - int{sizeof(CounterT)}, - int{sizeof(SampleT)}, - NumChannels, - NumActiveChannels, - IsEven}; - return policies(cc); + return convert_chained_policy(); } }; } // namespace detail::histogram diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index 3600ba679841..e53c4520602d 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1626,7 +1626,9 @@ struct histogram_tuning { _CCCL_HOST_DEVICE_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { - return {BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false, 0, 0}; + constexpr auto sweep = + cub::HistogramSweepPolicy{BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; + return {sweep, {sweep, 256, 0}, {sweep, 0, 0, 0, 0, 0, 0}, 0}; } }; @@ -1642,10 +1644,9 @@ struct mixed_counter_histogram_tuning _CCCL_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { - cub::HistogramPolicy policy{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false, 0}; - policy.dynamic_smem_bytes = 228352; - policy.dynamic_smem_even_4ch_max_bins = 8192; - return policy; + constexpr auto sweep = + cub::HistogramSweepPolicy{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; + return {sweep, {sweep, 512, 0}, {sweep, 57088, 228352, 2048, 28544, 19029, 8192}, 0}; } }; @@ -1805,42 +1806,51 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) // aggregate init constexpr auto p1 = cub::HistogramPolicy{ - 128, - 7, - 4, - cub::BLOCK_LOAD_DIRECT, - cub::CacheLoadModifier::LOAD_LDG, - false, - false, - 2048, - 12345, - 96, - 3, - 2, - 1024, - 4096, - 8192, - 16384}; + {128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, + {{96, 3, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, 512, 2}, + {{128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, + 12345, + 12345, + 1024, + 4096, + 8192, + 16384}, + 2048}; # if _CCCL_STD_VER >= 2020 // designated init constexpr auto p2 = cub::HistogramPolicy{ - .sweep_threads_per_block = 128, - .sweep_items_per_thread = 7, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false, - .init_kernel_pdl_trigger_max_bins = 2048, - .dynamic_smem_bytes = 12345, - .static_smem_threads_per_block = 96, - .static_smem_items_per_thread = 3, - .static_smem_min_blocks_per_sm = 2, - .dynamic_smem_range_max_bins = 1024, - .dynamic_smem_even_2ch_max_bins = 4096, - .dynamic_smem_even_3ch_max_bins = 8192, - .dynamic_smem_even_4ch_max_bins = 16384}; + .gmem = {.threads_per_block = 128, + .items_per_thread = 7, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .static_smem = {.sweep = {.threads_per_block = 96, + .items_per_thread = 3, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .max_bins = 512, + .min_blocks_per_sm = 2}, + .dynamic_smem = + {.sweep = {.threads_per_block = 128, + .items_per_thread = 7, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .max_bins = 12345, + .max_bytes = 12345, + .range_max_bins = 1024, + .even_2ch_max_bins = 4096, + .even_3ch_max_bins = 8192, + .even_4ch_max_bins = 16384}, + .init_kernel_pdl_trigger_max_bins = 2048}; # else // _CCCL_STD_VER >= 2020 constexpr auto p2 = p1; # endif // _CCCL_STD_VER >= 2020 @@ -1856,13 +1866,15 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) }; REQUIRE( to_string(p1) - == "HistogramPolicy { .sweep_threads_per_block = 128, .sweep_items_per_thread = 7, .vec_size = 4" - ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" - ", .work_stealing = 0, .init_kernel_pdl_trigger_max_bins = 2048" - ", .dynamic_smem_bytes = 12345, .static_smem_threads_per_block = 96" - ", .static_smem_items_per_thread = 3, .static_smem_min_blocks_per_sm = 2" - ", .dynamic_smem_range_max_bins = 1024, .dynamic_smem_even_2ch_max_bins = 4096" - ", .dynamic_smem_even_3ch_max_bins = 8192, .dynamic_smem_even_4ch_max_bins = 16384 }"); + == "HistogramPolicy { .gmem = { .threads_per_block = 128, .items_per_thread = 7, .vec_size = 4" + ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0, .work_stealing = 0 }" + ", .static_smem = { .sweep = { .threads_per_block = 96, .items_per_thread = 3, .vec_size = 4" + ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0, .work_stealing = 0 }" + ", .max_bins = 512, .min_blocks_per_sm = 2 }, .dynamic_smem = { .sweep = { .threads_per_block = 128" + ", .items_per_thread = 7, .vec_size = 4, .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG" + ", .rle_compress = 0, .work_stealing = 0 }, .max_bins = 12345, .max_bytes = 12345" + ", .range_max_bins = 1024, .even_2ch_max_bins = 4096, .even_3ch_max_bins = 8192" + ", .even_4ch_max_bins = 16384 }, .init_kernel_pdl_trigger_max_bins = 2048 }"); } C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget", "[histogram][device]") @@ -1875,13 +1887,15 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); - STATIC_REQUIRE(sm90_policy.dynamic_smem_bytes == 0); - STATIC_REQUIRE(sm100_policy.dynamic_smem_bytes == 228352); - STATIC_REQUIRE(sm100_wide_counter_policy.dynamic_smem_bytes == 0); - STATIC_REQUIRE(sm100_policy.dynamic_smem_range_max_bins == 2048); - STATIC_REQUIRE(sm100_policy.dynamic_smem_even_2ch_max_bins == 28544); - STATIC_REQUIRE(sm100_policy.dynamic_smem_even_3ch_max_bins == 19029); - STATIC_REQUIRE(sm100_policy.dynamic_smem_even_4ch_max_bins == 8192); + STATIC_REQUIRE(sm90_policy.static_smem.max_bins == 256); + STATIC_REQUIRE(sm100_policy.static_smem.max_bins == 512); + STATIC_REQUIRE(sm90_policy.dynamic_smem.max_bytes == 0); + STATIC_REQUIRE(sm100_policy.dynamic_smem.max_bytes == 228352); + STATIC_REQUIRE(sm100_wide_counter_policy.dynamic_smem.max_bytes == 0); + STATIC_REQUIRE(sm100_policy.dynamic_smem.range_max_bins == 2048); + STATIC_REQUIRE(sm100_policy.dynamic_smem.even_2ch_max_bins == 28544); + STATIC_REQUIRE(sm100_policy.dynamic_smem.even_3ch_max_bins == 19029); + STATIC_REQUIRE(sm100_policy.dynamic_smem.even_4ch_max_bins == 8192); STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57088, 4, 1)); STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57089, 4, 1)); diff --git a/cub/test/catch2_test_device_histogram_env_api.cu b/cub/test/catch2_test_device_histogram_env_api.cu index 2d71419d3636..659412bbfa33 100644 --- a/cub/test/catch2_test_device_histogram_env_api.cu +++ b/cub/test/catch2_test_device_histogram_env_api.cu @@ -418,13 +418,11 @@ struct HistogramPolicySelector { __host__ __device__ constexpr auto operator()(cuda::compute_capability cc) const -> cub::HistogramPolicy { - return {.sweep_threads_per_block = 128, - .sweep_items_per_thread = cc > cuda::compute_capability{9, 0} ? 16 : 7, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::LOAD_LDG, - .rle_compress = false, - .work_stealing = false, + const auto sweep = cub::HistogramSweepPolicy{ + 128, cc > cuda::compute_capability{9, 0} ? 16 : 7, 4, cub::BLOCK_LOAD_DIRECT, cub::LOAD_LDG, false, false}; + return {.gmem = sweep, + .static_smem = {.sweep = sweep, .max_bins = 256, .min_blocks_per_sm = 0}, + .dynamic_smem = {.sweep = sweep}, .init_kernel_pdl_trigger_max_bins = 2048}; } }; From 7f558efab9ad37aed7b7ea84f1aef6cda75834eb Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sat, 1 Aug 2026 16:17:13 +0000 Subject: [PATCH 20/45] [cub] Preserve SM90 histogram PDL selection --- cub/cub/device/dispatch/tuning/tuning_histogram.cuh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 6fa33250a8e2..e40943307d2b 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -422,7 +422,11 @@ struct policy_hub using StaticSmemPolicy = static_smem_policy; using DynamicSmemPolicy = dynamic_smem_policy; - static constexpr int init_kernel_pdl_trigger_max_bins = 2048; + static constexpr int init_kernel_pdl_trigger_max_bins = + NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value + && (sizeof(SampleT) == 1 || sizeof(SampleT) == 2) + ? 2048 + : 0; }; struct Policy1000 : detail::chained_policy<1000, Policy1000, Policy900> From 889b37b11fff8a6f96804cceb355470168ba4a71 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sat, 1 Aug 2026 20:43:28 +0000 Subject: [PATCH 21/45] [cub] Honor histogram memory preference in dynamic SMEM --- cub/cub/agent/agent_histogram.cuh | 60 +++++++++--- .../device/dispatch/dispatch_histogram.cuh | 4 +- .../dispatch/kernels/kernel_histogram.cuh | 51 +++++----- .../dispatch/tuning/tuning_histogram.cuh | 52 +++++----- cub/test/catch2_test_device_histogram_env.cu | 95 ++++++++++++++----- .../catch2_test_device_histogram_env_api.cu | 9 +- cub/test/catch2_test_enum_formatting.cu | 1 + 7 files changed, 185 insertions(+), 87 deletions(-) diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index db394f73ee6b..f52e96a38a12 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -36,7 +36,8 @@ CUB_NAMESPACE_BEGIN enum BlockHistogramMemoryPreference { GMEM, - SMEM + SMEM, + BLEND }; #if _CCCL_HOSTED() @@ -50,6 +51,8 @@ namespace detail return "GMEM"; case SMEM: return "SMEM"; + case BLEND: + return "BLEND"; } return ""; } @@ -85,6 +88,7 @@ template struct agent_histogram_policy @@ -97,6 +101,9 @@ struct agent_histogram_policy /// Whether to perform localized RLE to compress samples before histogramming static constexpr bool IS_RLE_COMPRESS = RleCompress; + /// Whether to prefer privatized shared-memory bins (versus privatized global-memory bins) + static constexpr BlockHistogramMemoryPreference MEM_PREFERENCE = MemoryPreference; + /// Whether to dequeue tiles from a global work queue static constexpr bool IS_WORK_STEALING = WorkStealing; @@ -121,8 +128,16 @@ template -using AgentHistogramPolicy CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") = detail:: - agent_histogram_policy; +using AgentHistogramPolicy + CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") = detail::agent_histogram_policy< + ThreadsPerBlock, + PixelsPerThread, + LoadAlgorithm, + LoadModifier, + RleCompress, + MemoryPreference, + WorkStealing, + VecSize>; namespace detail::histogram { @@ -204,6 +219,10 @@ struct AgentHistogram static constexpr bool is_rle_compress = AgentHistogramPolicyT::IS_RLE_COMPRESS; static constexpr bool is_work_stealing = AgentHistogramPolicyT::IS_WORK_STEALING; static constexpr CacheLoadModifier load_modifier = AgentHistogramPolicyT::LOAD_MODIFIER; + static constexpr auto mem_preference = + (UseDynamicSmem || PrivatizedSmemBins > 0) + ? BlockHistogramMemoryPreference{AgentHistogramPolicyT::MEM_PREFERENCE} + : GMEM; using SampleT = it_value_t; using PixelT = typename CubVector::Type; @@ -250,20 +269,23 @@ struct AgentHistogram const OutputDecodeOpT* output_decode_op; // determines output bin-id from privatized counter index, one for each // channel PrivatizedDecodeOpT* privatized_decode_op; // determines privatized counter index from sample, one for each channel + bool prefer_smem; // whether this block uses shared-memory privatization + _CCCL_DEVICE _CCCL_FORCEINLINE CounterT* PrivatizedHistogram(int channel) { - if constexpr (UseDynamicSmem) - { - return dyn_smem_histograms[channel]; - } - else if constexpr (PrivatizedSmemBins > 0) - { - return temp_storage.histograms[channel]; - } - else + if (prefer_smem) { - return d_privatized_histograms[channel]; + if constexpr (UseDynamicSmem) + { + return dyn_smem_histograms[channel]; + } + else + { + return temp_storage.histograms[channel]; + } } + + return d_privatized_histograms[channel]; } _CCCL_DEVICE _CCCL_FORCEINLINE void ZeroBinCounters() @@ -617,6 +639,7 @@ struct AgentHistogram , d_output_histograms(d_output_histograms) , output_decode_op(output_decode_op) , privatized_decode_op(privatized_decode_op) + , prefer_smem(mem_preference == SMEM || (mem_preference == BLEND && (blockIdx.x & 1))) { static_assert(!UseDynamicSmem, "AgentHistogram with UseDynamicSmem=true requires the dynamic-SMEM " @@ -652,6 +675,7 @@ struct AgentHistogram , d_output_histograms(d_output_histograms) , output_decode_op(output_decode_op) , privatized_decode_op(privatized_decode_op) + , prefer_smem(mem_preference == SMEM || (mem_preference == BLEND && (blockIdx.x & 1))) { static_assert(UseDynamicSmem, "Dynamic-SMEM AgentHistogram constructor requires UseDynamicSmem=true."); @@ -662,6 +686,16 @@ struct AgentHistogram this->dyn_smem_histograms[ch] = p; p += num_privatized_bins[ch]; } + + if (!prefer_smem) + { + const int block_id = static_cast((blockIdx.y * gridDim.x) + blockIdx.x); + for (int ch = 0; ch < NumActiveChannels; ++ch) + { + const auto offset = static_cast<::cuda::std::int64_t>(block_id) * num_privatized_bins[ch]; + this->d_privatized_histograms[ch] = d_privatized_histograms[ch] + offset; + } + } } //! @brief Consume image diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 39468fce6b82..115fa5089e88 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -347,7 +347,8 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( void* allocations[NUM_ALLOCATIONS] = {}; size_t allocation_sizes[NUM_ALLOCATIONS]; - constexpr bool requires_global_privatization = !UseDynamicSmem && !UseStaticSmem; + const bool requires_global_privatization = + (!UseDynamicSmem && !UseStaticSmem) || sweep.mem_preference != BlockHistogramMemoryPreference::SMEM; for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL) { allocation_sizes[CHANNEL] = @@ -816,6 +817,7 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_policy(long) -> HistogramPolicy sweep::LOAD_ALGORITHM, sweep::LOAD_MODIFIER, sweep::IS_RLE_COMPRESS, + sweep::MEM_PREFERENCE, sweep::IS_WORK_STEALING}; return {sweep_policy, {sweep_policy, 256, 0}, {sweep_policy, 0, 0, 0, 0, 0, 0}, 0}; } diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index a76ae5c7f647..3d5b4b436d14 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -728,14 +728,15 @@ __launch_bounds__(int(sweep_policy; + using AgentHistogramPolicyT = agent_histogram_policy< + sweep.threads_per_block, + sweep.items_per_thread, + sweep.load_algorithm, + sweep.load_modifier, + sweep.rle_compress, + sweep.mem_preference, + sweep.work_stealing, + sweep.vec_size>; using AgentHistogramT = AgentHistogram().dynamic_smem.sweep.thread static constexpr HistogramPolicy hp = current_policy(); static constexpr HistogramSweepPolicy sweep = hp.dynamic_smem.sweep; - using AgentHistogramPolicyT = - agent_histogram_policy; + using AgentHistogramPolicyT = agent_histogram_policy< + sweep.threads_per_block, + sweep.items_per_thread, + sweep.load_algorithm, + sweep.load_modifier, + sweep.rle_compress, + sweep.mem_preference, + sweep.work_stealing, + sweep.vec_size>; using AgentHistogramT = AgentHistogram; + using AgentHistogramPolicyT = agent_histogram_policy< + sweep.threads_per_block, + sweep.items_per_thread, + sweep.load_algorithm, + sweep.load_modifier, + sweep.rle_compress, + sweep.mem_preference, + sweep.work_stealing, + sweep.vec_size>; using AgentHistogramT = AgentHistogram(num_samples, 0); - auto env = cuda::execution::tune(mixed_counter_histogram_tuning{}); + auto env = cuda::execution::tune(mixed_counter_histogram_tuning<>{}); histogram_even( cuda::counting_iterator(0), @@ -1686,6 +1687,43 @@ C2H_TEST("DeviceHistogram supports narrower local counters than output counters" REQUIRE(d_histogram == c2h::host_vector(num_samples, 1)); } +C2H_TEST("Dynamic shared-memory histograms respect memory preference", "[histogram][device]") +{ + int current_device{}; + REQUIRE(cudaSuccess == cudaGetDevice(¤t_device)); + + cuda::compute_capability cc{}; + REQUIRE(cudaSuccess == cub::detail::ptx_compute_cap(cc, current_device)); + if (cc < cuda::compute_capability{10, 0}) + { + SKIP("The runtime-sized shared-memory histogram policy is currently tuned for SM100"); + } + + constexpr int num_samples = 16384; + constexpr int num_levels = 4097; + + const auto check_preference = [](auto preference) { + constexpr auto memory_preference = decltype(preference)::value; + auto d_histogram = c2h::device_vector(num_levels - 1, 0); + auto env = cuda::execution::tune(mixed_counter_histogram_tuning{}); + + histogram_even( + cuda::counting_iterator(0), + thrust::raw_pointer_cast(d_histogram.data()), + num_levels, + 0u, + static_cast(num_levels - 1), + num_samples, + env); + + REQUIRE(d_histogram == c2h::host_vector(num_levels - 1, 1)); + }; + + check_preference(cuda::std::integral_constant{}); + check_preference(cuda::std::integral_constant{}); + check_preference(cuda::std::integral_constant{}); +} + using block_sizes = c2h::type_list, cuda::std::integral_constant>; @@ -1806,9 +1844,9 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) // aggregate init constexpr auto p1 = cub::HistogramPolicy{ - {128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, - {{96, 3, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, 512, 2}, - {{128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, + {128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, cub::SMEM, false}, + {{96, 3, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, cub::SMEM, false}, 512, 2}, + {{128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, cub::SMEM, false}, 12345, 12345, 1024, @@ -1820,22 +1858,25 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) # if _CCCL_STD_VER >= 2020 // designated init constexpr auto p2 = cub::HistogramPolicy{ - .gmem = {.threads_per_block = 128, - .items_per_thread = 7, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .static_smem = {.sweep = {.threads_per_block = 96, - .items_per_thread = 3, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .max_bins = 512, - .min_blocks_per_sm = 2}, + .gmem = {.threads_per_block = 128, + .items_per_thread = 7, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .mem_preference = cub::SMEM, + .work_stealing = false}, + .static_smem = + {.sweep = {.threads_per_block = 96, + .items_per_thread = 3, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .mem_preference = cub::SMEM, + .work_stealing = false}, + .max_bins = 512, + .min_blocks_per_sm = 2}, .dynamic_smem = {.sweep = {.threads_per_block = 128, .items_per_thread = 7, @@ -1843,6 +1884,7 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) .load_algorithm = cub::BLOCK_LOAD_DIRECT, .load_modifier = cub::CacheLoadModifier::LOAD_LDG, .rle_compress = false, + .mem_preference = cub::SMEM, .work_stealing = false}, .max_bins = 12345, .max_bytes = 12345, @@ -1867,12 +1909,15 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) REQUIRE( to_string(p1) == "HistogramPolicy { .gmem = { .threads_per_block = 128, .items_per_thread = 7, .vec_size = 4" - ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0, .work_stealing = 0 }" + ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" + ", .mem_preference = SMEM, .work_stealing = 0 }" ", .static_smem = { .sweep = { .threads_per_block = 96, .items_per_thread = 3, .vec_size = 4" - ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0, .work_stealing = 0 }" + ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" + ", .mem_preference = SMEM, .work_stealing = 0 }" ", .max_bins = 512, .min_blocks_per_sm = 2 }, .dynamic_smem = { .sweep = { .threads_per_block = 128" ", .items_per_thread = 7, .vec_size = 4, .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG" - ", .rle_compress = 0, .work_stealing = 0 }, .max_bins = 12345, .max_bytes = 12345" + ", .rle_compress = 0, .mem_preference = SMEM, .work_stealing = 0 }, .max_bins = 12345" + ", .max_bytes = 12345" ", .range_max_bins = 1024, .even_2ch_max_bins = 4096, .even_3ch_max_bins = 8192" ", .even_4ch_max_bins = 16384 }, .init_kernel_pdl_trigger_max_bins = 2048 }"); } diff --git a/cub/test/catch2_test_device_histogram_env_api.cu b/cub/test/catch2_test_device_histogram_env_api.cu index 659412bbfa33..91d1009da665 100644 --- a/cub/test/catch2_test_device_histogram_env_api.cu +++ b/cub/test/catch2_test_device_histogram_env_api.cu @@ -419,7 +419,14 @@ struct HistogramPolicySelector __host__ __device__ constexpr auto operator()(cuda::compute_capability cc) const -> cub::HistogramPolicy { const auto sweep = cub::HistogramSweepPolicy{ - 128, cc > cuda::compute_capability{9, 0} ? 16 : 7, 4, cub::BLOCK_LOAD_DIRECT, cub::LOAD_LDG, false, false}; + 128, + cc > cuda::compute_capability{9, 0} ? 16 : 7, + 4, + cub::BLOCK_LOAD_DIRECT, + cub::LOAD_LDG, + false, + cub::SMEM, + false}; return {.gmem = sweep, .static_smem = {.sweep = sweep, .max_bins = 256, .min_blocks_per_sm = 0}, .dynamic_smem = {.sweep = sweep}, diff --git a/cub/test/catch2_test_enum_formatting.cu b/cub/test/catch2_test_enum_formatting.cu index e2bf8bdfb1c6..87c17e311729 100644 --- a/cub/test/catch2_test_enum_formatting.cu +++ b/cub/test/catch2_test_enum_formatting.cu @@ -49,6 +49,7 @@ void do_test(const Tester& tester) { tester(cub::BlockHistogramMemoryPreference::GMEM, "GMEM"); tester(cub::BlockHistogramMemoryPreference::SMEM, "SMEM"); + tester(cub::BlockHistogramMemoryPreference::BLEND, "BLEND"); tester(cub::BlockHistogramMemoryPreference(100), ""); } From a36b77d48d14424f18c22359d9154a908e5d66a9 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sun, 2 Aug 2026 09:03:12 +0000 Subject: [PATCH 22/45] [cub] Restore histogram memory preference tuning --- cub/benchmarks/bench/histogram/even.cu | 1 + .../bench/histogram/histogram_common.cuh | 26 +++++++++++++------ cub/benchmarks/bench/histogram/multi/even.cu | 1 + cub/benchmarks/bench/histogram/multi/range.cu | 1 + cub/benchmarks/bench/histogram/range.cu | 1 + 5 files changed, 22 insertions(+), 8 deletions(-) diff --git a/cub/benchmarks/bench/histogram/even.cu b/cub/benchmarks/bench/histogram/even.cu index 28465245171a..a175139b1b99 100644 --- a/cub/benchmarks/bench/histogram/even.cu +++ b/cub/benchmarks/bench/histogram/even.cu @@ -9,6 +9,7 @@ // %RANGE% TUNE_THREADS tpb 128:1024:32 // %RANGE% TUNE_RLE_COMPRESS rle 0:1:1 // %RANGE% TUNE_WORK_STEALING ws 0:1:1 +// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1 // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 diff --git a/cub/benchmarks/bench/histogram/histogram_common.cuh b/cub/benchmarks/bench/histogram/histogram_common.cuh index ab08b84beee4..4765630217f0 100644 --- a/cub/benchmarks/bench/histogram/histogram_common.cuh +++ b/cub/benchmarks/bench/histogram/histogram_common.cuh @@ -19,6 +19,14 @@ # define TUNE_VEC_SIZE (1 << TUNE_VEC_SIZE_POW) +# if TUNE_MEM_PREFERENCE == 0 +constexpr cub::BlockHistogramMemoryPreference MEM_PREFERENCE = cub::GMEM; +# elif TUNE_MEM_PREFERENCE == 1 +constexpr cub::BlockHistogramMemoryPreference MEM_PREFERENCE = cub::SMEM; +# else // TUNE_MEM_PREFERENCE == 2 +constexpr cub::BlockHistogramMemoryPreference MEM_PREFERENCE = cub::BLEND; +# endif // TUNE_MEM_PREFERENCE + # if TUNE_LOAD_ALGORITHM_ID == 0 # define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_DIRECT # elif TUNE_LOAD_ALGORITHM_ID == 1 @@ -37,14 +45,16 @@ struct bench_policy_selector ? (NUM_CHANNELS == 1 ? cub::BLOCK_LOAD_STRIPED : cub::BLOCK_LOAD_DIRECT) : TUNE_LOAD_ALGORITHM; - return {TUNE_THREADS, - TUNE_ITEMS, - TUNE_VEC_SIZE, - load_algorithm, - TUNE_LOAD_MODIFIER, - TUNE_RLE_COMPRESS, - TUNE_WORK_STEALING, - 2048}; // TODO(bgruber): make tunable + constexpr auto sweep = cub::HistogramSweepPolicy{ + TUNE_THREADS, + TUNE_ITEMS, + TUNE_VEC_SIZE, + load_algorithm, + TUNE_LOAD_MODIFIER, + TUNE_RLE_COMPRESS, + MEM_PREFERENCE, + TUNE_WORK_STEALING}; + return {sweep, {sweep, 256, 0}, {sweep, 0, 0, 0, 0, 0, 0}, 2048}; // TODO(bgruber): make tunable } }; #endif // !TUNE_BASE diff --git a/cub/benchmarks/bench/histogram/multi/even.cu b/cub/benchmarks/bench/histogram/multi/even.cu index 0d6b4c51e191..4131206a8bd7 100644 --- a/cub/benchmarks/bench/histogram/multi/even.cu +++ b/cub/benchmarks/bench/histogram/multi/even.cu @@ -9,6 +9,7 @@ // %RANGE% TUNE_THREADS tpb 128:1024:32 // %RANGE% TUNE_RLE_COMPRESS rle 0:1:1 // %RANGE% TUNE_WORK_STEALING ws 0:1:1 +// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1 // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 diff --git a/cub/benchmarks/bench/histogram/multi/range.cu b/cub/benchmarks/bench/histogram/multi/range.cu index 2d786ce95cbb..dcb402d7db20 100644 --- a/cub/benchmarks/bench/histogram/multi/range.cu +++ b/cub/benchmarks/bench/histogram/multi/range.cu @@ -11,6 +11,7 @@ // %RANGE% TUNE_THREADS tpb 128:1024:32 // %RANGE% TUNE_RLE_COMPRESS rle 0:1:1 // %RANGE% TUNE_WORK_STEALING ws 0:1:1 +// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1 // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 diff --git a/cub/benchmarks/bench/histogram/range.cu b/cub/benchmarks/bench/histogram/range.cu index 4c989ede589b..e1e808466395 100644 --- a/cub/benchmarks/bench/histogram/range.cu +++ b/cub/benchmarks/bench/histogram/range.cu @@ -11,6 +11,7 @@ // %RANGE% TUNE_THREADS tpb 128:1024:32 // %RANGE% TUNE_RLE_COMPRESS rle 0:1:1 // %RANGE% TUNE_WORK_STEALING ws 0:1:1 +// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1 // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 From dfb169fc00629e121c4f9c60984db204ee5d5bd8 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sun, 2 Aug 2026 09:22:11 +0000 Subject: [PATCH 23/45] [cub] Make histogram tier thresholds tunable --- cub/benchmarks/bench/histogram/even.cu | 9 +++++++++ cub/benchmarks/bench/histogram/histogram_common.cuh | 11 ++++++++++- cub/benchmarks/bench/histogram/multi/even.cu | 9 +++++++++ cub/benchmarks/bench/histogram/multi/range.cu | 9 +++++++++ cub/benchmarks/bench/histogram/range.cu | 9 +++++++++ 5 files changed, 46 insertions(+), 1 deletion(-) diff --git a/cub/benchmarks/bench/histogram/even.cu b/cub/benchmarks/bench/histogram/even.cu index a175139b1b99..b255db201b45 100644 --- a/cub/benchmarks/bench/histogram/even.cu +++ b/cub/benchmarks/bench/histogram/even.cu @@ -13,6 +13,15 @@ // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 +// %RANGE% TUNE_STATIC_SMEM_MAX_BINS smem_bins 0:512:256 +// %RANGE% TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM smem_blocks 0:4:1 +// %RANGE% TUNE_DYNAMIC_SMEM_MAX_BINS dyn_bins 0:57088:57088 +// %RANGE% TUNE_DYNAMIC_SMEM_MAX_BYTES dyn_bytes 0:228352:228352 +// %RANGE% TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS dyn_range_bins 0:2048:2048 +// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS dyn_even_2ch_bins 0:28544:28544 +// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_3CH_MAX_BINS dyn_even_3ch_bins 0:19029:19029 +// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_4CH_MAX_BINS dyn_even_4ch_bins 0:8192:8192 +// %RANGE% TUNE_INIT_KERNEL_PDL_TRIGGER_MAX_BINS pdl_bins 0:2048:2048 template static void even(nvbench::state& state, nvbench::type_list) diff --git a/cub/benchmarks/bench/histogram/histogram_common.cuh b/cub/benchmarks/bench/histogram/histogram_common.cuh index 4765630217f0..52d80bfed312 100644 --- a/cub/benchmarks/bench/histogram/histogram_common.cuh +++ b/cub/benchmarks/bench/histogram/histogram_common.cuh @@ -54,7 +54,16 @@ struct bench_policy_selector TUNE_RLE_COMPRESS, MEM_PREFERENCE, TUNE_WORK_STEALING}; - return {sweep, {sweep, 256, 0}, {sweep, 0, 0, 0, 0, 0, 0}, 2048}; // TODO(bgruber): make tunable + return {sweep, + {sweep, TUNE_STATIC_SMEM_MAX_BINS, TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM}, + {sweep, + TUNE_DYNAMIC_SMEM_MAX_BINS, + TUNE_DYNAMIC_SMEM_MAX_BYTES, + TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS, + TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS, + TUNE_DYNAMIC_SMEM_EVEN_3CH_MAX_BINS, + TUNE_DYNAMIC_SMEM_EVEN_4CH_MAX_BINS}, + TUNE_INIT_KERNEL_PDL_TRIGGER_MAX_BINS}; } }; #endif // !TUNE_BASE diff --git a/cub/benchmarks/bench/histogram/multi/even.cu b/cub/benchmarks/bench/histogram/multi/even.cu index 4131206a8bd7..a3ddfc377da3 100644 --- a/cub/benchmarks/bench/histogram/multi/even.cu +++ b/cub/benchmarks/bench/histogram/multi/even.cu @@ -13,6 +13,15 @@ // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 +// %RANGE% TUNE_STATIC_SMEM_MAX_BINS smem_bins 0:512:256 +// %RANGE% TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM smem_blocks 0:4:1 +// %RANGE% TUNE_DYNAMIC_SMEM_MAX_BINS dyn_bins 0:57088:57088 +// %RANGE% TUNE_DYNAMIC_SMEM_MAX_BYTES dyn_bytes 0:228352:228352 +// %RANGE% TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS dyn_range_bins 0:2048:2048 +// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS dyn_even_2ch_bins 0:28544:28544 +// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_3CH_MAX_BINS dyn_even_3ch_bins 0:19029:19029 +// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_4CH_MAX_BINS dyn_even_4ch_bins 0:8192:8192 +// %RANGE% TUNE_INIT_KERNEL_PDL_TRIGGER_MAX_BINS pdl_bins 0:2048:2048 template static void even(nvbench::state& state, nvbench::type_list) diff --git a/cub/benchmarks/bench/histogram/multi/range.cu b/cub/benchmarks/bench/histogram/multi/range.cu index dcb402d7db20..7e8c830caf76 100644 --- a/cub/benchmarks/bench/histogram/multi/range.cu +++ b/cub/benchmarks/bench/histogram/multi/range.cu @@ -15,6 +15,15 @@ // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 +// %RANGE% TUNE_STATIC_SMEM_MAX_BINS smem_bins 0:512:256 +// %RANGE% TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM smem_blocks 0:4:1 +// %RANGE% TUNE_DYNAMIC_SMEM_MAX_BINS dyn_bins 0:57088:57088 +// %RANGE% TUNE_DYNAMIC_SMEM_MAX_BYTES dyn_bytes 0:228352:228352 +// %RANGE% TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS dyn_range_bins 0:2048:2048 +// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS dyn_even_2ch_bins 0:28544:28544 +// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_3CH_MAX_BINS dyn_even_3ch_bins 0:19029:19029 +// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_4CH_MAX_BINS dyn_even_4ch_bins 0:8192:8192 +// %RANGE% TUNE_INIT_KERNEL_PDL_TRIGGER_MAX_BINS pdl_bins 0:2048:2048 template static void range(nvbench::state& state, nvbench::type_list) diff --git a/cub/benchmarks/bench/histogram/range.cu b/cub/benchmarks/bench/histogram/range.cu index e1e808466395..882d94e9493a 100644 --- a/cub/benchmarks/bench/histogram/range.cu +++ b/cub/benchmarks/bench/histogram/range.cu @@ -15,6 +15,15 @@ // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 +// %RANGE% TUNE_STATIC_SMEM_MAX_BINS smem_bins 0:512:256 +// %RANGE% TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM smem_blocks 0:4:1 +// %RANGE% TUNE_DYNAMIC_SMEM_MAX_BINS dyn_bins 0:57088:57088 +// %RANGE% TUNE_DYNAMIC_SMEM_MAX_BYTES dyn_bytes 0:228352:228352 +// %RANGE% TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS dyn_range_bins 0:2048:2048 +// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS dyn_even_2ch_bins 0:28544:28544 +// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_3CH_MAX_BINS dyn_even_3ch_bins 0:19029:19029 +// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_4CH_MAX_BINS dyn_even_4ch_bins 0:8192:8192 +// %RANGE% TUNE_INIT_KERNEL_PDL_TRIGGER_MAX_BINS pdl_bins 0:2048:2048 template static void range(nvbench::state& state, nvbench::type_list) From f37c8dac137e49d3fb3505e3cd318eb91869c5ea Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sun, 2 Aug 2026 10:56:48 +0000 Subject: [PATCH 24/45] [cub] Use typed histogram privatization modes --- cub/benchmarks/bench/histogram/even.cu | 4 +- .../bench/histogram/histogram_common.cuh | 12 +- cub/benchmarks/bench/histogram/multi/even.cu | 4 +- cub/benchmarks/bench/histogram/multi/range.cu | 4 +- cub/benchmarks/bench/histogram/range.cu | 4 +- cub/cub/agent/agent_histogram.cuh | 118 +++++++------ .../device/dispatch/dispatch_histogram.cuh | 161 ++++++++---------- .../dispatch/kernels/kernel_histogram.cuh | 72 ++++---- .../dispatch/tuning/tuning_histogram.cuh | 140 ++++++++------- cub/test/catch2_test_device_histogram_env.cu | 144 ++++++---------- .../catch2_test_device_histogram_env_api.cu | 23 ++- 11 files changed, 316 insertions(+), 370 deletions(-) diff --git a/cub/benchmarks/bench/histogram/even.cu b/cub/benchmarks/bench/histogram/even.cu index b255db201b45..a45c3c1aae47 100644 --- a/cub/benchmarks/bench/histogram/even.cu +++ b/cub/benchmarks/bench/histogram/even.cu @@ -9,13 +9,11 @@ // %RANGE% TUNE_THREADS tpb 128:1024:32 // %RANGE% TUNE_RLE_COMPRESS rle 0:1:1 // %RANGE% TUNE_WORK_STEALING ws 0:1:1 -// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1 // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 -// %RANGE% TUNE_STATIC_SMEM_MAX_BINS smem_bins 0:512:256 +// %RANGE% TUNE_STATIC_SMEM_MAX_BYTES smem_bytes 0:512:256 // %RANGE% TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM smem_blocks 0:4:1 -// %RANGE% TUNE_DYNAMIC_SMEM_MAX_BINS dyn_bins 0:57088:57088 // %RANGE% TUNE_DYNAMIC_SMEM_MAX_BYTES dyn_bytes 0:228352:228352 // %RANGE% TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS dyn_range_bins 0:2048:2048 // %RANGE% TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS dyn_even_2ch_bins 0:28544:28544 diff --git a/cub/benchmarks/bench/histogram/histogram_common.cuh b/cub/benchmarks/bench/histogram/histogram_common.cuh index 52d80bfed312..1f45cd84ba8a 100644 --- a/cub/benchmarks/bench/histogram/histogram_common.cuh +++ b/cub/benchmarks/bench/histogram/histogram_common.cuh @@ -19,14 +19,6 @@ # define TUNE_VEC_SIZE (1 << TUNE_VEC_SIZE_POW) -# if TUNE_MEM_PREFERENCE == 0 -constexpr cub::BlockHistogramMemoryPreference MEM_PREFERENCE = cub::GMEM; -# elif TUNE_MEM_PREFERENCE == 1 -constexpr cub::BlockHistogramMemoryPreference MEM_PREFERENCE = cub::SMEM; -# else // TUNE_MEM_PREFERENCE == 2 -constexpr cub::BlockHistogramMemoryPreference MEM_PREFERENCE = cub::BLEND; -# endif // TUNE_MEM_PREFERENCE - # if TUNE_LOAD_ALGORITHM_ID == 0 # define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_DIRECT # elif TUNE_LOAD_ALGORITHM_ID == 1 @@ -52,12 +44,10 @@ struct bench_policy_selector load_algorithm, TUNE_LOAD_MODIFIER, TUNE_RLE_COMPRESS, - MEM_PREFERENCE, TUNE_WORK_STEALING}; return {sweep, - {sweep, TUNE_STATIC_SMEM_MAX_BINS, TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM}, + {sweep, TUNE_STATIC_SMEM_MAX_BYTES, TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM}, {sweep, - TUNE_DYNAMIC_SMEM_MAX_BINS, TUNE_DYNAMIC_SMEM_MAX_BYTES, TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS, TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS, diff --git a/cub/benchmarks/bench/histogram/multi/even.cu b/cub/benchmarks/bench/histogram/multi/even.cu index a3ddfc377da3..22df064790f3 100644 --- a/cub/benchmarks/bench/histogram/multi/even.cu +++ b/cub/benchmarks/bench/histogram/multi/even.cu @@ -9,13 +9,11 @@ // %RANGE% TUNE_THREADS tpb 128:1024:32 // %RANGE% TUNE_RLE_COMPRESS rle 0:1:1 // %RANGE% TUNE_WORK_STEALING ws 0:1:1 -// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1 // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 -// %RANGE% TUNE_STATIC_SMEM_MAX_BINS smem_bins 0:512:256 +// %RANGE% TUNE_STATIC_SMEM_MAX_BYTES smem_bytes 0:512:256 // %RANGE% TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM smem_blocks 0:4:1 -// %RANGE% TUNE_DYNAMIC_SMEM_MAX_BINS dyn_bins 0:57088:57088 // %RANGE% TUNE_DYNAMIC_SMEM_MAX_BYTES dyn_bytes 0:228352:228352 // %RANGE% TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS dyn_range_bins 0:2048:2048 // %RANGE% TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS dyn_even_2ch_bins 0:28544:28544 diff --git a/cub/benchmarks/bench/histogram/multi/range.cu b/cub/benchmarks/bench/histogram/multi/range.cu index 7e8c830caf76..f38f44cb6c94 100644 --- a/cub/benchmarks/bench/histogram/multi/range.cu +++ b/cub/benchmarks/bench/histogram/multi/range.cu @@ -11,13 +11,11 @@ // %RANGE% TUNE_THREADS tpb 128:1024:32 // %RANGE% TUNE_RLE_COMPRESS rle 0:1:1 // %RANGE% TUNE_WORK_STEALING ws 0:1:1 -// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1 // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 -// %RANGE% TUNE_STATIC_SMEM_MAX_BINS smem_bins 0:512:256 +// %RANGE% TUNE_STATIC_SMEM_MAX_BYTES smem_bytes 0:512:256 // %RANGE% TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM smem_blocks 0:4:1 -// %RANGE% TUNE_DYNAMIC_SMEM_MAX_BINS dyn_bins 0:57088:57088 // %RANGE% TUNE_DYNAMIC_SMEM_MAX_BYTES dyn_bytes 0:228352:228352 // %RANGE% TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS dyn_range_bins 0:2048:2048 // %RANGE% TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS dyn_even_2ch_bins 0:28544:28544 diff --git a/cub/benchmarks/bench/histogram/range.cu b/cub/benchmarks/bench/histogram/range.cu index 882d94e9493a..674a7871862c 100644 --- a/cub/benchmarks/bench/histogram/range.cu +++ b/cub/benchmarks/bench/histogram/range.cu @@ -11,13 +11,11 @@ // %RANGE% TUNE_THREADS tpb 128:1024:32 // %RANGE% TUNE_RLE_COMPRESS rle 0:1:1 // %RANGE% TUNE_WORK_STEALING ws 0:1:1 -// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1 // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 -// %RANGE% TUNE_STATIC_SMEM_MAX_BINS smem_bins 0:512:256 +// %RANGE% TUNE_STATIC_SMEM_MAX_BYTES smem_bytes 0:512:256 // %RANGE% TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM smem_blocks 0:4:1 -// %RANGE% TUNE_DYNAMIC_SMEM_MAX_BINS dyn_bins 0:57088:57088 // %RANGE% TUNE_DYNAMIC_SMEM_MAX_BYTES dyn_bytes 0:228352:228352 // %RANGE% TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS dyn_range_bins 0:2048:2048 // %RANGE% TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS dyn_even_2ch_bins 0:28544:28544 diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index f52e96a38a12..73b0319a1821 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -88,9 +88,9 @@ template + int VecSize = 4, + int PrivatizedSmemMaxBytes = 0> struct agent_histogram_policy { /// Threads per thread block @@ -101,12 +101,12 @@ struct agent_histogram_policy /// Whether to perform localized RLE to compress samples before histogramming static constexpr bool IS_RLE_COMPRESS = RleCompress; - /// Whether to prefer privatized shared-memory bins (versus privatized global-memory bins) - static constexpr BlockHistogramMemoryPreference MEM_PREFERENCE = MemoryPreference; - /// Whether to dequeue tiles from a global work queue static constexpr bool IS_WORK_STEALING = WorkStealing; + /// Maximum compile-time-sized shared-memory allocation for privatized bins + static constexpr int PRIVATIZED_SMEM_MAX_BYTES = PrivatizedSmemMaxBytes; + /// Vector size for samples loading (1, 2, 4) static constexpr int VEC_SIZE = VecSize; static_assert(VEC_SIZE == 1 || VEC_SIZE == 2 || VEC_SIZE == 4); @@ -128,19 +128,40 @@ template -using AgentHistogramPolicy - CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") = detail::agent_histogram_policy< - ThreadsPerBlock, - PixelsPerThread, - LoadAlgorithm, - LoadModifier, - RleCompress, - MemoryPreference, - WorkStealing, - VecSize>; +struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") AgentHistogramPolicy + : detail::agent_histogram_policy +{ + static constexpr BlockHistogramMemoryPreference MEM_PREFERENCE = MemoryPreference; +}; namespace detail::histogram { +struct HistogramPrivatizedStaticSmem +{}; + +struct HistogramPrivatizedDynamicSmem +{}; + +struct HistogramPrivatizedGmem +{}; + +template +inline constexpr bool is_privatized_static_smem_v = + ::cuda::std::is_same_v; + +template +inline constexpr bool is_privatized_dynamic_smem_v = + ::cuda::std::is_same_v; + +template +inline constexpr bool is_privatized_gmem_v = ::cuda::std::is_same_v; + // Return a native pixel pointer (specialized for CacheModifiedInputIterator types) template _CCCL_DEVICE _CCCL_FORCEINLINE auto NativePointer(CacheModifiedInputIterator itr) @@ -188,13 +209,13 @@ _CCCL_DEVICE _CCCL_FORCEINLINE auto NativePointer(IteratorT itr) //! @tparam OffsetT //! Signed integer type for global offsets //! -//! @tparam UseDynamicSmem -//! Whether the privatized histogram is supplied separately in dynamic shared memory. +//! @tparam PrivatizationMode +//! Storage mode for the privatized histogram. //! //! @tparam OutputCounterT //! Integer type for final output histogram bins. May be wider than `CounterT`. template struct AgentHistogram { static_assert(sizeof(CounterT) <= sizeof(OutputCounterT), "The output histogram counter must be at least as wide as the local counter"); - static constexpr int privatized_smem_bins = PrivatizedSmemBins; + static_assert(is_privatized_static_smem_v || is_privatized_dynamic_smem_v + || is_privatized_gmem_v); + static constexpr bool uses_static_smem = is_privatized_static_smem_v; + static constexpr bool uses_dynamic_smem = is_privatized_dynamic_smem_v; + static constexpr bool uses_gmem = is_privatized_gmem_v; + static constexpr int static_smem_slots_per_channel = + uses_static_smem ? AgentHistogramPolicyT::PRIVATIZED_SMEM_MAX_BYTES / int{sizeof(CounterT)} / NumActiveChannels : 1; + static_assert(!uses_static_smem || static_smem_slots_per_channel > 1, + "Static-SMEM privatization requires room for at least one bin and its padding counter"); + static constexpr int privatized_smem_bins = uses_static_smem ? static_smem_slots_per_channel - 1 : 0; static constexpr int vec_size = AgentHistogramPolicyT::VEC_SIZE; static constexpr int threads_per_block = AgentHistogramPolicyT::BLOCK_THREADS; static constexpr int pixels_per_thread = AgentHistogramPolicyT::PIXELS_PER_THREAD; @@ -219,10 +248,6 @@ struct AgentHistogram static constexpr bool is_rle_compress = AgentHistogramPolicyT::IS_RLE_COMPRESS; static constexpr bool is_work_stealing = AgentHistogramPolicyT::IS_WORK_STEALING; static constexpr CacheLoadModifier load_modifier = AgentHistogramPolicyT::LOAD_MODIFIER; - static constexpr auto mem_preference = - (UseDynamicSmem || PrivatizedSmemBins > 0) - ? BlockHistogramMemoryPreference{AgentHistogramPolicyT::MEM_PREFERENCE} - : GMEM; using SampleT = it_value_t; using PixelT = typename CubVector::Type; @@ -245,7 +270,7 @@ struct AgentHistogram struct _TempStorage { - CounterT histograms[NumActiveChannels][PrivatizedSmemBins + 1]; + CounterT histograms[NumActiveChannels][privatized_smem_bins + 1]; int tile_idx; union @@ -269,23 +294,20 @@ struct AgentHistogram const OutputDecodeOpT* output_decode_op; // determines output bin-id from privatized counter index, one for each // channel PrivatizedDecodeOpT* privatized_decode_op; // determines privatized counter index from sample, one for each channel - bool prefer_smem; // whether this block uses shared-memory privatization - _CCCL_DEVICE _CCCL_FORCEINLINE CounterT* PrivatizedHistogram(int channel) { - if (prefer_smem) + if constexpr (uses_dynamic_smem) { - if constexpr (UseDynamicSmem) - { - return dyn_smem_histograms[channel]; - } - else - { - return temp_storage.histograms[channel]; - } + return dyn_smem_histograms[channel]; + } + else if constexpr (uses_static_smem) + { + return temp_storage.histograms[channel]; + } + else + { + return d_privatized_histograms[channel]; } - - return d_privatized_histograms[channel]; } _CCCL_DEVICE _CCCL_FORCEINLINE void ZeroBinCounters() @@ -639,13 +661,12 @@ struct AgentHistogram , d_output_histograms(d_output_histograms) , output_decode_op(output_decode_op) , privatized_decode_op(privatized_decode_op) - , prefer_smem(mem_preference == SMEM || (mem_preference == BLEND && (blockIdx.x & 1))) { - static_assert(!UseDynamicSmem, - "AgentHistogram with UseDynamicSmem=true requires the dynamic-SMEM " + static_assert(!uses_dynamic_smem, + "AgentHistogram with dynamic-SMEM privatization requires the dynamic-SMEM " "constructor that takes an extern __shared__ base pointer."); - if constexpr (PrivatizedSmemBins == 0) + if constexpr (uses_gmem) { const int block_id = static_cast((blockIdx.y * gridDim.x) + blockIdx.x); for (int ch = 0; ch < NumActiveChannels; ++ch) @@ -675,9 +696,8 @@ struct AgentHistogram , d_output_histograms(d_output_histograms) , output_decode_op(output_decode_op) , privatized_decode_op(privatized_decode_op) - , prefer_smem(mem_preference == SMEM || (mem_preference == BLEND && (blockIdx.x & 1))) { - static_assert(UseDynamicSmem, "Dynamic-SMEM AgentHistogram constructor requires UseDynamicSmem=true."); + static_assert(uses_dynamic_smem, "Dynamic-SMEM AgentHistogram constructor requires dynamic-SMEM mode."); CounterT* p = dyn_smem_histogram_base; _CCCL_PRAGMA_UNROLL_FULL() @@ -686,16 +706,6 @@ struct AgentHistogram this->dyn_smem_histograms[ch] = p; p += num_privatized_bins[ch]; } - - if (!prefer_smem) - { - const int block_id = static_cast((blockIdx.y * gridDim.x) + blockIdx.x); - for (int ch = 0; ch < NumActiveChannels; ++ch) - { - const auto offset = static_cast<::cuda::std::int64_t>(block_id) * num_privatized_bins[ch]; - this->d_privatized_histograms[ch] = d_privatized_histograms[ch] + offset; - } - } } //! @brief Consume image diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 115fa5089e88..1c4c39da8598 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -91,12 +91,12 @@ struct DeviceHistogramKernelSource } /// Returns the default histogram sweep kernel that receives pre-initialized decode operators from the host. - template + template _CCCL_HIDE_FROM_ABI CUB_RUNTIME_FUNCTION static constexpr auto HistogramSweepKernel() { return &DeviceHistogramSweepKernel< PolicyT, - UseStaticSmem, + PrivatizationMode, NUM_CHANNELS, NUM_ACTIVE_CHANNELS, SampleIteratorT, @@ -124,7 +124,7 @@ struct DeviceHistogramKernelSource /// Returns the device-init histogram sweep kernel that initializes decode operators from level arrays in the kernel. template (); auto sweep_kernel = [&] { - if constexpr (UseDynamicSmem) + if constexpr (is_privatized_dynamic_smem_v) { static_assert(!IsDeviceInit, "Dynamic shared-memory histograms require host-initialized transforms"); using output_decode_op_t = typename FirstLevelArrayT::value_type; @@ -262,7 +261,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( { return kernel_source.template HistogramSweepKernelDeviceInit< PolicySelector, - UseStaticSmem, + PrivatizationMode, FirstLevelArrayT, SecondLevelArrayT, IsEven, @@ -273,20 +272,21 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( using output_decode_op_t = typename FirstLevelArrayT::value_type; using privatized_decode_op_t = typename SecondLevelArrayT::value_type; return kernel_source - .template HistogramSweepKernel(); + .template HistogramSweepKernel(); } }(); constexpr auto tier = - UseDynamicSmem ? privatization_tier::dynamic_smem - : UseStaticSmem ? privatization_tier::static_smem - : privatization_tier::gmem; + is_privatized_dynamic_smem_v ? privatization_tier::dynamic_smem + : is_privatized_static_smem_v + ? privatization_tier::static_smem + : privatization_tier::gmem; const HistogramSweepPolicy sweep = sweep_policy(active_policy); const int threads_per_block = sweep.threads_per_block; const int items_per_thread = sweep.items_per_thread; int dynamic_smem_bytes = 0; - if constexpr (UseDynamicSmem) + if constexpr (is_privatized_dynamic_smem_v) { for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel) { @@ -294,7 +294,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( } NV_IF_TARGET(NV_IS_HOST, ({ if (const auto error = CubDebug(launcher_factory.set_max_dynamic_smem_size_for( - sweep_kernel, active_policy.dynamic_smem.max_bytes))) + sweep_kernel, active_policy.dynamic_smem.max_privatized_smem_bytes))) { return error; } @@ -347,8 +347,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( void* allocations[NUM_ALLOCATIONS] = {}; size_t allocation_sizes[NUM_ALLOCATIONS]; - const bool requires_global_privatization = - (!UseDynamicSmem && !UseStaticSmem) || sweep.mem_preference != BlockHistogramMemoryPreference::SMEM; + constexpr bool requires_global_privatization = is_privatized_gmem_v; for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL) { allocation_sizes[CHANNEL] = @@ -593,14 +592,13 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device } const HistogramPolicy active_policy = policy_selector(cc); - if (!should_use_static_smem(active_policy, max_num_output_bins)) + if (!should_use_static_smem(active_policy, max_num_output_bins, int{kernel_source.CounterSize()}, NUM_ACTIVE_CHANNELS)) { // Dispatch global-memory-privatized approach if (const auto error = CubDebug( (detail::histogram::dispatch( @@ -630,8 +628,7 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device if (const auto error = CubDebug( (detail::histogram::dispatch( @@ -769,8 +766,7 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device if (const auto error = CubDebug( (detail::histogram::dispatch( @@ -817,9 +813,8 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_policy(long) -> HistogramPolicy sweep::LOAD_ALGORITHM, sweep::LOAD_MODIFIER, sweep::IS_RLE_COMPRESS, - sweep::MEM_PREFERENCE, sweep::IS_WORK_STEALING}; - return {sweep_policy, {sweep_policy, 256, 0}, {sweep_policy, 0, 0, 0, 0, 0, 0}, 0}; + return {sweep_policy, {sweep_policy, 257 * sizeof(unsigned int), 0}, {sweep_policy, 0, 0, 0, 0, 0}, 0}; } // TODO(bgruber): drop in CCCL 4.0 @@ -920,8 +915,7 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( if (const auto error = CubDebug( (detail::histogram::dispatch( @@ -985,30 +979,28 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( privatized_decode_op[channel].Init(d_levels[channel], num_output_levels[channel]); } - return CubDebug( - (detail::histogram::dispatch( - d_temp_storage, - temp_storage_bytes, - d_samples, - d_output_histograms, - num_output_levels, - num_output_levels, - output_decode_op, - privatized_decode_op, - max_num_output_bins, - num_row_pixels, - num_rows, - row_stride_samples, - stream, - policy_selector, - kernel_source, - launcher_factory))); + return CubDebug((detail::histogram::dispatch( + d_temp_storage, + temp_storage_bytes, + d_samples, + d_output_histograms, + num_output_levels, + num_output_levels, + output_decode_op, + privatized_decode_op, + max_num_output_bins, + num_row_pixels, + num_rows, + row_stride_samples, + stream, + policy_selector, + kernel_source, + launcher_factory))); } } @@ -1020,14 +1012,14 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( } // Dispatch - if (!should_use_static_smem(active_policy, max_num_output_bins)) + if (!should_use_static_smem( + active_policy, max_num_output_bins, int{kernel_source.CounterSize()}, NUM_ACTIVE_CHANNELS)) { // Too many bins to keep in shared memory. if (const auto error = CubDebug( (detail::histogram::dispatch( @@ -1057,8 +1049,7 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( if (const auto error = CubDebug( (detail::histogram::dispatch( @@ -1168,8 +1159,7 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t dispatch_even( if (const auto error = CubDebug( (detail::histogram::dispatch( @@ -1240,39 +1230,37 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t dispatch_even( if (should_use_dynamic_smem( active_policy, max_num_output_bins, int{kernel_source.CounterSize()}, NUM_ACTIVE_CHANNELS)) { - return CubDebug( - (detail::histogram::dispatch( - d_temp_storage, - temp_storage_bytes, - d_samples, - d_output_histograms, - num_output_levels, - num_output_levels, - output_decode_op, - privatized_decode_op, - max_num_output_bins, - num_row_pixels, - num_rows, - row_stride_samples, - stream, - policy_selector, - kernel_source, - launcher_factory))); + return CubDebug((detail::histogram::dispatch( + d_temp_storage, + temp_storage_bytes, + d_samples, + d_output_histograms, + num_output_levels, + num_output_levels, + output_decode_op, + privatized_decode_op, + max_num_output_bins, + num_row_pixels, + num_rows, + row_stride_samples, + stream, + policy_selector, + kernel_source, + launcher_factory))); } - if (!should_use_static_smem(active_policy, max_num_output_bins)) + if (!should_use_static_smem( + active_policy, max_num_output_bins, int{kernel_source.CounterSize()}, NUM_ACTIVE_CHANNELS)) { if (const auto error = CubDebug( (detail::histogram::dispatch( @@ -1301,8 +1289,7 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t dispatch_even( if (const auto error = CubDebug( (detail::histogram::dispatch( diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index 3d5b4b436d14..8d519e0df586 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -627,8 +627,8 @@ _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramInitKernel( //! @tparam PolicySelector //! Selects the tuning policy //! -//! @tparam UseStaticSmem -//! Whether the privatized histogram is stored in compile-time-sized shared memory +//! @tparam PrivatizationMode +//! Storage mode for the privatized histogram //! //! @tparam NumChannels //! Number of channels interleaved in the input data (may be greater than the number of channels @@ -692,7 +692,7 @@ _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramInitKernel( //! @param tile_queue //! Drain queue descriptor for dynamically mapping tile data onto thread blocks template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(sweep_policy( - current_policy()) - .threads_per_block), - int(UseStaticSmem ? current_policy().static_smem.min_blocks_per_sm : 0)) +__launch_bounds__( + int(sweep_policy< + is_privatized_static_smem_v ? privatization_tier::static_smem : privatization_tier::gmem>( + current_policy()) + .threads_per_block), + int(is_privatized_static_smem_v + ? current_policy().static_smem.min_blocks_per_sm + : 0)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepKernel( const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, @@ -723,9 +727,8 @@ __launch_bounds__(int(sweep_policy tile_queue) { static constexpr HistogramPolicy hp = current_policy(); - static constexpr auto sweep = - sweep_policy(hp); - static constexpr int privatized_smem_bins = UseStaticSmem ? hp.static_smem.max_bins : 0; + static constexpr auto sweep = sweep_policy< + is_privatized_static_smem_v ? privatization_tier::static_smem : privatization_tier::gmem>(hp); // Thread block type for compositing input tiles using AgentHistogramPolicyT = agent_histogram_policy< @@ -734,12 +737,12 @@ __launch_bounds__(int(sweep_policy; + sweep.vec_size, + is_privatized_static_smem_v ? hp.static_smem.max_privatized_smem_bytes : 0>; using AgentHistogramT = AgentHistogram; - static_assert(AgentHistogramT::privatized_smem_bins == privatized_smem_bins); // Shared memory for AgentHistogram __shared__ typename AgentHistogramT::TempStorage temp_storage; @@ -811,18 +812,17 @@ __launch_bounds__(int(current_policy().dynamic_smem.sweep.thread static constexpr HistogramPolicy hp = current_policy(); static constexpr HistogramSweepPolicy sweep = hp.dynamic_smem.sweep; - using AgentHistogramPolicyT = agent_histogram_policy< - sweep.threads_per_block, - sweep.items_per_thread, - sweep.load_algorithm, - sweep.load_modifier, - sweep.rle_compress, - sweep.mem_preference, - sweep.work_stealing, - sweep.vec_size>; + using AgentHistogramPolicyT = + agent_histogram_policy; using AgentHistogramT = AgentHistogram().dynamic_smem.sweep.thread PrivatizedDecodeOpT, OutputDecodeOpT, OffsetT, - true, OutputCounterT>; __shared__ typename AgentHistogramT::TempStorage temp_storage; @@ -870,8 +869,8 @@ __launch_bounds__(int(current_policy().dynamic_smem.sweep.thread //! @tparam PolicySelector //! Selects the tuning policy //! -//! @tparam UseStaticSmem -//! Whether the privatized histogram is stored in compile-time-sized shared memory +//! @tparam PrivatizationMode +//! Storage mode for the privatized histogram //! //! @tparam NumChannels //! Number of channels interleaved in the input data (may be greater than the number of channels @@ -947,7 +946,7 @@ __launch_bounds__(int(current_policy().dynamic_smem.sweep.thread //! @param tile_queue //! Drain queue descriptor for dynamically mapping tile data onto thread blocks template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(sweep_policy( +__launch_bounds__(int(sweep_policy ? privatization_tier::static_smem + : privatization_tier::gmem>( current_policy()) .threads_per_block)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDeviceInitKernel( @@ -981,9 +981,8 @@ __launch_bounds__(int(sweep_policy tile_queue) { static constexpr HistogramPolicy hp = current_policy(); - static constexpr auto sweep = - sweep_policy(hp); - static constexpr int privatized_smem_bins = UseStaticSmem ? hp.static_smem.max_bins : 0; + static constexpr auto sweep = sweep_policy< + is_privatized_static_smem_v ? privatization_tier::static_smem : privatization_tier::gmem>(hp); OutputDecodeOpT output_decode_op[NumActiveChannels]; PrivatizedDecodeOpT privatized_decode_op[NumActiveChannels]; @@ -1018,12 +1017,12 @@ __launch_bounds__(int(sweep_policy; + sweep.vec_size, + is_privatized_static_smem_v ? hp.static_smem.max_privatized_smem_bytes : 0>; using AgentHistogramT = AgentHistogram; // Shared memory for AgentHistogram diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index d988e0228275..767d7c62c4cd 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -34,7 +34,6 @@ struct HistogramSweepPolicy BlockLoadAlgorithm load_algorithm; //!< Algorithm used for loading samples CacheLoadModifier load_modifier; //!< Cache modifier used for loading samples bool rle_compress; //!< Whether to locally run-length encode samples - BlockHistogramMemoryPreference mem_preference; //!< Preferred storage for privatized bins bool work_stealing; //!< Whether blocks dequeue tiles from a global queue [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool @@ -43,7 +42,7 @@ struct HistogramSweepPolicy return lhs.threads_per_block == rhs.threads_per_block && lhs.items_per_thread == rhs.items_per_thread && lhs.vec_size == rhs.vec_size && lhs.load_algorithm == rhs.load_algorithm && lhs.load_modifier == rhs.load_modifier && lhs.rle_compress == rhs.rle_compress - && lhs.mem_preference == rhs.mem_preference && lhs.work_stealing == rhs.work_stealing; + && lhs.work_stealing == rhs.work_stealing; } }; @@ -51,13 +50,14 @@ struct HistogramSweepPolicy struct HistogramStaticSmemPolicy { HistogramSweepPolicy sweep; - int max_bins; //!< Maximum number of bins per channel + int max_privatized_smem_bytes; //!< Maximum compile-time-sized shared-memory allocation int min_blocks_per_sm; //!< Minimum blocks per SM requested through launch bounds [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(const HistogramStaticSmemPolicy& lhs, const HistogramStaticSmemPolicy& rhs) noexcept { - return lhs.sweep == rhs.sweep && lhs.max_bins == rhs.max_bins && lhs.min_blocks_per_sm == rhs.min_blocks_per_sm; + return lhs.sweep == rhs.sweep && lhs.max_privatized_smem_bytes == rhs.max_privatized_smem_bytes + && lhs.min_blocks_per_sm == rhs.min_blocks_per_sm; } }; @@ -65,8 +65,7 @@ struct HistogramStaticSmemPolicy struct HistogramDynamicSmemPolicy { HistogramSweepPolicy sweep; - int max_bins; //!< Maximum bins per channel for a single-channel histogram - int max_bytes; //!< Maximum total dynamic shared-memory allocation + int max_privatized_smem_bytes; //!< Maximum runtime-sized shared-memory allocation int range_max_bins; //!< Maximum bins per channel for multi-channel HistogramRange int even_2ch_max_bins; //!< Maximum bins per channel for two-channel HistogramEven int even_3ch_max_bins; //!< Maximum bins per channel for three-channel HistogramEven @@ -75,7 +74,7 @@ struct HistogramDynamicSmemPolicy [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(const HistogramDynamicSmemPolicy& lhs, const HistogramDynamicSmemPolicy& rhs) noexcept { - return lhs.sweep == rhs.sweep && lhs.max_bins == rhs.max_bins && lhs.max_bytes == rhs.max_bytes + return lhs.sweep == rhs.sweep && lhs.max_privatized_smem_bytes == rhs.max_privatized_smem_bytes && lhs.range_max_bins == rhs.range_max_bins && lhs.even_2ch_max_bins == rhs.even_2ch_max_bins && lhs.even_3ch_max_bins == rhs.even_3ch_max_bins && lhs.even_4ch_max_bins == rhs.even_4ch_max_bins; } @@ -110,19 +109,19 @@ struct HistogramPolicy << "{ .threads_per_block = " << sweep.threads_per_block << ", .items_per_thread = " << sweep.items_per_thread << ", .vec_size = " << sweep.vec_size << ", .load_algorithm = " << sweep.load_algorithm << ", .load_modifier = " << sweep.load_modifier << ", .rle_compress = " << sweep.rle_compress - << ", .mem_preference = " << sweep.mem_preference << ", .work_stealing = " << sweep.work_stealing << " }"; + << ", .work_stealing = " << sweep.work_stealing << " }"; }; os << "HistogramPolicy { .gmem = "; print_sweep(p.gmem); os << ", .static_smem = { .sweep = "; print_sweep(p.static_smem.sweep); - os << ", .max_bins = " << p.static_smem.max_bins << ", .min_blocks_per_sm = " << p.static_smem.min_blocks_per_sm - << " }, .dynamic_smem = { .sweep = "; + os << ", .max_privatized_smem_bytes = " << p.static_smem.max_privatized_smem_bytes + << ", .min_blocks_per_sm = " << p.static_smem.min_blocks_per_sm << " }, .dynamic_smem = { .sweep = "; print_sweep(p.dynamic_smem.sweep); return os - << ", .max_bins = " << p.dynamic_smem.max_bins << ", .max_bytes = " << p.dynamic_smem.max_bytes - << ", .range_max_bins = " << p.dynamic_smem.range_max_bins << ", .even_2ch_max_bins = " - << p.dynamic_smem.even_2ch_max_bins << ", .even_3ch_max_bins = " << p.dynamic_smem.even_3ch_max_bins + << ", .max_privatized_smem_bytes = " << p.dynamic_smem.max_privatized_smem_bytes << ", .range_max_bins = " + << p.dynamic_smem.range_max_bins << ", .even_2ch_max_bins = " << p.dynamic_smem.even_2ch_max_bins + << ", .even_3ch_max_bins = " << p.dynamic_smem.even_3ch_max_bins << ", .even_4ch_max_bins = " << p.dynamic_smem.even_4ch_max_bins << " }, .init_kernel_pdl_trigger_max_bins = " << p.init_kernel_pdl_trigger_max_bins << " }"; } @@ -131,30 +130,28 @@ struct HistogramPolicy namespace detail::histogram { -template +template struct static_smem_policy { - using SweepPolicyT = SweepPolicy; - static constexpr int MAX_BINS = MaxBins; - static constexpr int MIN_BLOCKS_PER_SM = MinBlocksPerSm; + using SweepPolicyT = SweepPolicy; + static constexpr int MAX_PRIVATIZED_SMEM_BYTES = MaxPrivatizedSmemBytes; + static constexpr int MIN_BLOCKS_PER_SM = MinBlocksPerSm; }; template struct dynamic_smem_policy { - using SweepPolicyT = SweepPolicy; - static constexpr int MAX_BINS = MaxBins; - static constexpr int MAX_BYTES = MaxBytes; - static constexpr int RANGE_MAX_BINS = RangeMaxBins; - static constexpr int EVEN_2CH_MAX_BINS = Even2chMaxBins; - static constexpr int EVEN_3CH_MAX_BINS = Even3chMaxBins; - static constexpr int EVEN_4CH_MAX_BINS = Even4chMaxBins; + using SweepPolicyT = SweepPolicy; + static constexpr int MAX_PRIVATIZED_SMEM_BYTES = MaxPrivatizedSmemBytes; + static constexpr int RANGE_MAX_BINS = RangeMaxBins; + static constexpr int EVEN_2CH_MAX_BINS = Even2chMaxBins; + static constexpr int EVEN_3CH_MAX_BINS = Even3chMaxBins; + static constexpr int EVEN_4CH_MAX_BINS = Even4chMaxBins; }; enum class privatization_tier @@ -181,9 +178,33 @@ template } } -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool should_use_static_smem(const HistogramPolicy& policy, int num_bins) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int max_privatized_smem_bins( + int max_privatized_smem_bytes, int counter_size, int num_active_channels, int padding_bins_per_channel = 0) { - return num_bins > 0 && num_bins <= policy.static_smem.max_bins; + if (max_privatized_smem_bytes <= 0 || counter_size <= 0 || num_active_channels <= 0) + { + return 0; + } + const int slots_per_channel = max_privatized_smem_bytes / counter_size / num_active_channels; + return slots_per_channel > padding_bins_per_channel ? slots_per_channel - padding_bins_per_channel : 0; +} + +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int +max_privatized_static_smem_bins(const HistogramPolicy& policy, int counter_size, int num_active_channels) +{ + return max_privatized_smem_bins(policy.static_smem.max_privatized_smem_bytes, counter_size, num_active_channels, 1); +} + +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int +max_privatized_dynamic_smem_bins(const HistogramPolicy& policy, int counter_size, int num_active_channels) +{ + return max_privatized_smem_bins(policy.dynamic_smem.max_privatized_smem_bytes, counter_size, num_active_channels); +} + +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool +should_use_static_smem(const HistogramPolicy& policy, int num_bins, int counter_size, int num_active_channels) +{ + return num_bins > 0 && num_bins <= max_privatized_static_smem_bins(policy, counter_size, num_active_channels); } template @@ -195,16 +216,15 @@ should_use_dynamic_smem(const HistogramPolicy& policy, int num_bins, int counter // Multi-channel paths use explicit per-channel caps in addition to the byte // budget because their channel-interleaved launch shapes have distinct // measured crossover points. - if (policy.dynamic_smem.max_bytes <= 0 || num_bins <= 0 || counter_size <= 0 || num_active_channels <= 0) + if (num_bins <= 0) { return false; } - const bool prefer_dynamic_smem = - counter_size > int{sizeof(unsigned int)} || !should_use_static_smem(policy, num_bins); - const size_t required_bytes = size_t(num_bins) * size_t(num_active_channels) * size_t(counter_size); + const bool prefer_dynamic_smem = counter_size > int{sizeof(unsigned int)} + || !should_use_static_smem(policy, num_bins, counter_size, num_active_channels); - int max_bins = policy.dynamic_smem.max_bins; + int max_bins = max_privatized_dynamic_smem_bins(policy, counter_size, num_active_channels); if (num_active_channels > 1) { if constexpr (IsEven) @@ -220,8 +240,7 @@ should_use_dynamic_smem(const HistogramPolicy& policy, int num_bins, int counter } } - return prefer_dynamic_smem && max_bins > 0 && num_bins <= max_bins - && required_bytes <= static_cast(policy.dynamic_smem.max_bytes); + return prefer_dynamic_smem && max_bins > 0 && num_bins <= max_bins; } // TODO(bgruber): drop in CCCL 4.0 @@ -391,11 +410,10 @@ struct policy_hub struct Policy500 : detail::chained_policy<500, Policy500, Policy500> { // TODO This might be worth it to separate usual histogram and the multi one - using AgentHistogramPolicyT = - agent_histogram_policy<384, t_scale(16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false>; - using GmemPolicy = AgentHistogramPolicyT; - using StaticSmemPolicy = static_smem_policy; - using DynamicSmemPolicy = dynamic_smem_policy; + using AgentHistogramPolicyT = agent_histogram_policy<384, t_scale(16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, false>; + using GmemPolicy = AgentHistogramPolicyT; + using StaticSmemPolicy = static_smem_policy; + using DynamicSmemPolicy = dynamic_smem_policy; static constexpr int init_kernel_pdl_trigger_max_bins = 0; }; @@ -411,7 +429,6 @@ struct policy_hub Tuning::load_algorithm, Tuning::load_modifier, Tuning::rle_compress, - SMEM, Tuning::work_stealing>; template @@ -422,8 +439,8 @@ struct policy_hub sm90_tuning()>>(0)); using GmemPolicy = AgentHistogramPolicyT; - using StaticSmemPolicy = static_smem_policy; - using DynamicSmemPolicy = dynamic_smem_policy; + using StaticSmemPolicy = static_smem_policy; + using DynamicSmemPolicy = dynamic_smem_policy; static constexpr int init_kernel_pdl_trigger_max_bins = NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value @@ -436,15 +453,14 @@ struct policy_hub { // Use values from tuning if a specialization exists, otherwise pick Policy900 template - _CCCL_HOST_DEVICE_API static auto select_agent_policy(int) -> agent_histogram_policy< - Tuning::threads_per_block, - Tuning::items_per_thread, - Tuning::load_algorithm, - Tuning::load_modifier, - Tuning::rle_compress, - SMEM, - Tuning::work_stealing, - Tuning::vec_size>; + _CCCL_HOST_DEVICE_API static auto select_agent_policy(int) + -> agent_histogram_policy; template _CCCL_HOST_DEVICE_API static auto select_agent_policy(long) -> typename Policy900::AgentHistogramPolicyT; @@ -455,7 +471,7 @@ struct policy_hub 0)); using MultiChannelAgentHistogramPolicyT = - agent_histogram_policy<1024, t_scale(IsEven ? 8 : 16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false, 4>; + agent_histogram_policy<1024, t_scale(IsEven ? 8 : 16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, false, 4>; static constexpr bool use_sm100_multi_channel_policy = NumChannels >= 2 && sizeof(CounterT) == 4 && is_primitive::value; @@ -499,17 +515,17 @@ struct policy_hub AgentHistogramPolicyT::LOAD_ALGORITHM, AgentHistogramPolicyT::LOAD_MODIFIER, AgentHistogramPolicyT::IS_RLE_COMPRESS, - AgentHistogramPolicyT::MEM_PREFERENCE, AgentHistogramPolicyT::IS_WORK_STEALING, - AgentHistogramPolicyT::VEC_SIZE>; + AgentHistogramPolicyT::VEC_SIZE, + 513 * sizeof(CounterT) * NumActiveChannels>; - using GmemPolicy = AgentHistogramPolicyT; - using StaticSmemPolicy = static_smem_policy; + using GmemPolicy = AgentHistogramPolicyT; + using StaticSmemPolicy = + static_smem_policy; static constexpr int dynamic_smem_max_bytes = has_dynamic_smem_tuning ? 232448 - 4096 : 0; using DynamicSmemPolicy = dynamic_smem_policy StaticSweepPolicy::LOAD_ALGORITHM, StaticSweepPolicy::LOAD_MODIFIER, StaticSweepPolicy::IS_RLE_COMPRESS, - StaticSweepPolicy::MEM_PREFERENCE, StaticSweepPolicy::IS_WORK_STEALING}; } @@ -545,10 +560,11 @@ template using dynamic_smem = typename ActivePolicy::DynamicSmemPolicy; return { convert_sweep_policy(), - {convert_sweep_policy(), static_smem::MAX_BINS, static_smem::MIN_BLOCKS_PER_SM}, + {convert_sweep_policy(), + static_smem::MAX_PRIVATIZED_SMEM_BYTES, + static_smem::MIN_BLOCKS_PER_SM}, {convert_sweep_policy(), - dynamic_smem::MAX_BINS, - dynamic_smem::MAX_BYTES, + dynamic_smem::MAX_PRIVATIZED_SMEM_BYTES, dynamic_smem::RANGE_MAX_BINS, dynamic_smem::EVEN_2CH_MAX_BINS, dynamic_smem::EVEN_3CH_MAX_BINS, diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index 02bf5414c70a..706282600689 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1627,8 +1627,8 @@ struct histogram_tuning _CCCL_HOST_DEVICE_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { constexpr auto sweep = - cub::HistogramSweepPolicy{BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, cub::SMEM, false}; - return {sweep, {sweep, 256, 0}, {sweep, 0, 0, 0, 0, 0, 0}, 0}; + cub::HistogramSweepPolicy{BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; + return {sweep, {sweep, 257 * sizeof(unsigned int), 0}, {sweep, 0, 0, 0, 0, 0}, 0}; } }; @@ -1638,7 +1638,6 @@ struct histogram_tuning_with_local_counter : histogram_tuning using local_counter_type = LocalCounterT; }; -template struct mixed_counter_histogram_tuning { using local_counter_type = unsigned int; @@ -1646,8 +1645,8 @@ struct mixed_counter_histogram_tuning _CCCL_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { constexpr auto sweep = - cub::HistogramSweepPolicy{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, MemoryPreference, false}; - return {sweep, {sweep, 512, 0}, {sweep, 57088, 228352, 2048, 28544, 19029, 8192}, 0}; + cub::HistogramSweepPolicy{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; + return {sweep, {sweep, 513 * sizeof(unsigned int), 0}, {sweep, 228352, 2048, 28544, 19029, 8192}, 0}; } }; @@ -1673,7 +1672,7 @@ C2H_TEST("DeviceHistogram supports narrower local counters than output counters" constexpr int num_samples = 4096; constexpr int num_levels = num_samples + 1; auto d_histogram = c2h::device_vector(num_samples, 0); - auto env = cuda::execution::tune(mixed_counter_histogram_tuning<>{}); + auto env = cuda::execution::tune(mixed_counter_histogram_tuning{}); histogram_even( cuda::counting_iterator(0), @@ -1687,43 +1686,6 @@ C2H_TEST("DeviceHistogram supports narrower local counters than output counters" REQUIRE(d_histogram == c2h::host_vector(num_samples, 1)); } -C2H_TEST("Dynamic shared-memory histograms respect memory preference", "[histogram][device]") -{ - int current_device{}; - REQUIRE(cudaSuccess == cudaGetDevice(¤t_device)); - - cuda::compute_capability cc{}; - REQUIRE(cudaSuccess == cub::detail::ptx_compute_cap(cc, current_device)); - if (cc < cuda::compute_capability{10, 0}) - { - SKIP("The runtime-sized shared-memory histogram policy is currently tuned for SM100"); - } - - constexpr int num_samples = 16384; - constexpr int num_levels = 4097; - - const auto check_preference = [](auto preference) { - constexpr auto memory_preference = decltype(preference)::value; - auto d_histogram = c2h::device_vector(num_levels - 1, 0); - auto env = cuda::execution::tune(mixed_counter_histogram_tuning{}); - - histogram_even( - cuda::counting_iterator(0), - thrust::raw_pointer_cast(d_histogram.data()), - num_levels, - 0u, - static_cast(num_levels - 1), - num_samples, - env); - - REQUIRE(d_histogram == c2h::host_vector(num_levels - 1, 1)); - }; - - check_preference(cuda::std::integral_constant{}); - check_preference(cuda::std::integral_constant{}); - check_preference(cuda::std::integral_constant{}); -} - using block_sizes = c2h::type_list, cuda::std::integral_constant>; @@ -1844,54 +1806,43 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) // aggregate init constexpr auto p1 = cub::HistogramPolicy{ - {128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, cub::SMEM, false}, - {{96, 3, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, cub::SMEM, false}, 512, 2}, - {{128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, cub::SMEM, false}, - 12345, - 12345, - 1024, - 4096, - 8192, - 16384}, + {128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, + {{96, 3, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, 2052, 2}, + {{128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, 12345, 1024, 4096, 8192, 16384}, 2048}; # if _CCCL_STD_VER >= 2020 // designated init constexpr auto p2 = cub::HistogramPolicy{ - .gmem = {.threads_per_block = 128, - .items_per_thread = 7, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .mem_preference = cub::SMEM, - .work_stealing = false}, - .static_smem = - {.sweep = {.threads_per_block = 96, - .items_per_thread = 3, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .mem_preference = cub::SMEM, - .work_stealing = false}, - .max_bins = 512, - .min_blocks_per_sm = 2}, + .gmem = {.threads_per_block = 128, + .items_per_thread = 7, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .static_smem = {.sweep = {.threads_per_block = 96, + .items_per_thread = 3, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .max_privatized_smem_bytes = 2052, + .min_blocks_per_sm = 2}, .dynamic_smem = - {.sweep = {.threads_per_block = 128, - .items_per_thread = 7, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .mem_preference = cub::SMEM, - .work_stealing = false}, - .max_bins = 12345, - .max_bytes = 12345, - .range_max_bins = 1024, - .even_2ch_max_bins = 4096, - .even_3ch_max_bins = 8192, - .even_4ch_max_bins = 16384}, + {.sweep = {.threads_per_block = 128, + .items_per_thread = 7, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .max_privatized_smem_bytes = 12345, + .range_max_bins = 1024, + .even_2ch_max_bins = 4096, + .even_3ch_max_bins = 8192, + .even_4ch_max_bins = 16384}, .init_kernel_pdl_trigger_max_bins = 2048}; # else // _CCCL_STD_VER >= 2020 constexpr auto p2 = p1; @@ -1910,14 +1861,14 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) to_string(p1) == "HistogramPolicy { .gmem = { .threads_per_block = 128, .items_per_thread = 7, .vec_size = 4" ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" - ", .mem_preference = SMEM, .work_stealing = 0 }" + ", .work_stealing = 0 }" ", .static_smem = { .sweep = { .threads_per_block = 96, .items_per_thread = 3, .vec_size = 4" ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" - ", .mem_preference = SMEM, .work_stealing = 0 }" - ", .max_bins = 512, .min_blocks_per_sm = 2 }, .dynamic_smem = { .sweep = { .threads_per_block = 128" + ", .work_stealing = 0 }" + ", .max_privatized_smem_bytes = 2052, .min_blocks_per_sm = 2 }, .dynamic_smem = { .sweep = { .threads_per_block " + "= 128" ", .items_per_thread = 7, .vec_size = 4, .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG" - ", .rle_compress = 0, .mem_preference = SMEM, .work_stealing = 0 }, .max_bins = 12345" - ", .max_bytes = 12345" + ", .rle_compress = 0, .work_stealing = 0 }, .max_privatized_smem_bytes = 12345" ", .range_max_bins = 1024, .even_2ch_max_bins = 4096, .even_3ch_max_bins = 8192" ", .even_4ch_max_bins = 16384 }, .init_kernel_pdl_trigger_max_bins = 2048 }"); } @@ -1932,11 +1883,14 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); - STATIC_REQUIRE(sm90_policy.static_smem.max_bins == 256); - STATIC_REQUIRE(sm100_policy.static_smem.max_bins == 512); - STATIC_REQUIRE(sm90_policy.dynamic_smem.max_bytes == 0); - STATIC_REQUIRE(sm100_policy.dynamic_smem.max_bytes == 228352); - STATIC_REQUIRE(sm100_wide_counter_policy.dynamic_smem.max_bytes == 0); + STATIC_REQUIRE(cub::detail::histogram::max_privatized_static_smem_bins(sm90_policy, 4, 1) == 256); + STATIC_REQUIRE(cub::detail::histogram::max_privatized_static_smem_bins(sm100_policy, 4, 1) == 512); + STATIC_REQUIRE(cub::detail::histogram::max_privatized_static_smem_bins(sm100_policy, 4, 4) == 127); + STATIC_REQUIRE(sm90_policy.dynamic_smem.max_privatized_smem_bytes == 0); + STATIC_REQUIRE(sm100_policy.dynamic_smem.max_privatized_smem_bytes == 228352); + STATIC_REQUIRE(sm100_wide_counter_policy.dynamic_smem.max_privatized_smem_bytes == 0); + STATIC_REQUIRE(cub::detail::histogram::max_privatized_dynamic_smem_bins(sm100_policy, 4, 1) == 57088); + STATIC_REQUIRE(cub::detail::histogram::max_privatized_dynamic_smem_bins(sm100_policy, 4, 4) == 14272); STATIC_REQUIRE(sm100_policy.dynamic_smem.range_max_bins == 2048); STATIC_REQUIRE(sm100_policy.dynamic_smem.even_2ch_max_bins == 28544); STATIC_REQUIRE(sm100_policy.dynamic_smem.even_3ch_max_bins == 19029); diff --git a/cub/test/catch2_test_device_histogram_env_api.cu b/cub/test/catch2_test_device_histogram_env_api.cu index 91d1009da665..c083a0fab8a6 100644 --- a/cub/test/catch2_test_device_histogram_env_api.cu +++ b/cub/test/catch2_test_device_histogram_env_api.cu @@ -419,18 +419,17 @@ struct HistogramPolicySelector __host__ __device__ constexpr auto operator()(cuda::compute_capability cc) const -> cub::HistogramPolicy { const auto sweep = cub::HistogramSweepPolicy{ - 128, - cc > cuda::compute_capability{9, 0} ? 16 : 7, - 4, - cub::BLOCK_LOAD_DIRECT, - cub::LOAD_LDG, - false, - cub::SMEM, - false}; - return {.gmem = sweep, - .static_smem = {.sweep = sweep, .max_bins = 256, .min_blocks_per_sm = 0}, - .dynamic_smem = {.sweep = sweep}, - .init_kernel_pdl_trigger_max_bins = 2048}; + 128, cc > cuda::compute_capability{9, 0} ? 16 : 7, 4, cub::BLOCK_LOAD_DIRECT, cub::LOAD_LDG, false, false}; + return { + .gmem = sweep, + .static_smem = {.sweep = sweep, .max_privatized_smem_bytes = 257 * sizeof(unsigned int), .min_blocks_per_sm = 0}, + .dynamic_smem = {.sweep = sweep, + .max_privatized_smem_bytes = 0, + .range_max_bins = 0, + .even_2ch_max_bins = 0, + .even_3ch_max_bins = 0, + .even_4ch_max_bins = 0}, + .init_kernel_pdl_trigger_max_bins = 2048}; } }; // example-end histogram-even-policy-selector From 30c45115cd0fb139903dd891f048bf0faae3e906 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Wed, 5 Aug 2026 07:48:44 +0000 Subject: [PATCH 25/45] [cub] Address histogram privatization review feedback --- .../bench/histogram/histogram_common.cuh | 2 +- cub/cub/agent/agent_histogram.cuh | 248 ++++++------------ .../device/dispatch/dispatch_histogram.cuh | 12 +- .../dispatch/kernels/kernel_histogram.cuh | 34 +-- .../dispatch/tuning/tuning_histogram.cuh | 145 +++++----- ...test_device_histogram_custom_policy_hub.cu | 2 +- cub/test/catch2_test_device_histogram_env.cu | 19 +- .../catch2_test_device_histogram_env_api.cu | 18 +- cub/test/catch2_test_enum_formatting.cu | 9 - 9 files changed, 209 insertions(+), 280 deletions(-) diff --git a/cub/benchmarks/bench/histogram/histogram_common.cuh b/cub/benchmarks/bench/histogram/histogram_common.cuh index 1f45cd84ba8a..55c67f27688c 100644 --- a/cub/benchmarks/bench/histogram/histogram_common.cuh +++ b/cub/benchmarks/bench/histogram/histogram_common.cuh @@ -37,7 +37,7 @@ struct bench_policy_selector ? (NUM_CHANNELS == 1 ? cub::BLOCK_LOAD_STRIPED : cub::BLOCK_LOAD_DIRECT) : TUNE_LOAD_ALGORITHM; - constexpr auto sweep = cub::HistogramSweepPolicy{ + constexpr auto sweep = cub::HistogramKernelConfig{ TUNE_THREADS, TUNE_ITEMS, TUNE_VEC_SIZE, diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index 73b0319a1821..27793e0857b7 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -23,9 +23,6 @@ #include #include -#include -#include -#include #include #include #include @@ -33,53 +30,6 @@ CUB_NAMESPACE_BEGIN -enum BlockHistogramMemoryPreference -{ - GMEM, - SMEM, - BLEND -}; - -#if _CCCL_HOSTED() -namespace detail -{ -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const char* to_string(BlockHistogramMemoryPreference mempref) noexcept -{ - switch (mempref) - { - case GMEM: - return "GMEM"; - case SMEM: - return "SMEM"; - case BLEND: - return "BLEND"; - } - return ""; -} -} // namespace detail - -inline ::std::ostream& operator<<(::std::ostream& os, BlockHistogramMemoryPreference mempref) -{ - return os << CUB_NS_QUALIFIER::detail::to_string(mempref); -} -#endif // _CCCL_HOSTED() - -CUB_NAMESPACE_END - -#if __cpp_lib_format >= 201907L && !defined(_CCCL_DOXYGEN_INVOKED) -template <::cuda::std::same_as CharT> -struct std::formatter : formatter -{ - template - auto format(const CUB_NS_QUALIFIER::BlockHistogramMemoryPreference& mempref, FmtCtx& ctx) const - { - return formatter::format(CUB_NS_QUALIFIER::detail::to_string(mempref), ctx); - } -}; -#endif // __cpp_lib_format >= 201907L && !defined(_CCCL_DOXYGEN_INVOKED) - -CUB_NAMESPACE_BEGIN - namespace detail { //! Parameterizable tuning policy type for AgentHistogram @@ -89,8 +39,8 @@ template + int VecSize = 4, + int PrivatizedStaticSmemBytes = 0> struct agent_histogram_policy { /// Threads per thread block @@ -105,7 +55,7 @@ struct agent_histogram_policy static constexpr bool IS_WORK_STEALING = WorkStealing; /// Maximum compile-time-sized shared-memory allocation for privatized bins - static constexpr int PRIVATIZED_SMEM_MAX_BYTES = PrivatizedSmemMaxBytes; + static constexpr int PRIVATIZED_STATIC_SMEM_BYTES = PrivatizedStaticSmemBytes; /// Vector size for samples loading (1, 2, 4) static constexpr int VEC_SIZE = VecSize; @@ -125,20 +75,10 @@ template -struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") AgentHistogramPolicy - : detail::agent_histogram_policy -{ - static constexpr BlockHistogramMemoryPreference MEM_PREFERENCE = MemoryPreference; -}; +using AgentHistogramPolicy CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") = detail:: + agent_histogram_policy; namespace detail::histogram { @@ -182,9 +122,8 @@ _CCCL_DEVICE _CCCL_FORCEINLINE auto NativePointer(IteratorT itr) //! @tparam AgentHistogramPolicyT //! Parameterized AgentHistogramPolicy tuning policy type //! -//! @tparam PrivatizedSmemBins -//! Number of privatized shared-memory histogram bins of any channel. Zero indicates privatized -//! counters to be maintained in device-accessible memory. +//! @tparam PrivatizationMode +//! Storage mode for the privatized histogram. //! //! @tparam NumChannels //! Number of channels interleaved in the input data. Supports up to four channels. @@ -209,9 +148,6 @@ _CCCL_DEVICE _CCCL_FORCEINLINE auto NativePointer(IteratorT itr) //! @tparam OffsetT //! Signed integer type for global offsets //! -//! @tparam PrivatizationMode -//! Storage mode for the privatized histogram. -//! //! @tparam OutputCounterT //! Integer type for final output histogram bins. May be wider than `CounterT`. template ; static constexpr bool uses_dynamic_smem = is_privatized_dynamic_smem_v; static constexpr bool uses_gmem = is_privatized_gmem_v; - static constexpr int static_smem_slots_per_channel = - uses_static_smem ? AgentHistogramPolicyT::PRIVATIZED_SMEM_MAX_BYTES / int{sizeof(CounterT)} / NumActiveChannels : 1; - static_assert(!uses_static_smem || static_smem_slots_per_channel > 1, - "Static-SMEM privatization requires room for at least one bin and its padding counter"); - static constexpr int privatized_smem_bins = uses_static_smem ? static_smem_slots_per_channel - 1 : 0; + static constexpr int privatized_static_smem_bins = + uses_static_smem ? AgentHistogramPolicyT::PRIVATIZED_STATIC_SMEM_BYTES / int{sizeof(CounterT)} / NumActiveChannels + : 0; + static_assert(!uses_static_smem || privatized_static_smem_bins > 0, + "Static-SMEM privatization requires room for at least one bin"); static constexpr int vec_size = AgentHistogramPolicyT::VEC_SIZE; static constexpr int threads_per_block = AgentHistogramPolicyT::BLOCK_THREADS; static constexpr int pixels_per_thread = AgentHistogramPolicyT::PIXELS_PER_THREAD; @@ -270,7 +206,11 @@ struct AgentHistogram struct _TempStorage { - CounterT histograms[NumActiveChannels][privatized_smem_bins + 1]; + // The one-element fallback keeps this type well-formed for modes that do not + // use the compile-time-sized histogram. Static-SMEM mode uses exactly the + // configured number of bins; out-of-range samples are rejected before the + // atomic update and therefore require no padding bin. + CounterT privatized_histogram[NumActiveChannels][privatized_static_smem_bins > 0 ? privatized_static_smem_bins : 1]; int tile_idx; union @@ -283,14 +223,14 @@ struct AgentHistogram using TempStorage = Uninitialized<_TempStorage>; - _TempStorage& temp_storage; + _TempStorage& static_smem_storage; WrappedSampleIteratorT d_wrapped_samples; // with cache modifier applied, if possible SampleT* d_native_samples; // possibly nullptr if unavailable const int* num_output_bins; // one for each channel const int* num_privatized_bins; // one for each channel - CounterT* d_privatized_histograms[NumActiveChannels]; // one for each channel - CounterT* dyn_smem_histograms[NumActiveChannels]; // dynamic shared-memory channel bases, when enabled - OutputCounterT** d_output_histograms; // final output, in global memory + CounterT* gmem_privatized_histograms[NumActiveChannels]; // one for each channel + CounterT* dyn_smem_privatized_histograms[NumActiveChannels]; // dynamic shared-memory channel bases, when enabled + OutputCounterT** output_histogram; // final output, in global memory const OutputDecodeOpT* output_decode_op; // determines output bin-id from privatized counter index, one for each // channel PrivatizedDecodeOpT* privatized_decode_op; // determines privatized counter index from sample, one for each channel @@ -298,15 +238,15 @@ struct AgentHistogram { if constexpr (uses_dynamic_smem) { - return dyn_smem_histograms[channel]; + return dyn_smem_privatized_histograms[channel]; } else if constexpr (uses_static_smem) { - return temp_storage.histograms[channel]; + return static_smem_storage.privatized_histogram[channel]; } else { - return d_privatized_histograms[channel]; + return gmem_privatized_histograms[channel]; } } @@ -326,33 +266,6 @@ struct AgentHistogram __syncthreads(); } - // Update final output histograms from privatized histograms - _CCCL_DEVICE _CCCL_FORCEINLINE void StoreOutputImpl() - { - // Barrier to make sure all threads are done updating counters - __syncthreads(); - - // Apply privatized bin counts to output bin counts - _CCCL_PRAGMA_UNROLL_FULL() - for (int ch = 0; ch < NumActiveChannels; ++ch) - { - CounterT* privatized_histogram = PrivatizedHistogram(ch); - const int channel_bins = num_privatized_bins[ch]; - for (int bin = static_cast(threadIdx.x); bin < channel_bins; bin += threads_per_block) - { - int output_bin = -1; - const CounterT count = privatized_histogram[bin]; - const bool is_valid = count > 0; - output_decode_op[ch].template BinSelect(static_cast(bin), output_bin, is_valid); - - if (output_bin >= 0) - { - atomicAdd(&d_output_histograms[ch][output_bin], static_cast(count)); - } - } - } - } - // Accumulate pixels. Specialized for RLE compression. _CCCL_DEVICE _CCCL_FORCEINLINE void AccumulatePixels( SampleT samples[pixels_per_thread][NumChannels], @@ -430,14 +343,14 @@ struct AgentHistogram using AliasedVecs = VecT[vecs_per_thread]; WrappedVecsIteratorT d_wrapped_vecs(reinterpret_cast(d_native_samples + block_offset)); // Load using a wrapped vec iterator - BlockLoadVecT{temp_storage.vec_load}.Load(d_wrapped_vecs, reinterpret_cast(samples)); + BlockLoadVecT{static_smem_storage.vec_load}.Load(d_wrapped_vecs, reinterpret_cast(samples)); } else { using AliasedPixels = PixelT[pixels_per_thread]; WrappedPixelIteratorT d_wrapped_pixels(reinterpret_cast(d_native_samples + block_offset)); // Load using a wrapped pixel iterator - BlockLoadPixelT{temp_storage.pixel_load}.Load(d_wrapped_pixels, reinterpret_cast(samples)); + BlockLoadPixelT{static_smem_storage.pixel_load}.Load(d_wrapped_pixels, reinterpret_cast(samples)); } } @@ -455,7 +368,7 @@ struct AgentHistogram { // Load using sample iterator using AliasedSamples = SampleT[samples_per_thread]; - BlockLoadSampleT{temp_storage.sample_load}.Load( + BlockLoadSampleT{static_smem_storage.sample_load}.Load( d_wrapped_samples + block_offset, reinterpret_cast(samples)); } } @@ -469,13 +382,13 @@ struct AgentHistogram int valid_pixels = valid_samples / NumChannels; // Load using a wrapped pixel iterator - BlockLoadPixelT{temp_storage.pixel_load}.Load( + BlockLoadPixelT{static_smem_storage.pixel_load}.Load( d_wrapped_pixels, reinterpret_cast(samples), valid_pixels); } else { using AliasedSamples = SampleT[samples_per_thread]; - BlockLoadSampleT{temp_storage.sample_load}.Load( + BlockLoadSampleT{static_smem_storage.sample_load}.Load( d_wrapped_samples + block_offset, reinterpret_cast(samples), valid_samples); } } @@ -568,12 +481,12 @@ struct AgentHistogram // Get next tile if (threadIdx.x == 0) { - temp_storage.tile_idx = tile_queue.Drain(1) + num_even_share_tiles; + static_smem_storage.tile_idx = tile_queue.Drain(1) + num_even_share_tiles; } __syncthreads(); - tile_idx = temp_storage.tile_idx; + tile_idx = static_smem_storage.tile_idx; } } @@ -621,8 +534,8 @@ struct AgentHistogram //! @brief Constructor //! - //! @param temp_storage - //! Reference to temp_storage + //! @param static_smem_storage + //! Shared storage used by the agent //! //! @param d_samples //! Input data to reduce @@ -633,78 +546,58 @@ struct AgentHistogram //! @param num_privatized_bins //! The number bins per privatized histogram //! - //! @param d_output_histograms + //! @param output_histogram //! Reference to final output histograms //! - //! @param d_privatized_histograms - //! Reference to privatized histograms + //! @param gmem_privatized_histograms + //! Global-memory privatized histograms, or `nullptr` entries for a shared-memory mode //! //! @param output_decode_op //! The transform operator for determining output bin-ids from privatized counter indices, one for each channel //! //! @param privatized_decode_op //! The transform operator for determining privatized counter indices from samples, one for each channel + //! + //! @param dyn_smem_privatized_histograms + //! Base of the runtime-sized shared-memory histogram, or `nullptr` for a static-SMEM or global-memory mode _CCCL_DEVICE _CCCL_FORCEINLINE AgentHistogram( - TempStorage& temp_storage, + TempStorage& static_smem_storage, SampleIteratorT d_samples, const int* num_output_bins, const int* num_privatized_bins, - OutputCounterT** d_output_histograms, - CounterT** d_privatized_histograms, + OutputCounterT** output_histogram, + CounterT** gmem_privatized_histograms, const OutputDecodeOpT* output_decode_op, - PrivatizedDecodeOpT* privatized_decode_op) - : temp_storage(temp_storage.Alias()) + PrivatizedDecodeOpT* privatized_decode_op, + CounterT* dyn_smem_privatized_histograms) + : static_smem_storage(static_smem_storage.Alias()) , d_wrapped_samples(d_samples) , d_native_samples(NativePointer(d_wrapped_samples)) , num_output_bins(num_output_bins) , num_privatized_bins(num_privatized_bins) - , d_output_histograms(d_output_histograms) + , output_histogram(output_histogram) , output_decode_op(output_decode_op) , privatized_decode_op(privatized_decode_op) { - static_assert(!uses_dynamic_smem, - "AgentHistogram with dynamic-SMEM privatization requires the dynamic-SMEM " - "constructor that takes an extern __shared__ base pointer."); - - if constexpr (uses_gmem) + if constexpr (uses_dynamic_smem) { - const int block_id = static_cast((blockIdx.y * gridDim.x) + blockIdx.x); + _CCCL_ASSERT(dyn_smem_privatized_histograms != nullptr, "Dynamic-SMEM mode requires a shared-memory base"); + CounterT* channel_histogram = dyn_smem_privatized_histograms; + _CCCL_PRAGMA_UNROLL_FULL() for (int ch = 0; ch < NumActiveChannels; ++ch) { - const auto offset = static_cast<::cuda::std::int64_t>(block_id) * num_privatized_bins[ch]; - this->d_privatized_histograms[ch] = d_privatized_histograms[ch] + offset; + this->dyn_smem_privatized_histograms[ch] = channel_histogram; + channel_histogram += num_privatized_bins[ch]; } } - } - - //! @brief Constructor for a histogram stored in dynamic shared memory - _CCCL_DEVICE _CCCL_FORCEINLINE AgentHistogram( - TempStorage& temp_storage, - SampleIteratorT d_samples, - const int* num_output_bins, - const int* num_privatized_bins, - OutputCounterT** d_output_histograms, - CounterT** d_privatized_histograms, - const OutputDecodeOpT* output_decode_op, - PrivatizedDecodeOpT* privatized_decode_op, - CounterT* dyn_smem_histogram_base) - : temp_storage(temp_storage.Alias()) - , d_wrapped_samples(d_samples) - , d_native_samples(NativePointer(d_wrapped_samples)) - , num_output_bins(num_output_bins) - , num_privatized_bins(num_privatized_bins) - , d_output_histograms(d_output_histograms) - , output_decode_op(output_decode_op) - , privatized_decode_op(privatized_decode_op) - { - static_assert(uses_dynamic_smem, "Dynamic-SMEM AgentHistogram constructor requires dynamic-SMEM mode."); - - CounterT* p = dyn_smem_histogram_base; - _CCCL_PRAGMA_UNROLL_FULL() - for (int ch = 0; ch < NumActiveChannels; ++ch) + else if constexpr (uses_gmem) { - this->dyn_smem_histograms[ch] = p; - p += num_privatized_bins[ch]; + const int block_id = static_cast((blockIdx.y * gridDim.x) + blockIdx.x); + for (int ch = 0; ch < NumActiveChannels; ++ch) + { + const auto offset = static_cast<::cuda::std::int64_t>(block_id) * num_privatized_bins[ch]; + this->gmem_privatized_histograms[ch] = gmem_privatized_histograms[ch] + offset; + } } } @@ -769,7 +662,28 @@ struct AgentHistogram //! Store privatized histogram to device-accessible memory. Specialized for privatized shared-memory counters _CCCL_DEVICE _CCCL_FORCEINLINE void StoreOutput() { - StoreOutputImpl(); + // Barrier to make sure all threads are done updating counters + __syncthreads(); + + // Apply privatized bin counts to output bin counts + _CCCL_PRAGMA_UNROLL_FULL() + for (int ch = 0; ch < NumActiveChannels; ++ch) + { + CounterT* privatized_histogram = PrivatizedHistogram(ch); + const int channel_bins = num_privatized_bins[ch]; + for (int bin = static_cast(threadIdx.x); bin < channel_bins; bin += threads_per_block) + { + int output_bin = -1; + const CounterT count = privatized_histogram[bin]; + const bool is_valid = count > 0; + output_decode_op[ch].template BinSelect(static_cast(bin), output_bin, is_valid); + + if (output_bin >= 0) + { + atomicAdd(&output_histogram[ch][output_bin], static_cast(count)); + } + } + } } }; } // namespace detail::histogram diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 1c4c39da8598..f33dd22a7a84 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -281,9 +281,9 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( : is_privatized_static_smem_v ? privatization_tier::static_smem : privatization_tier::gmem; - const HistogramSweepPolicy sweep = sweep_policy(active_policy); - const int threads_per_block = sweep.threads_per_block; - const int items_per_thread = sweep.items_per_thread; + const HistogramKernelConfig sweep = kernel_config(active_policy); + const int threads_per_block = sweep.threads_per_block; + const int items_per_thread = sweep.items_per_thread; int dynamic_smem_bytes = 0; if constexpr (is_privatized_dynamic_smem_v) @@ -805,8 +805,8 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_policy(int) template _CCCL_HOST_DEVICE_API constexpr auto convert_policy(long) -> HistogramPolicy { - using sweep = typename ActivePolicy::AgentHistogramPolicyT; - const auto sweep_policy = HistogramSweepPolicy{ + using sweep = typename ActivePolicy::AgentHistogramPolicyT; + const auto kernel_config = HistogramKernelConfig{ sweep::BLOCK_THREADS, sweep::PIXELS_PER_THREAD, sweep::VEC_SIZE, @@ -814,7 +814,7 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_policy(long) -> HistogramPolicy sweep::LOAD_MODIFIER, sweep::IS_RLE_COMPRESS, sweep::IS_WORK_STEALING}; - return {sweep_policy, {sweep_policy, 257 * sizeof(unsigned int), 0}, {sweep_policy, 0, 0, 0, 0, 0}, 0}; + return {kernel_config, {kernel_config, 256 * sizeof(unsigned int), 0}, {kernel_config, 0, 0, 0, 0, 0}, 0}; } // TODO(bgruber): drop in CCCL 4.0 diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index 8d519e0df586..25a4222bd9dc 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -705,7 +705,7 @@ template #endif // _CCCL_HAS_CONCEPTS() __launch_bounds__( - int(sweep_policy< + int(kernel_config< is_privatized_static_smem_v ? privatization_tier::static_smem : privatization_tier::gmem>( current_policy()) .threads_per_block), @@ -727,7 +727,7 @@ __launch_bounds__( GridQueue tile_queue) { static constexpr HistogramPolicy hp = current_policy(); - static constexpr auto sweep = sweep_policy< + static constexpr auto sweep = kernel_config< is_privatized_static_smem_v ? privatization_tier::static_smem : privatization_tier::gmem>(hp); // Thread block type for compositing input tiles @@ -753,17 +753,18 @@ __launch_bounds__( OutputCounterT>; // Shared memory for AgentHistogram - __shared__ typename AgentHistogramT::TempStorage temp_storage; + __shared__ typename AgentHistogramT::TempStorage static_smem_storage; AgentHistogramT agent( - temp_storage, + static_smem_storage, d_samples, num_output_bins_wrapper.data(), num_privatized_bins_wrapper.data(), d_output_histograms_wrapper.data(), d_privatized_histograms_wrapper.data(), output_decode_op_wrapper.data(), - privatized_decode_op_wrapper.data()); + privatized_decode_op_wrapper.data(), + nullptr); // Initialize counters agent.InitBinCounters(); @@ -794,7 +795,7 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(current_policy().dynamic_smem.sweep.threads_per_block)) +__launch_bounds__(int(current_policy().dynamic_smem.kernel.threads_per_block)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDynamicSmemKernel( const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, @@ -809,8 +810,8 @@ __launch_bounds__(int(current_policy().dynamic_smem.sweep.thread const int tiles_per_row, GridQueue tile_queue) { - static constexpr HistogramPolicy hp = current_policy(); - static constexpr HistogramSweepPolicy sweep = hp.dynamic_smem.sweep; + static constexpr HistogramPolicy hp = current_policy(); + static constexpr HistogramKernelConfig sweep = hp.dynamic_smem.kernel; using AgentHistogramPolicyT = agent_histogram_policy().dynamic_smem.sweep.thread OffsetT, OutputCounterT>; - __shared__ typename AgentHistogramT::TempStorage temp_storage; + __shared__ typename AgentHistogramT::TempStorage static_smem_storage; extern __shared__ __align__(16) unsigned char dynamic_smem[]; OutputDecodeOpT output_decode_op[NumActiveChannels]; @@ -847,7 +848,7 @@ __launch_bounds__(int(current_policy().dynamic_smem.sweep.thread } AgentHistogramT agent( - temp_storage, + static_smem_storage, d_samples, num_output_bins_wrapper.data(), num_privatized_bins_wrapper.data(), @@ -962,8 +963,8 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(sweep_policy ? privatization_tier::static_smem - : privatization_tier::gmem>( +__launch_bounds__(int(kernel_config ? privatization_tier::static_smem + : privatization_tier::gmem>( current_policy()) .threads_per_block)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDeviceInitKernel( @@ -981,7 +982,7 @@ __launch_bounds__(int(sweep_policy tile_queue) { static constexpr HistogramPolicy hp = current_policy(); - static constexpr auto sweep = sweep_policy< + static constexpr auto sweep = kernel_config< is_privatized_static_smem_v ? privatization_tier::static_smem : privatization_tier::gmem>(hp); OutputDecodeOpT output_decode_op[NumActiveChannels]; @@ -1033,17 +1034,18 @@ __launch_bounds__(int(sweep_policy; // Shared memory for AgentHistogram - __shared__ typename AgentHistogramT::TempStorage temp_storage; + __shared__ typename AgentHistogramT::TempStorage static_smem_storage; AgentHistogramT agent( - temp_storage, + static_smem_storage, d_samples, num_output_bins_wrapper.data(), num_privatized_bins_wrapper.data(), d_output_histograms_wrapper.data(), d_privatized_histograms_wrapper.data(), output_decode_op, - privatized_decode_op); + privatized_decode_op, + nullptr); // Initialize counters agent.InitBinCounters(); diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 767d7c62c4cd..8e086862bdb9 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -25,8 +25,8 @@ CUB_NAMESPACE_BEGIN -//! Tuning policy for one DeviceHistogram sweep kernel. -struct HistogramSweepPolicy +//! Runtime launch configuration for one DeviceHistogram kernel. +struct HistogramKernelConfig { int threads_per_block; //!< Number of threads in a CUDA block int items_per_thread; //!< Number of items processed per thread @@ -37,7 +37,7 @@ struct HistogramSweepPolicy bool work_stealing; //!< Whether blocks dequeue tiles from a global queue [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool - operator==(const HistogramSweepPolicy& lhs, const HistogramSweepPolicy& rhs) noexcept + operator==(const HistogramKernelConfig& lhs, const HistogramKernelConfig& rhs) noexcept { return lhs.threads_per_block == rhs.threads_per_block && lhs.items_per_thread == rhs.items_per_thread && lhs.vec_size == rhs.vec_size && lhs.load_algorithm == rhs.load_algorithm @@ -49,14 +49,14 @@ struct HistogramSweepPolicy //! Tuning policy for the compile-time-sized shared-memory histogram kernel. struct HistogramStaticSmemPolicy { - HistogramSweepPolicy sweep; + HistogramKernelConfig kernel; int max_privatized_smem_bytes; //!< Maximum compile-time-sized shared-memory allocation int min_blocks_per_sm; //!< Minimum blocks per SM requested through launch bounds [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(const HistogramStaticSmemPolicy& lhs, const HistogramStaticSmemPolicy& rhs) noexcept { - return lhs.sweep == rhs.sweep && lhs.max_privatized_smem_bytes == rhs.max_privatized_smem_bytes + return lhs.kernel == rhs.kernel && lhs.max_privatized_smem_bytes == rhs.max_privatized_smem_bytes && lhs.min_blocks_per_sm == rhs.min_blocks_per_sm; } }; @@ -64,7 +64,7 @@ struct HistogramStaticSmemPolicy //! Tuning policy for the runtime-sized shared-memory histogram kernel. struct HistogramDynamicSmemPolicy { - HistogramSweepPolicy sweep; + HistogramKernelConfig kernel; int max_privatized_smem_bytes; //!< Maximum runtime-sized shared-memory allocation int range_max_bins; //!< Maximum bins per channel for multi-channel HistogramRange int even_2ch_max_bins; //!< Maximum bins per channel for two-channel HistogramEven @@ -74,7 +74,7 @@ struct HistogramDynamicSmemPolicy [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(const HistogramDynamicSmemPolicy& lhs, const HistogramDynamicSmemPolicy& rhs) noexcept { - return lhs.sweep == rhs.sweep && lhs.max_privatized_smem_bytes == rhs.max_privatized_smem_bytes + return lhs.kernel == rhs.kernel && lhs.max_privatized_smem_bytes == rhs.max_privatized_smem_bytes && lhs.range_max_bins == rhs.range_max_bins && lhs.even_2ch_max_bins == rhs.even_2ch_max_bins && lhs.even_3ch_max_bins == rhs.even_3ch_max_bins && lhs.even_4ch_max_bins == rhs.even_4ch_max_bins; } @@ -83,10 +83,10 @@ struct HistogramDynamicSmemPolicy //! The tuning policy for all DeviceHistogram kernel variants. struct HistogramPolicy { - HistogramSweepPolicy gmem; + HistogramKernelConfig gmem; HistogramStaticSmemPolicy static_smem; HistogramDynamicSmemPolicy dynamic_smem; - int init_kernel_pdl_trigger_max_bins; + int init_kernel_pdl_trigger_max_bins; //!< Common init-kernel PDL threshold, independent of accumulation tier [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(const HistogramPolicy& lhs, const HistogramPolicy& rhs) noexcept @@ -104,20 +104,20 @@ struct HistogramPolicy #if _CCCL_HOSTED() friend ::std::ostream& operator<<(::std::ostream& os, const HistogramPolicy& p) { - const auto print_sweep = [&](const HistogramSweepPolicy& sweep) -> ::std::ostream& { + const auto print_kernel = [&](const HistogramKernelConfig& kernel) -> ::std::ostream& { return os - << "{ .threads_per_block = " << sweep.threads_per_block << ", .items_per_thread = " << sweep.items_per_thread - << ", .vec_size = " << sweep.vec_size << ", .load_algorithm = " << sweep.load_algorithm - << ", .load_modifier = " << sweep.load_modifier << ", .rle_compress = " << sweep.rle_compress - << ", .work_stealing = " << sweep.work_stealing << " }"; + << "{ .threads_per_block = " << kernel.threads_per_block + << ", .items_per_thread = " << kernel.items_per_thread << ", .vec_size = " << kernel.vec_size + << ", .load_algorithm = " << kernel.load_algorithm << ", .load_modifier = " << kernel.load_modifier + << ", .rle_compress = " << kernel.rle_compress << ", .work_stealing = " << kernel.work_stealing << " }"; }; os << "HistogramPolicy { .gmem = "; - print_sweep(p.gmem); - os << ", .static_smem = { .sweep = "; - print_sweep(p.static_smem.sweep); + print_kernel(p.gmem); + os << ", .static_smem = { .kernel = "; + print_kernel(p.static_smem.kernel); os << ", .max_privatized_smem_bytes = " << p.static_smem.max_privatized_smem_bytes - << ", .min_blocks_per_sm = " << p.static_smem.min_blocks_per_sm << " }, .dynamic_smem = { .sweep = "; - print_sweep(p.dynamic_smem.sweep); + << ", .min_blocks_per_sm = " << p.static_smem.min_blocks_per_sm << " }, .dynamic_smem = { .kernel = "; + print_kernel(p.dynamic_smem.kernel); return os << ", .max_privatized_smem_bytes = " << p.dynamic_smem.max_privatized_smem_bytes << ", .range_max_bins = " << p.dynamic_smem.range_max_bins << ", .even_2ch_max_bins = " << p.dynamic_smem.even_2ch_max_bins @@ -130,23 +130,28 @@ struct HistogramPolicy namespace detail::histogram { -template -struct static_smem_policy +// Compile-time policy wrappers carried by each node in CUB's architecture +// policy chain. They pair an AgentHistogram launch policy with the storage +// limits needed by that kernel variant. `make_kernel_config` below converts +// these type-level values into the runtime `HistogramPolicy` selected by +// dispatch. +template +struct static_smem_chained_policy { - using SweepPolicyT = SweepPolicy; + using AgentPolicyT = AgentPolicy; static constexpr int MAX_PRIVATIZED_SMEM_BYTES = MaxPrivatizedSmemBytes; static constexpr int MIN_BLOCKS_PER_SM = MinBlocksPerSm; }; -template -struct dynamic_smem_policy +struct dynamic_smem_chained_policy { - using SweepPolicyT = SweepPolicy; + using AgentPolicyT = AgentPolicy; static constexpr int MAX_PRIVATIZED_SMEM_BYTES = MaxPrivatizedSmemBytes; static constexpr int RANGE_MAX_BINS = RangeMaxBins; static constexpr int EVEN_2CH_MAX_BINS = Even2chMaxBins; @@ -162,15 +167,15 @@ enum class privatization_tier }; template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const HistogramSweepPolicy& sweep_policy(const HistogramPolicy& policy) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const HistogramKernelConfig& kernel_config(const HistogramPolicy& policy) { if constexpr (Tier == privatization_tier::static_smem) { - return policy.static_smem.sweep; + return policy.static_smem.kernel; } else if constexpr (Tier == privatization_tier::dynamic_smem) { - return policy.dynamic_smem.sweep; + return policy.dynamic_smem.kernel; } else { @@ -178,21 +183,20 @@ template } } -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int max_privatized_smem_bins( - int max_privatized_smem_bytes, int counter_size, int num_active_channels, int padding_bins_per_channel = 0) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int +max_privatized_smem_bins(int max_privatized_smem_bytes, int counter_size, int num_active_channels) { if (max_privatized_smem_bytes <= 0 || counter_size <= 0 || num_active_channels <= 0) { return 0; } - const int slots_per_channel = max_privatized_smem_bytes / counter_size / num_active_channels; - return slots_per_channel > padding_bins_per_channel ? slots_per_channel - padding_bins_per_channel : 0; + return max_privatized_smem_bytes / counter_size / num_active_channels; } [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int max_privatized_static_smem_bins(const HistogramPolicy& policy, int counter_size, int num_active_channels) { - return max_privatized_smem_bins(policy.static_smem.max_privatized_smem_bytes, counter_size, num_active_channels, 1); + return max_privatized_smem_bins(policy.static_smem.max_privatized_smem_bytes, counter_size, num_active_channels); } [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int @@ -398,6 +402,17 @@ struct sm100_tuning struct policy_hub { + static constexpr int pre_sm100_static_smem_max_bins = 256; + static constexpr int sm100_static_smem_max_bins = 512; + static constexpr int pdl_trigger_max_bins = 2048; + static constexpr int sm100_opt_in_smem_bytes = 232448; + static constexpr int sm100_non_histogram_smem_reserve = 4096; + static constexpr int sm100_dynamic_smem_max_bytes = sm100_opt_in_smem_bytes - sm100_non_histogram_smem_reserve; + static constexpr int sm100_range_dynamic_smem_max_bins = 2048; + static constexpr int sm100_even_2ch_dynamic_smem_max_bins = 28544; + static constexpr int sm100_even_3ch_dynamic_smem_max_bins = 19029; + static constexpr int sm100_even_4ch_dynamic_smem_max_bins = 8192; + // TODO(bgruber): move inside t_scale in C++14 static constexpr int v_scale = (sizeof(SampleT) + sizeof(int) - 1) / sizeof(int); @@ -412,8 +427,10 @@ struct policy_hub // TODO This might be worth it to separate usual histogram and the multi one using AgentHistogramPolicyT = agent_histogram_policy<384, t_scale(16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, false>; using GmemPolicy = AgentHistogramPolicyT; - using StaticSmemPolicy = static_smem_policy; - using DynamicSmemPolicy = dynamic_smem_policy; + using StaticSmemPolicy = + static_smem_chained_policy; + using DynamicSmemPolicy = dynamic_smem_chained_policy; static constexpr int init_kernel_pdl_trigger_max_bins = 0; }; @@ -438,14 +455,16 @@ struct policy_hub decltype(select_agent_policy< sm90_tuning()>>(0)); - using GmemPolicy = AgentHistogramPolicyT; - using StaticSmemPolicy = static_smem_policy; - using DynamicSmemPolicy = dynamic_smem_policy; + using GmemPolicy = AgentHistogramPolicyT; + using StaticSmemPolicy = + static_smem_chained_policy; + using DynamicSmemPolicy = dynamic_smem_chained_policy; static constexpr int init_kernel_pdl_trigger_max_bins = NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value && (sizeof(SampleT) == 1 || sizeof(SampleT) == 2) - ? 2048 + ? pdl_trigger_max_bins : 0; }; @@ -488,7 +507,7 @@ struct policy_hub static constexpr int init_kernel_pdl_trigger_max_bins = NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value && (sizeof(SampleT) == 1 || sizeof(SampleT) == 2 || sizeof(SampleT) == 4 || sizeof(SampleT) == 8) - ? 2048 + ? pdl_trigger_max_bins : 0; static constexpr bool use_range_multi_static_smem_policy = !IsEven && NumChannels >= 2 && sizeof(CounterT) == 4 && is_primitive::value; @@ -509,7 +528,7 @@ struct policy_hub static constexpr int static_smem_min_blocks_per_sm = use_range_multi_static_smem_policy || use_range_u64_static_smem_policy ? 3 : 0; - using StaticSmemSweepPolicy = agent_histogram_policy< + using StaticSmemAgentPolicyT = agent_histogram_policy< static_smem_threads_per_block, static_smem_items_per_thread, AgentHistogramPolicyT::LOAD_ALGORITHM, @@ -517,20 +536,22 @@ struct policy_hub AgentHistogramPolicyT::IS_RLE_COMPRESS, AgentHistogramPolicyT::IS_WORK_STEALING, AgentHistogramPolicyT::VEC_SIZE, - 513 * sizeof(CounterT) * NumActiveChannels>; + sm100_static_smem_max_bins * sizeof(CounterT) * NumActiveChannels>; using GmemPolicy = AgentHistogramPolicyT; using StaticSmemPolicy = - static_smem_policy; + static_smem_chained_policy; - static constexpr int dynamic_smem_max_bytes = has_dynamic_smem_tuning ? 232448 - 4096 : 0; + static constexpr int dynamic_smem_max_bytes = has_dynamic_smem_tuning ? sm100_dynamic_smem_max_bytes : 0; using DynamicSmemPolicy = - dynamic_smem_policy; + dynamic_smem_chained_policy; }; using MaxPolicy = Policy1000; @@ -541,16 +562,16 @@ template concept histogram_policy_selector = policy_selector; #endif // _CCCL_HAS_CONCEPTS() -template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto convert_sweep_policy() -> HistogramSweepPolicy +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto make_kernel_config() -> HistogramKernelConfig { - return {StaticSweepPolicy::BLOCK_THREADS, - StaticSweepPolicy::PIXELS_PER_THREAD, - StaticSweepPolicy::VEC_SIZE, - StaticSweepPolicy::LOAD_ALGORITHM, - StaticSweepPolicy::LOAD_MODIFIER, - StaticSweepPolicy::IS_RLE_COMPRESS, - StaticSweepPolicy::IS_WORK_STEALING}; + return {AgentPolicy::BLOCK_THREADS, + AgentPolicy::PIXELS_PER_THREAD, + AgentPolicy::VEC_SIZE, + AgentPolicy::LOAD_ALGORITHM, + AgentPolicy::LOAD_MODIFIER, + AgentPolicy::IS_RLE_COMPRESS, + AgentPolicy::IS_WORK_STEALING}; } template @@ -559,11 +580,11 @@ template using static_smem = typename ActivePolicy::StaticSmemPolicy; using dynamic_smem = typename ActivePolicy::DynamicSmemPolicy; return { - convert_sweep_policy(), - {convert_sweep_policy(), + make_kernel_config(), + {make_kernel_config(), static_smem::MAX_PRIVATIZED_SMEM_BYTES, static_smem::MIN_BLOCKS_PER_SM}, - {convert_sweep_policy(), + {make_kernel_config(), dynamic_smem::MAX_PRIVATIZED_SMEM_BYTES, dynamic_smem::RANGE_MAX_BINS, dynamic_smem::EVEN_2CH_MAX_BINS, diff --git a/cub/test/catch2_test_device_histogram_custom_policy_hub.cu b/cub/test/catch2_test_device_histogram_custom_policy_hub.cu index 9e976687d335..8dc8649f7ff8 100644 --- a/cub/test/catch2_test_device_histogram_custom_policy_hub.cu +++ b/cub/test/catch2_test_device_histogram_custom_policy_hub.cu @@ -23,7 +23,7 @@ struct my_policy_hub // simplified from Policy500 of the CUB histogram tunings struct MaxPolicy : cub::detail::chained_policy<500, MaxPolicy, MaxPolicy> { - using AgentHistogramPolicyT = AgentHistogramPolicy<384, 16, BLOCK_LOAD_DIRECT, LOAD_LDG, true, SMEM, false>; + using AgentHistogramPolicyT = AgentHistogramPolicy<384, 16, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false>; static constexpr int init_kernel_pdl_trigger_max_bins = 2048; }; }; diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index 706282600689..dc95aef54b39 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1627,8 +1627,8 @@ struct histogram_tuning _CCCL_HOST_DEVICE_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { constexpr auto sweep = - cub::HistogramSweepPolicy{BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; - return {sweep, {sweep, 257 * sizeof(unsigned int), 0}, {sweep, 0, 0, 0, 0, 0}, 0}; + cub::HistogramKernelConfig{BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; + return {sweep, {sweep, 256 * sizeof(unsigned int), 0}, {sweep, 0, 0, 0, 0, 0}, 0}; } }; @@ -1645,8 +1645,8 @@ struct mixed_counter_histogram_tuning _CCCL_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { constexpr auto sweep = - cub::HistogramSweepPolicy{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; - return {sweep, {sweep, 513 * sizeof(unsigned int), 0}, {sweep, 228352, 2048, 28544, 19029, 8192}, 0}; + cub::HistogramKernelConfig{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; + return {sweep, {sweep, 512 * sizeof(unsigned int), 0}, {sweep, 228352, 2048, 28544, 19029, 8192}, 0}; } }; @@ -1821,7 +1821,7 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) .load_modifier = cub::CacheLoadModifier::LOAD_LDG, .rle_compress = false, .work_stealing = false}, - .static_smem = {.sweep = {.threads_per_block = 96, + .static_smem = {.kernel = {.threads_per_block = 96, .items_per_thread = 3, .vec_size = 4, .load_algorithm = cub::BLOCK_LOAD_DIRECT, @@ -1831,7 +1831,7 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) .max_privatized_smem_bytes = 2052, .min_blocks_per_sm = 2}, .dynamic_smem = - {.sweep = {.threads_per_block = 128, + {.kernel = {.threads_per_block = 128, .items_per_thread = 7, .vec_size = 4, .load_algorithm = cub::BLOCK_LOAD_DIRECT, @@ -1862,10 +1862,11 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) == "HistogramPolicy { .gmem = { .threads_per_block = 128, .items_per_thread = 7, .vec_size = 4" ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" ", .work_stealing = 0 }" - ", .static_smem = { .sweep = { .threads_per_block = 96, .items_per_thread = 3, .vec_size = 4" + ", .static_smem = { .kernel = { .threads_per_block = 96, .items_per_thread = 3, .vec_size = 4" ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" ", .work_stealing = 0 }" - ", .max_privatized_smem_bytes = 2052, .min_blocks_per_sm = 2 }, .dynamic_smem = { .sweep = { .threads_per_block " + ", .max_privatized_smem_bytes = 2052, .min_blocks_per_sm = 2 }, .dynamic_smem = { .kernel = { " + ".threads_per_block " "= 128" ", .items_per_thread = 7, .vec_size = 4, .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG" ", .rle_compress = 0, .work_stealing = 0 }, .max_privatized_smem_bytes = 12345" @@ -1885,7 +1886,7 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE(cub::detail::histogram::max_privatized_static_smem_bins(sm90_policy, 4, 1) == 256); STATIC_REQUIRE(cub::detail::histogram::max_privatized_static_smem_bins(sm100_policy, 4, 1) == 512); - STATIC_REQUIRE(cub::detail::histogram::max_privatized_static_smem_bins(sm100_policy, 4, 4) == 127); + STATIC_REQUIRE(cub::detail::histogram::max_privatized_static_smem_bins(sm100_policy, 4, 4) == 128); STATIC_REQUIRE(sm90_policy.dynamic_smem.max_privatized_smem_bytes == 0); STATIC_REQUIRE(sm100_policy.dynamic_smem.max_privatized_smem_bytes == 228352); STATIC_REQUIRE(sm100_wide_counter_policy.dynamic_smem.max_privatized_smem_bytes == 0); diff --git a/cub/test/catch2_test_device_histogram_env_api.cu b/cub/test/catch2_test_device_histogram_env_api.cu index c083a0fab8a6..ad64e49eae51 100644 --- a/cub/test/catch2_test_device_histogram_env_api.cu +++ b/cub/test/catch2_test_device_histogram_env_api.cu @@ -418,17 +418,17 @@ struct HistogramPolicySelector { __host__ __device__ constexpr auto operator()(cuda::compute_capability cc) const -> cub::HistogramPolicy { - const auto sweep = cub::HistogramSweepPolicy{ + const auto sweep = cub::HistogramKernelConfig{ 128, cc > cuda::compute_capability{9, 0} ? 16 : 7, 4, cub::BLOCK_LOAD_DIRECT, cub::LOAD_LDG, false, false}; return { - .gmem = sweep, - .static_smem = {.sweep = sweep, .max_privatized_smem_bytes = 257 * sizeof(unsigned int), .min_blocks_per_sm = 0}, - .dynamic_smem = {.sweep = sweep, - .max_privatized_smem_bytes = 0, - .range_max_bins = 0, - .even_2ch_max_bins = 0, - .even_3ch_max_bins = 0, - .even_4ch_max_bins = 0}, + .gmem = sweep, + .static_smem = {.kernel = sweep, .max_privatized_smem_bytes = 256 * sizeof(unsigned int), .min_blocks_per_sm = 0}, + .dynamic_smem = {.kernel = sweep, + .max_privatized_smem_bytes = 0, + .range_max_bins = 0, + .even_2ch_max_bins = 0, + .even_3ch_max_bins = 0, + .even_4ch_max_bins = 0}, .init_kernel_pdl_trigger_max_bins = 2048}; } }; diff --git a/cub/test/catch2_test_enum_formatting.cu b/cub/test/catch2_test_enum_formatting.cu index 87c17e311729..e75d3d27eefc 100644 --- a/cub/test/catch2_test_enum_formatting.cu +++ b/cub/test/catch2_test_enum_formatting.cu @@ -45,15 +45,6 @@ struct FormatTester template void do_test(const Tester& tester) { - // BlockHistogramMemoryPreference - { - tester(cub::BlockHistogramMemoryPreference::GMEM, "GMEM"); - tester(cub::BlockHistogramMemoryPreference::SMEM, "SMEM"); - tester(cub::BlockHistogramMemoryPreference::BLEND, "BLEND"); - - tester(cub::BlockHistogramMemoryPreference(100), ""); - } - // RadixSortStoreAlgorithm { tester(cub::RadixSortStoreAlgorithm::RADIX_SORT_STORE_DIRECT, "RADIX_SORT_STORE_DIRECT"); From d00fc4d7c95d0dc222e9e9ea0658f1ff1208fff8 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Wed, 5 Aug 2026 07:59:41 +0000 Subject: [PATCH 26/45] [cub] Use privatization mode tags consistently --- cub/cub/agent/agent_histogram.cuh | 4 ++++ .../device/dispatch/dispatch_histogram.cuh | 7 +----- .../dispatch/kernels/kernel_histogram.cuh | 23 ++++++------------- .../dispatch/tuning/tuning_histogram.cuh | 17 +++++--------- 4 files changed, 18 insertions(+), 33 deletions(-) diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index 27793e0857b7..e7ef066d9c95 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -91,6 +91,10 @@ struct HistogramPrivatizedDynamicSmem struct HistogramPrivatizedGmem {}; +inline constexpr auto histogram_privatized_static_smem = HistogramPrivatizedStaticSmem{}; +inline constexpr auto histogram_privatized_dynamic_smem = HistogramPrivatizedDynamicSmem{}; +inline constexpr auto histogram_privatized_gmem = HistogramPrivatizedGmem{}; + template inline constexpr bool is_privatized_static_smem_v = ::cuda::std::is_same_v; diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index f33dd22a7a84..021d6ac30d6e 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -276,12 +276,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( } }(); - constexpr auto tier = - is_privatized_dynamic_smem_v ? privatization_tier::dynamic_smem - : is_privatized_static_smem_v - ? privatization_tier::static_smem - : privatization_tier::gmem; - const HistogramKernelConfig sweep = kernel_config(active_policy); + const HistogramKernelConfig sweep = kernel_config(active_policy, PrivatizationMode{}); const int threads_per_block = sweep.threads_per_block; const int items_per_thread = sweep.items_per_thread; diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index 25a4222bd9dc..fef7f13a2342 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -704,14 +704,10 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__( - int(kernel_config< - is_privatized_static_smem_v ? privatization_tier::static_smem : privatization_tier::gmem>( - current_policy()) - .threads_per_block), - int(is_privatized_static_smem_v - ? current_policy().static_smem.min_blocks_per_sm - : 0)) +__launch_bounds__(int(kernel_config(current_policy(), PrivatizationMode{}).threads_per_block), + int(is_privatized_static_smem_v + ? current_policy().static_smem.min_blocks_per_sm + : 0)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepKernel( const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, @@ -727,8 +723,7 @@ __launch_bounds__( GridQueue tile_queue) { static constexpr HistogramPolicy hp = current_policy(); - static constexpr auto sweep = kernel_config< - is_privatized_static_smem_v ? privatization_tier::static_smem : privatization_tier::gmem>(hp); + static constexpr auto sweep = kernel_config(hp, PrivatizationMode{}); // Thread block type for compositing input tiles using AgentHistogramPolicyT = agent_histogram_policy< @@ -963,10 +958,7 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(kernel_config ? privatization_tier::static_smem - : privatization_tier::gmem>( - current_policy()) - .threads_per_block)) +__launch_bounds__(int(kernel_config(current_policy(), PrivatizationMode{}).threads_per_block)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDeviceInitKernel( const SampleIteratorT d_samples, ::cuda::std::array num_output_bins_wrapper, @@ -982,8 +974,7 @@ __launch_bounds__(int(kernel_config tile_queue) { static constexpr HistogramPolicy hp = current_policy(); - static constexpr auto sweep = kernel_config< - is_privatized_static_smem_v ? privatization_tier::static_smem : privatization_tier::gmem>(hp); + static constexpr auto sweep = kernel_config(hp, PrivatizationMode{}); OutputDecodeOpT output_decode_op[NumActiveChannels]; PrivatizedDecodeOpT privatized_decode_op[NumActiveChannels]; diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 8e086862bdb9..09db88d8c611 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -159,26 +159,21 @@ struct dynamic_smem_chained_policy static constexpr int EVEN_4CH_MAX_BINS = Even4chMaxBins; }; -enum class privatization_tier +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const HistogramKernelConfig& +kernel_config(const HistogramPolicy& policy, PrivatizationMode) { - gmem, - static_smem, - dynamic_smem -}; - -template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const HistogramKernelConfig& kernel_config(const HistogramPolicy& policy) -{ - if constexpr (Tier == privatization_tier::static_smem) + if constexpr (is_privatized_static_smem_v) { return policy.static_smem.kernel; } - else if constexpr (Tier == privatization_tier::dynamic_smem) + else if constexpr (is_privatized_dynamic_smem_v) { return policy.dynamic_smem.kernel; } else { + static_assert(is_privatized_gmem_v); return policy.gmem; } } From 88f7ecc92c08974eb29ed8085a24e6591b8381df Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Wed, 5 Aug 2026 09:17:40 +0000 Subject: [PATCH 27/45] [cub] Simplify histogram tuning policy selection --- .../device/dispatch/dispatch_histogram.cuh | 36 +- .../dispatch/tuning/tuning_histogram.cuh | 318 ++++++++---------- cub/test/catch2_test_device_histogram_env.cu | 44 +-- 3 files changed, 192 insertions(+), 206 deletions(-) diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 021d6ac30d6e..acb492b60cf0 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -790,15 +790,7 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device // TODO(bgruber): drop in CCCL 4.0 template -_CCCL_HOST_DEVICE_API constexpr auto convert_policy(int) - -> decltype(typename ActivePolicy::GmemPolicy{}, HistogramPolicy{}) -{ - return convert_chained_policy(); -} - -// TODO(bgruber): drop in CCCL 4.0 -template -_CCCL_HOST_DEVICE_API constexpr auto convert_policy(long) -> HistogramPolicy +_CCCL_HOST_DEVICE_API constexpr auto convert_legacy_policy() -> HistogramPolicy { using sweep = typename ActivePolicy::AgentHistogramPolicyT; const auto kernel_config = HistogramKernelConfig{ @@ -814,7 +806,7 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_policy(long) -> HistogramPolicy // TODO(bgruber): drop in CCCL 4.0 template -struct policy_selector_from_max_policy +struct policy_selector_from_hub { private: struct dispatch_t @@ -824,7 +816,7 @@ private: template _CCCL_HOST_DEVICE_API constexpr cudaError_t Invoke() { - policy = convert_policy(0); + policy = convert_legacy_policy(); return cudaSuccess; } }; @@ -839,7 +831,7 @@ public: _CCCL_VERIFY(MaxPolicy::Invoke(cc.get() * 10, dispatch) == cudaSuccess, ""); return policy; }), - ({ return convert_policy(0); })); + ({ return convert_legacy_policy(); })); } }; @@ -1440,6 +1432,14 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc KernelLauncherFactory launcher_factory = {}, [[maybe_unused]] MaxPolicyT max_policy = {}) { + using default_policy_hub = + detail::histogram::policy_hub; + static constexpr bool uses_default_policy = + ::cuda::std::is_void_v && ::cuda::std::is_same_v; + using policy_selector_t = ::cuda::std::_If< + uses_default_policy, + detail::histogram::policy_selector_from_types, + detail::histogram::policy_selector_from_hub>; return detail::histogram::dispatch_range( d_temp_storage, temp_storage_bytes, @@ -1452,7 +1452,7 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc row_stride_samples, stream, is_byte_sample, - detail::histogram::policy_selector_from_max_policy{}, + policy_selector_t{}, kernel_source, launcher_factory); } @@ -1527,6 +1527,14 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc KernelLauncherFactory launcher_factory = {}, [[maybe_unused]] MaxPolicyT max_policy = {}) { + using default_policy_hub = + detail::histogram::policy_hub; + static constexpr bool uses_default_policy = + ::cuda::std::is_void_v && ::cuda::std::is_same_v; + using policy_selector_t = ::cuda::std::_If< + uses_default_policy, + detail::histogram::policy_selector_from_types, + detail::histogram::policy_selector_from_hub>; return detail::histogram::dispatch_even( d_temp_storage, temp_storage_bytes, @@ -1540,7 +1548,7 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc row_stride_samples, stream, is_byte_sample, - detail::histogram::policy_selector_from_max_policy{}, + policy_selector_t{}, kernel_source, launcher_factory); } diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 09db88d8c611..ecb3f71e69a6 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -130,34 +130,16 @@ struct HistogramPolicy namespace detail::histogram { -// Compile-time policy wrappers carried by each node in CUB's architecture -// policy chain. They pair an AgentHistogram launch policy with the storage -// limits needed by that kernel variant. `make_kernel_config` below converts -// these type-level values into the runtime `HistogramPolicy` selected by -// dispatch. -template -struct static_smem_chained_policy -{ - using AgentPolicyT = AgentPolicy; - static constexpr int MAX_PRIVATIZED_SMEM_BYTES = MaxPrivatizedSmemBytes; - static constexpr int MIN_BLOCKS_PER_SM = MinBlocksPerSm; -}; - -template -struct dynamic_smem_chained_policy -{ - using AgentPolicyT = AgentPolicy; - static constexpr int MAX_PRIVATIZED_SMEM_BYTES = MaxPrivatizedSmemBytes; - static constexpr int RANGE_MAX_BINS = RangeMaxBins; - static constexpr int EVEN_2CH_MAX_BINS = Even2chMaxBins; - static constexpr int EVEN_3CH_MAX_BINS = Even3chMaxBins; - static constexpr int EVEN_4CH_MAX_BINS = Even4chMaxBins; -}; +inline constexpr int pre_sm100_static_smem_max_bins = 256; +inline constexpr int sm100_static_smem_max_bins = 512; +inline constexpr int pdl_trigger_max_bins = 2048; +inline constexpr int sm100_opt_in_smem_bytes = 232448; +inline constexpr int sm100_non_histogram_smem_reserve = 4096; +inline constexpr int sm100_dynamic_smem_max_bytes = sm100_opt_in_smem_bytes - sm100_non_histogram_smem_reserve; +inline constexpr int sm100_range_dynamic_smem_max_bins = 2048; +inline constexpr int sm100_even_2ch_dynamic_smem_max_bins = 28544; +inline constexpr int sm100_even_3ch_dynamic_smem_max_bins = 19029; +inline constexpr int sm100_even_4ch_dynamic_smem_max_bins = 8192; template [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const HistogramKernelConfig& @@ -367,47 +349,12 @@ struct sm100_tuning -struct sm100_tuning -{ - static constexpr int items_per_thread = 12; - static constexpr int threads_per_block = 768; - static constexpr bool rle_compress = true; - static constexpr bool work_stealing = false; - static constexpr CacheLoadModifier load_modifier = LOAD_LDG; - static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_DIRECT; - static constexpr int vec_size = 1 << 2; -}; - -template -struct sm100_tuning -{ - static constexpr int items_per_thread = 6; - static constexpr int threads_per_block = 768; - static constexpr bool rle_compress = true; - static constexpr bool work_stealing = false; - static constexpr CacheLoadModifier load_modifier = LOAD_LDG; - static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_DIRECT; - static constexpr int vec_size = 1 << 2; -}; - -// sample_size 2 retains the SM90 launch shape while using the SM100 shared-memory policy. +// sample_size 2/4/8 retain the SM90 launch shape in the legacy policy hub. // TODO(bgruber): drop in CCCL 4.0 template struct policy_hub { - static constexpr int pre_sm100_static_smem_max_bins = 256; - static constexpr int sm100_static_smem_max_bins = 512; - static constexpr int pdl_trigger_max_bins = 2048; - static constexpr int sm100_opt_in_smem_bytes = 232448; - static constexpr int sm100_non_histogram_smem_reserve = 4096; - static constexpr int sm100_dynamic_smem_max_bytes = sm100_opt_in_smem_bytes - sm100_non_histogram_smem_reserve; - static constexpr int sm100_range_dynamic_smem_max_bins = 2048; - static constexpr int sm100_even_2ch_dynamic_smem_max_bins = 28544; - static constexpr int sm100_even_3ch_dynamic_smem_max_bins = 19029; - static constexpr int sm100_even_4ch_dynamic_smem_max_bins = 8192; - // TODO(bgruber): move inside t_scale in C++14 static constexpr int v_scale = (sizeof(SampleT) + sizeof(int) - 1) / sizeof(int); @@ -421,12 +368,6 @@ struct policy_hub { // TODO This might be worth it to separate usual histogram and the multi one using AgentHistogramPolicyT = agent_histogram_policy<384, t_scale(16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, false>; - using GmemPolicy = AgentHistogramPolicyT; - using StaticSmemPolicy = - static_smem_chained_policy; - using DynamicSmemPolicy = dynamic_smem_chained_policy; - static constexpr int init_kernel_pdl_trigger_max_bins = 0; }; @@ -450,12 +391,6 @@ struct policy_hub decltype(select_agent_policy< sm90_tuning()>>(0)); - using GmemPolicy = AgentHistogramPolicyT; - using StaticSmemPolicy = - static_smem_chained_policy; - using DynamicSmemPolicy = dynamic_smem_chained_policy; - static constexpr int init_kernel_pdl_trigger_max_bins = NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value && (sizeof(SampleT) == 1 || sizeof(SampleT) == 2) @@ -479,74 +414,16 @@ struct policy_hub template _CCCL_HOST_DEVICE_API static auto select_agent_policy(long) -> typename Policy900::AgentHistogramPolicyT; - using SelectedAgentHistogramPolicyT = + using AgentHistogramPolicyT = decltype(select_agent_policy< sm100_tuning()>>( 0)); - using MultiChannelAgentHistogramPolicyT = - agent_histogram_policy<1024, t_scale(IsEven ? 8 : 16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, false, 4>; - - static constexpr bool use_sm100_multi_channel_policy = - NumChannels >= 2 && sizeof(CounterT) == 4 && is_primitive::value; - - using AgentHistogramPolicyT = - ::cuda::std::_If; - - static constexpr bool has_dynamic_smem_tuning = - sizeof(CounterT) == 4 && is_primitive::value - && ((NumChannels == 1 && NumActiveChannels == 1 - && (sizeof(SampleT) == 1 || sizeof(SampleT) == 4 || sizeof(SampleT) == 8)) - || NumChannels >= 2); - static constexpr int init_kernel_pdl_trigger_max_bins = NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value && (sizeof(SampleT) == 1 || sizeof(SampleT) == 2 || sizeof(SampleT) == 4 || sizeof(SampleT) == 8) ? pdl_trigger_max_bins : 0; - static constexpr bool use_range_multi_static_smem_policy = - !IsEven && NumChannels >= 2 && sizeof(CounterT) == 4 && is_primitive::value; - static constexpr bool use_range_u32_static_smem_policy = - !IsEven && NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value - && sizeof(SampleT) == 4; - static constexpr bool use_range_u64_static_smem_policy = - !IsEven && NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value - && sizeof(SampleT) == 8; - - static constexpr int static_smem_threads_per_block = - use_range_multi_static_smem_policy || use_range_u64_static_smem_policy ? 384 - : use_range_u32_static_smem_policy - ? 768 - : AgentHistogramPolicyT::BLOCK_THREADS; - static constexpr int static_smem_items_per_thread = - use_range_u64_static_smem_policy ? t_scale(16) : AgentHistogramPolicyT::PIXELS_PER_THREAD; - static constexpr int static_smem_min_blocks_per_sm = - use_range_multi_static_smem_policy || use_range_u64_static_smem_policy ? 3 : 0; - - using StaticSmemAgentPolicyT = agent_histogram_policy< - static_smem_threads_per_block, - static_smem_items_per_thread, - AgentHistogramPolicyT::LOAD_ALGORITHM, - AgentHistogramPolicyT::LOAD_MODIFIER, - AgentHistogramPolicyT::IS_RLE_COMPRESS, - AgentHistogramPolicyT::IS_WORK_STEALING, - AgentHistogramPolicyT::VEC_SIZE, - sm100_static_smem_max_bins * sizeof(CounterT) * NumActiveChannels>; - - using GmemPolicy = AgentHistogramPolicyT; - using StaticSmemPolicy = - static_smem_chained_policy; - - static constexpr int dynamic_smem_max_bytes = has_dynamic_smem_tuning ? sm100_dynamic_smem_max_bytes : 0; - using DynamicSmemPolicy = - dynamic_smem_chained_policy; }; using MaxPolicy = Policy1000; @@ -557,52 +434,147 @@ template concept histogram_policy_selector = policy_selector; #endif // _CCCL_HAS_CONCEPTS() -template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto make_kernel_config() -> HistogramKernelConfig +struct policy_selector { - return {AgentPolicy::BLOCK_THREADS, - AgentPolicy::PIXELS_PER_THREAD, - AgentPolicy::VEC_SIZE, - AgentPolicy::LOAD_ALGORITHM, - AgentPolicy::LOAD_MODIFIER, - AgentPolicy::IS_RLE_COMPRESS, - AgentPolicy::IS_WORK_STEALING}; -} + bool sample_is_primitive; + int sample_size_bytes; + int counter_size_bytes; + int num_channels; + int num_active_channels; + bool is_even; + +private: + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int t_scale(int nominal_items_per_thread) const + { + const int sample_scale = (sample_size_bytes + int{sizeof(int)} - 1) / int{sizeof(int)}; + return (::cuda::std::max) (nominal_items_per_thread / num_active_channels / sample_scale, 1); + } -template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto convert_chained_policy() -> HistogramPolicy -{ - using static_smem = typename ActivePolicy::StaticSmemPolicy; - using dynamic_smem = typename ActivePolicy::DynamicSmemPolicy; - return { - make_kernel_config(), - {make_kernel_config(), - static_smem::MAX_PRIVATIZED_SMEM_BYTES, - static_smem::MIN_BLOCKS_PER_SM}, - {make_kernel_config(), - dynamic_smem::MAX_PRIVATIZED_SMEM_BYTES, - dynamic_smem::RANGE_MAX_BINS, - dynamic_smem::EVEN_2CH_MAX_BINS, - dynamic_smem::EVEN_3CH_MAX_BINS, - dynamic_smem::EVEN_4CH_MAX_BINS}, - ActivePolicy::init_kernel_pdl_trigger_max_bins}; -} + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto default_kernel_config() const -> HistogramKernelConfig + { + return {384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + } -template -struct policy_selector_from_types -{ - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto sm90_kernel_config() const -> HistogramKernelConfig { - using hub = policy_hub; - if (cc >= ::cuda::compute_capability{10, 0}) + if (num_channels == 1 && num_active_channels == 1 && counter_size_bytes == 4 && sample_is_primitive) { - return convert_chained_policy(); + if (sample_size_bytes == 1) + { + return {768, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, false, false}; + } + if (sample_size_bytes == 2) + { + return {960, 10, 4, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, false}; + } } - if (cc >= ::cuda::compute_capability{9, 0}) + return default_kernel_config(); + } + + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto sm100_kernel_config() const -> HistogramKernelConfig + { + if (num_channels >= 2 && counter_size_bytes == 4 && sample_is_primitive) + { + return {1024, t_scale(is_even ? 8 : 16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + } + if (num_channels == 1 && num_active_channels == 1 && counter_size_bytes == 4 && sample_is_primitive) { - return convert_chained_policy(); + if (sample_size_bytes == 1) + { + return is_even ? HistogramKernelConfig{928, 12, 4, BLOCK_LOAD_DIRECT, LOAD_CA, false, false} + : HistogramKernelConfig{448, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, false, false}; + } + if (sample_size_bytes == 4) + { + return {768, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + } + if (sample_size_bytes == 8) + { + return {768, 6, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + } } - return convert_chained_policy(); + return sm90_kernel_config(); + } + +public: + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy + { + const bool single_channel = num_channels == 1 && num_active_channels == 1; + if (cc >= ::cuda::compute_capability{10, 0}) + { + const HistogramKernelConfig kernel = sm100_kernel_config(); + const bool range_multi_static = !is_even && num_channels >= 2 && counter_size_bytes == 4 && sample_is_primitive; + const bool range_u32_static = + !is_even && single_channel && counter_size_bytes == 4 && sample_is_primitive && sample_size_bytes == 4; + const bool range_u64_static = + !is_even && single_channel && counter_size_bytes == 4 && sample_is_primitive && sample_size_bytes == 8; + HistogramKernelConfig static_kernel = kernel; + if (range_multi_static || range_u64_static) + { + static_kernel.threads_per_block = 384; + } + else if (range_u32_static) + { + static_kernel.threads_per_block = 768; + } + if (range_u64_static) + { + static_kernel.items_per_thread = t_scale(16); + } + + const bool has_dynamic_smem_tuning = + counter_size_bytes == 4 && sample_is_primitive + && ((single_channel && (sample_size_bytes == 1 || sample_size_bytes == 4 || sample_size_bytes == 8)) + || num_channels >= 2); + const int dynamic_smem_bytes = has_dynamic_smem_tuning ? sm100_dynamic_smem_max_bytes : 0; + const int dynamic_range_bins = has_dynamic_smem_tuning ? sm100_range_dynamic_smem_max_bins : 0; + const int dynamic_even_2ch_bins = has_dynamic_smem_tuning ? sm100_even_2ch_dynamic_smem_max_bins : 0; + const int dynamic_even_3ch_bins = has_dynamic_smem_tuning ? sm100_even_3ch_dynamic_smem_max_bins : 0; + const int dynamic_even_4ch_bins = has_dynamic_smem_tuning ? sm100_even_4ch_dynamic_smem_max_bins : 0; + const int pdl_bins = + single_channel && counter_size_bytes == 4 && sample_is_primitive + && (sample_size_bytes == 1 || sample_size_bytes == 2 || sample_size_bytes == 4 || sample_size_bytes == 8) + ? pdl_trigger_max_bins + : 0; + return { + kernel, + {static_kernel, + sm100_static_smem_max_bins * counter_size_bytes * num_active_channels, + range_multi_static || range_u64_static ? 3 : 0}, + {kernel, + dynamic_smem_bytes, + dynamic_range_bins, + dynamic_even_2ch_bins, + dynamic_even_3ch_bins, + dynamic_even_4ch_bins}, + pdl_bins}; + } + + const HistogramKernelConfig kernel = + cc >= ::cuda::compute_capability{9, 0} ? sm90_kernel_config() : default_kernel_config(); + const int pdl_bins = + cc >= ::cuda::compute_capability{9, 0} && single_channel && counter_size_bytes == 4 && sample_is_primitive + && (sample_size_bytes == 1 || sample_size_bytes == 2) + ? pdl_trigger_max_bins + : 0; + return {kernel, + {kernel, pre_sm100_static_smem_max_bins * counter_size_bytes * num_active_channels, 0}, + {kernel, 0, 0, 0, 0, 0}, + pdl_bins}; + } +}; + +#if _CCCL_HAS_CONCEPTS() +static_assert(histogram_policy_selector); +#endif // _CCCL_HAS_CONCEPTS() + +template +struct policy_selector_from_types +{ + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy + { + return policy_selector{ + is_primitive_v, int{sizeof(SampleT)}, int{sizeof(CounterT)}, NumChannels, NumActiveChannels, IsEven}(cc); } }; } // namespace detail::histogram diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index dc95aef54b39..ac488a59314d 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1896,6 +1896,27 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE(sm100_policy.dynamic_smem.even_2ch_max_bins == 28544); STATIC_REQUIRE(sm100_policy.dynamic_smem.even_3ch_max_bins == 19029); STATIC_REQUIRE(sm100_policy.dynamic_smem.even_4ch_max_bins == 8192); + STATIC_REQUIRE(sm100_policy.gmem.threads_per_block == 768); + STATIC_REQUIRE(sm100_policy.gmem.items_per_thread == 12); + STATIC_REQUIRE(sm100_policy.static_smem.kernel == sm100_policy.gmem); + + constexpr auto sm100_range_u64_policy = + cub::detail::histogram::policy_selector_from_types{}( + cuda::compute_capability{10, 0}); + STATIC_REQUIRE(sm100_range_u64_policy.gmem.threads_per_block == 768); + STATIC_REQUIRE(sm100_range_u64_policy.gmem.items_per_thread == 6); + STATIC_REQUIRE(sm100_range_u64_policy.static_smem.kernel.threads_per_block == 384); + STATIC_REQUIRE(sm100_range_u64_policy.static_smem.kernel.items_per_thread == 8); + STATIC_REQUIRE(sm100_range_u64_policy.static_smem.min_blocks_per_sm == 3); + + constexpr auto sm100_multi_range_policy = + cub::detail::histogram::policy_selector_from_types{}( + cuda::compute_capability{10, 0}); + STATIC_REQUIRE(sm100_multi_range_policy.gmem.threads_per_block == 1024); + STATIC_REQUIRE(sm100_multi_range_policy.gmem.items_per_thread == 5); + STATIC_REQUIRE(sm100_multi_range_policy.static_smem.kernel.threads_per_block == 384); + STATIC_REQUIRE(sm100_multi_range_policy.static_smem.kernel.items_per_thread == 5); + STATIC_REQUIRE(sm100_multi_range_policy.static_smem.min_blocks_per_sm == 3); STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57088, 4, 1)); STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57089, 4, 1)); @@ -1909,24 +1930,9 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 19030, 4, 3)); using max_policy_t = typename cub::detail::histogram::policy_hub::MaxPolicy; - const auto legacy_sm100_policy = - cub::detail::histogram::policy_selector_from_max_policy{}(cuda::compute_capability{10, 0}); - REQUIRE(legacy_sm100_policy == sm100_policy); - - using range_max_policy_t = - typename cub::detail::histogram::policy_hub::MaxPolicy; - const auto legacy_range_policy = - cub::detail::histogram::policy_selector_from_max_policy{}(cuda::compute_capability{10, 0}); - constexpr auto range_policy = - cub::detail::histogram::policy_selector_from_types{}( - cuda::compute_capability{10, 0}); - REQUIRE(legacy_range_policy == range_policy); - - using multi_max_policy_t = typename cub::detail::histogram::policy_hub::MaxPolicy; - const auto legacy_multi_policy = - cub::detail::histogram::policy_selector_from_max_policy{}(cuda::compute_capability{10, 0}); - constexpr auto multi_policy = cub::detail::histogram::policy_selector_from_types{}( - cuda::compute_capability{10, 0}); - REQUIRE(legacy_multi_policy == multi_policy); + const auto legacy_policy = + cub::detail::histogram::policy_selector_from_hub{}(cuda::compute_capability{10, 0}); + REQUIRE(legacy_policy.static_smem.max_privatized_smem_bytes == 256 * sizeof(unsigned int)); + REQUIRE(legacy_policy.dynamic_smem.max_privatized_smem_bytes == 0); } #endif // _CCCL_COMPILER(GCC, >=, 8) From ab404521c319451323be18d5d3dc05aaeefc883e Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Wed, 5 Aug 2026 10:37:25 +0000 Subject: [PATCH 28/45] [cub] Collapse histogram tuning into one policy --- .../bench/histogram/histogram_common.cuh | 35 +- cub/cub/agent/agent_histogram.cuh | 100 ++-- .../device/dispatch/dispatch_histogram.cuh | 36 +- .../dispatch/kernels/kernel_histogram.cuh | 134 +++-- .../dispatch/tuning/tuning_histogram.cuh | 476 +++++++++++------- cub/test/catch2_test_device_histogram_env.cu | 226 ++++++--- .../catch2_test_device_histogram_env_api.cu | 41 +- 7 files changed, 637 insertions(+), 411 deletions(-) diff --git a/cub/benchmarks/bench/histogram/histogram_common.cuh b/cub/benchmarks/bench/histogram/histogram_common.cuh index 55c67f27688c..ab5f23c8524e 100644 --- a/cub/benchmarks/bench/histogram/histogram_common.cuh +++ b/cub/benchmarks/bench/histogram/histogram_common.cuh @@ -37,23 +37,36 @@ struct bench_policy_selector ? (NUM_CHANNELS == 1 ? cub::BLOCK_LOAD_STRIPED : cub::BLOCK_LOAD_DIRECT) : TUNE_LOAD_ALGORITHM; - constexpr auto sweep = cub::HistogramKernelConfig{ + return { TUNE_THREADS, TUNE_ITEMS, TUNE_VEC_SIZE, load_algorithm, TUNE_LOAD_MODIFIER, TUNE_RLE_COMPRESS, - TUNE_WORK_STEALING}; - return {sweep, - {sweep, TUNE_STATIC_SMEM_MAX_BYTES, TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM}, - {sweep, - TUNE_DYNAMIC_SMEM_MAX_BYTES, - TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS, - TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS, - TUNE_DYNAMIC_SMEM_EVEN_3CH_MAX_BINS, - TUNE_DYNAMIC_SMEM_EVEN_4CH_MAX_BINS}, - TUNE_INIT_KERNEL_PDL_TRIGGER_MAX_BINS}; + TUNE_WORK_STEALING, + TUNE_THREADS, + TUNE_ITEMS, + TUNE_VEC_SIZE, + load_algorithm, + TUNE_LOAD_MODIFIER, + TUNE_RLE_COMPRESS, + TUNE_WORK_STEALING, + TUNE_STATIC_SMEM_MAX_BYTES, + TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM, + TUNE_THREADS, + TUNE_ITEMS, + TUNE_VEC_SIZE, + load_algorithm, + TUNE_LOAD_MODIFIER, + TUNE_RLE_COMPRESS, + TUNE_WORK_STEALING, + TUNE_DYNAMIC_SMEM_MAX_BYTES, + TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS, + TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS, + TUNE_DYNAMIC_SMEM_EVEN_3CH_MAX_BINS, + TUNE_DYNAMIC_SMEM_EVEN_4CH_MAX_BINS, + TUNE_INIT_KERNEL_PDL_TRIGGER_MAX_BINS}; } }; #endif // !TUNE_BASE diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index e7ef066d9c95..550f471343a4 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -32,40 +32,23 @@ CUB_NAMESPACE_BEGIN namespace detail { -//! Parameterizable tuning policy type for AgentHistogram template -struct agent_histogram_policy + int VecSize = 4> +struct legacy_agent_histogram_policy { - /// Threads per thread block - static constexpr int BLOCK_THREADS = ThreadsPerBlock; - /// Pixels per thread (per tile of input) + static constexpr int BLOCK_THREADS = ThreadsPerBlock; static constexpr int PIXELS_PER_THREAD = PixelsPerThread; - - /// Whether to perform localized RLE to compress samples before histogramming - static constexpr bool IS_RLE_COMPRESS = RleCompress; - - /// Whether to dequeue tiles from a global work queue + static constexpr bool IS_RLE_COMPRESS = RleCompress; static constexpr bool IS_WORK_STEALING = WorkStealing; - - /// Maximum compile-time-sized shared-memory allocation for privatized bins - static constexpr int PRIVATIZED_STATIC_SMEM_BYTES = PrivatizedStaticSmemBytes; - - /// Vector size for samples loading (1, 2, 4) - static constexpr int VEC_SIZE = VecSize; + static constexpr int VEC_SIZE = VecSize; static_assert(VEC_SIZE == 1 || VEC_SIZE == 2 || VEC_SIZE == 4); - - ///< The BlockLoad algorithm to use static constexpr BlockLoadAlgorithm LOAD_ALGORITHM = LoadAlgorithm; - - ///< Cache load modifier for reading input elements - static constexpr CacheLoadModifier LOAD_MODIFIER = LoadModifier; + static constexpr CacheLoadModifier LOAD_MODIFIER = LoadModifier; }; } // namespace detail @@ -77,8 +60,15 @@ template -using AgentHistogramPolicy CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") = detail:: - agent_histogram_policy; +using AgentHistogramPolicy + CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") = detail::legacy_agent_histogram_policy< + ThreadsPerBlock, + PixelsPerThread, + LoadAlgorithm, + LoadModifier, + RleCompress, + WorkStealing, + VecSize>; namespace detail::histogram { @@ -123,8 +113,29 @@ _CCCL_DEVICE _CCCL_FORCEINLINE auto NativePointer(IteratorT itr) //! @brief AgentHistogram implements a stateful abstraction of CUDA thread blocks for participating //! in device-wide histogram . //! -//! @tparam AgentHistogramPolicyT -//! Parameterized AgentHistogramPolicy tuning policy type +//! @tparam ThreadsPerBlock +//! Number of threads in a thread block. +//! +//! @tparam PixelsPerThread +//! Number of pixels processed per thread. +//! +//! @tparam LoadAlgorithm +//! BlockLoad algorithm used to load samples. +//! +//! @tparam LoadModifier +//! Cache modifier used to load samples. +//! +//! @tparam RleCompress +//! Whether to locally run-length encode samples. +//! +//! @tparam WorkStealing +//! Whether blocks dequeue work from a global queue. +//! +//! @tparam VecSize +//! Vector width used to load samples. +//! +//! @tparam PrivatizedStaticSmemBytes +//! Compile-time-sized shared-memory allocation, or zero for another storage mode. //! //! @tparam PrivatizationMode //! Storage mode for the privatized histogram. @@ -154,7 +165,14 @@ _CCCL_DEVICE _CCCL_FORCEINLINE auto NativePointer(IteratorT itr) //! //! @tparam OutputCounterT //! Integer type for final output histogram bins. May be wider than `CounterT`. -template ; static constexpr bool uses_gmem = is_privatized_gmem_v; static constexpr int privatized_static_smem_bins = - uses_static_smem ? AgentHistogramPolicyT::PRIVATIZED_STATIC_SMEM_BYTES / int{sizeof(CounterT)} / NumActiveChannels - : 0; + uses_static_smem ? PrivatizedStaticSmemBytes / int{sizeof(CounterT)} / NumActiveChannels : 0; static_assert(!uses_static_smem || privatized_static_smem_bins > 0, "Static-SMEM privatization requires room for at least one bin"); - static constexpr int vec_size = AgentHistogramPolicyT::VEC_SIZE; - static constexpr int threads_per_block = AgentHistogramPolicyT::BLOCK_THREADS; - static constexpr int pixels_per_thread = AgentHistogramPolicyT::PIXELS_PER_THREAD; + static constexpr int vec_size = VecSize; + static constexpr int threads_per_block = ThreadsPerBlock; + static constexpr int pixels_per_thread = PixelsPerThread; static constexpr int samples_per_thread = pixels_per_thread * NumChannels; static constexpr int vecs_per_thread = samples_per_thread / vec_size; static constexpr int tile_pixels = pixels_per_thread * threads_per_block; static constexpr int tile_samples = samples_per_thread * threads_per_block; - static constexpr bool is_rle_compress = AgentHistogramPolicyT::IS_RLE_COMPRESS; - static constexpr bool is_work_stealing = AgentHistogramPolicyT::IS_WORK_STEALING; - static constexpr CacheLoadModifier load_modifier = AgentHistogramPolicyT::LOAD_MODIFIER; + static constexpr bool is_rle_compress = RleCompress; + static constexpr bool is_work_stealing = WorkStealing; + static constexpr CacheLoadModifier load_modifier = LoadModifier; + static_assert(vec_size == 1 || vec_size == 2 || vec_size == 4); using SampleT = it_value_t; using PixelT = typename CubVector::Type; @@ -202,11 +220,9 @@ struct AgentHistogram SampleIteratorT>; using WrappedPixelIteratorT = CacheModifiedInputIterator; using WrappedVecsIteratorT = CacheModifiedInputIterator; - using BlockLoadSampleT = - BlockLoad; - using BlockLoadPixelT = - BlockLoad; - using BlockLoadVecT = BlockLoad; + using BlockLoadSampleT = BlockLoad; + using BlockLoadPixelT = BlockLoad; + using BlockLoadVecT = BlockLoad; struct _TempStorage { @@ -429,7 +445,7 @@ struct AgentHistogram bool is_valid[pixels_per_thread]; LoadTile(block_offset, valid_samples, samples); - MarkValid(is_valid, valid_samples); + MarkValid(is_valid, valid_samples); AccumulatePixels(samples, is_valid, ::cuda::std::bool_constant{}); } diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index acb492b60cf0..515a93f4046a 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -276,9 +276,8 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( } }(); - const HistogramKernelConfig sweep = kernel_config(active_policy, PrivatizationMode{}); - const int threads_per_block = sweep.threads_per_block; - const int items_per_thread = sweep.items_per_thread; + const int threads_per_block = detail::histogram::threads_per_block(active_policy, PrivatizationMode{}); + const int items_per_thread = detail::histogram::items_per_thread(active_policy, PrivatizationMode{}); int dynamic_smem_bytes = 0; if constexpr (is_privatized_dynamic_smem_v) @@ -289,7 +288,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( } NV_IF_TARGET(NV_IS_HOST, ({ if (const auto error = CubDebug(launcher_factory.set_max_dynamic_smem_size_for( - sweep_kernel, active_policy.dynamic_smem.max_privatized_smem_bytes))) + sweep_kernel, active_policy.dynamic_smem_max_privatized_bytes))) { return error; } @@ -792,16 +791,37 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device template _CCCL_HOST_DEVICE_API constexpr auto convert_legacy_policy() -> HistogramPolicy { - using sweep = typename ActivePolicy::AgentHistogramPolicyT; - const auto kernel_config = HistogramKernelConfig{ + using sweep = typename ActivePolicy::AgentHistogramPolicyT; + return { sweep::BLOCK_THREADS, sweep::PIXELS_PER_THREAD, sweep::VEC_SIZE, sweep::LOAD_ALGORITHM, sweep::LOAD_MODIFIER, sweep::IS_RLE_COMPRESS, - sweep::IS_WORK_STEALING}; - return {kernel_config, {kernel_config, 256 * sizeof(unsigned int), 0}, {kernel_config, 0, 0, 0, 0, 0}, 0}; + sweep::IS_WORK_STEALING, + sweep::BLOCK_THREADS, + sweep::PIXELS_PER_THREAD, + sweep::VEC_SIZE, + sweep::LOAD_ALGORITHM, + sweep::LOAD_MODIFIER, + sweep::IS_RLE_COMPRESS, + sweep::IS_WORK_STEALING, + 256 * sizeof(unsigned int), + 0, + sweep::BLOCK_THREADS, + sweep::PIXELS_PER_THREAD, + sweep::VEC_SIZE, + sweep::LOAD_ALGORITHM, + sweep::LOAD_MODIFIER, + sweep::IS_RLE_COMPRESS, + sweep::IS_WORK_STEALING, + 0, + 0, + 0, + 0, + 0, + 0}; } // TODO(bgruber): drop in CCCL 4.0 diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index fef7f13a2342..82508a4993c1 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -704,9 +704,9 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(kernel_config(current_policy(), PrivatizationMode{}).threads_per_block), +__launch_bounds__(int(threads_per_block(current_policy(), PrivatizationMode{})), int(is_privatized_static_smem_v - ? current_policy().static_smem.min_blocks_per_sm + ? current_policy().static_smem_min_blocks_per_sm : 0)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepKernel( const SampleIteratorT d_samples, @@ -722,30 +722,25 @@ __launch_bounds__(int(kernel_config(current_policy(), Privatizat const int tiles_per_row, GridQueue tile_queue) { - static constexpr HistogramPolicy hp = current_policy(); - static constexpr auto sweep = kernel_config(hp, PrivatizationMode{}); - - // Thread block type for compositing input tiles - using AgentHistogramPolicyT = agent_histogram_policy< - sweep.threads_per_block, - sweep.items_per_thread, - sweep.load_algorithm, - sweep.load_modifier, - sweep.rle_compress, - sweep.work_stealing, - sweep.vec_size, - is_privatized_static_smem_v ? hp.static_smem.max_privatized_smem_bytes : 0>; - using AgentHistogramT = - AgentHistogram; + using AgentHistogramT = AgentHistogram< + threads_per_block(current_policy(), PrivatizationMode{}), + items_per_thread(current_policy(), PrivatizationMode{}), + load_algorithm(current_policy(), PrivatizationMode{}), + load_modifier(current_policy(), PrivatizationMode{}), + rle_compress(current_policy(), PrivatizationMode{}), + work_stealing(current_policy(), PrivatizationMode{}), + vec_size(current_policy(), PrivatizationMode{}), + is_privatized_static_smem_v ? current_policy().static_smem_max_privatized_bytes + : 0, + PrivatizationMode, + NumChannels, + NumActiveChannels, + SampleIteratorT, + CounterT, + PrivatizedDecodeOpT, + OutputDecodeOpT, + OffsetT, + OutputCounterT>; // Shared memory for AgentHistogram __shared__ typename AgentHistogramT::TempStorage static_smem_storage; @@ -790,7 +785,7 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(current_policy().dynamic_smem.kernel.threads_per_block)) +__launch_bounds__(int(current_policy().dynamic_smem_threads_per_block)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDynamicSmemKernel( const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, @@ -805,28 +800,24 @@ __launch_bounds__(int(current_policy().dynamic_smem.kernel.threa const int tiles_per_row, GridQueue tile_queue) { - static constexpr HistogramPolicy hp = current_policy(); - static constexpr HistogramKernelConfig sweep = hp.dynamic_smem.kernel; - - using AgentHistogramPolicyT = - agent_histogram_policy; - using AgentHistogramT = - AgentHistogram; + using AgentHistogramT = AgentHistogram< + current_policy().dynamic_smem_threads_per_block, + current_policy().dynamic_smem_items_per_thread, + current_policy().dynamic_smem_load_algorithm, + current_policy().dynamic_smem_load_modifier, + current_policy().dynamic_smem_rle_compress, + current_policy().dynamic_smem_work_stealing, + current_policy().dynamic_smem_vec_size, + 0, + HistogramPrivatizedDynamicSmem, + NumChannels, + NumActiveChannels, + SampleIteratorT, + CounterT, + PrivatizedDecodeOpT, + OutputDecodeOpT, + OffsetT, + OutputCounterT>; __shared__ typename AgentHistogramT::TempStorage static_smem_storage; extern __shared__ __align__(16) unsigned char dynamic_smem[]; @@ -958,7 +949,7 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(kernel_config(current_policy(), PrivatizationMode{}).threads_per_block)) +__launch_bounds__(int(threads_per_block(current_policy(), PrivatizationMode{}))) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDeviceInitKernel( const SampleIteratorT d_samples, ::cuda::std::array num_output_bins_wrapper, @@ -973,9 +964,6 @@ __launch_bounds__(int(kernel_config(current_policy(), Privatizat const int tiles_per_row, const GridQueue tile_queue) { - static constexpr HistogramPolicy hp = current_policy(); - static constexpr auto sweep = kernel_config(hp, PrivatizationMode{}); - OutputDecodeOpT output_decode_op[NumActiveChannels]; PrivatizedDecodeOpT privatized_decode_op[NumActiveChannels]; if constexpr (IsEven) @@ -1002,27 +990,25 @@ __launch_bounds__(int(kernel_config(current_policy(), Privatizat } } - // Thread block type for compositing input tiles - using AgentHistogramPolicyT = agent_histogram_policy< - sweep.threads_per_block, - sweep.items_per_thread, - sweep.load_algorithm, - sweep.load_modifier, - sweep.rle_compress, - sweep.work_stealing, - sweep.vec_size, - is_privatized_static_smem_v ? hp.static_smem.max_privatized_smem_bytes : 0>; - using AgentHistogramT = - AgentHistogram; + using AgentHistogramT = AgentHistogram< + threads_per_block(current_policy(), PrivatizationMode{}), + items_per_thread(current_policy(), PrivatizationMode{}), + load_algorithm(current_policy(), PrivatizationMode{}), + load_modifier(current_policy(), PrivatizationMode{}), + rle_compress(current_policy(), PrivatizationMode{}), + work_stealing(current_policy(), PrivatizationMode{}), + vec_size(current_policy(), PrivatizationMode{}), + is_privatized_static_smem_v ? current_policy().static_smem_max_privatized_bytes + : 0, + PrivatizationMode, + NumChannels, + NumActiveChannels, + SampleIteratorT, + CounterT, + PrivatizedDecodeOpT, + OutputDecodeOpT, + OffsetT, + OutputCounterT>; // Shared memory for AgentHistogram __shared__ typename AgentHistogramT::TempStorage static_smem_storage; diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index ecb3f71e69a6..2c04a46b6c41 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -25,73 +25,70 @@ CUB_NAMESPACE_BEGIN -//! Runtime launch configuration for one DeviceHistogram kernel. -struct HistogramKernelConfig -{ - int threads_per_block; //!< Number of threads in a CUDA block - int items_per_thread; //!< Number of items processed per thread - int vec_size; //!< Vectorization size for loading samples - BlockLoadAlgorithm load_algorithm; //!< Algorithm used for loading samples - CacheLoadModifier load_modifier; //!< Cache modifier used for loading samples - bool rle_compress; //!< Whether to locally run-length encode samples - bool work_stealing; //!< Whether blocks dequeue tiles from a global queue - - [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool - operator==(const HistogramKernelConfig& lhs, const HistogramKernelConfig& rhs) noexcept - { - return lhs.threads_per_block == rhs.threads_per_block && lhs.items_per_thread == rhs.items_per_thread - && lhs.vec_size == rhs.vec_size && lhs.load_algorithm == rhs.load_algorithm - && lhs.load_modifier == rhs.load_modifier && lhs.rle_compress == rhs.rle_compress - && lhs.work_stealing == rhs.work_stealing; - } -}; - -//! Tuning policy for the compile-time-sized shared-memory histogram kernel. -struct HistogramStaticSmemPolicy -{ - HistogramKernelConfig kernel; - int max_privatized_smem_bytes; //!< Maximum compile-time-sized shared-memory allocation - int min_blocks_per_sm; //!< Minimum blocks per SM requested through launch bounds - - [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool - operator==(const HistogramStaticSmemPolicy& lhs, const HistogramStaticSmemPolicy& rhs) noexcept - { - return lhs.kernel == rhs.kernel && lhs.max_privatized_smem_bytes == rhs.max_privatized_smem_bytes - && lhs.min_blocks_per_sm == rhs.min_blocks_per_sm; - } -}; - -//! Tuning policy for the runtime-sized shared-memory histogram kernel. -struct HistogramDynamicSmemPolicy -{ - HistogramKernelConfig kernel; - int max_privatized_smem_bytes; //!< Maximum runtime-sized shared-memory allocation - int range_max_bins; //!< Maximum bins per channel for multi-channel HistogramRange - int even_2ch_max_bins; //!< Maximum bins per channel for two-channel HistogramEven - int even_3ch_max_bins; //!< Maximum bins per channel for three-channel HistogramEven - int even_4ch_max_bins; //!< Maximum bins per channel for four-channel HistogramEven - - [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool - operator==(const HistogramDynamicSmemPolicy& lhs, const HistogramDynamicSmemPolicy& rhs) noexcept - { - return lhs.kernel == rhs.kernel && lhs.max_privatized_smem_bytes == rhs.max_privatized_smem_bytes - && lhs.range_max_bins == rhs.range_max_bins && lhs.even_2ch_max_bins == rhs.even_2ch_max_bins - && lhs.even_3ch_max_bins == rhs.even_3ch_max_bins && lhs.even_4ch_max_bins == rhs.even_4ch_max_bins; - } -}; - //! The tuning policy for all DeviceHistogram kernel variants. struct HistogramPolicy { - HistogramKernelConfig gmem; - HistogramStaticSmemPolicy static_smem; - HistogramDynamicSmemPolicy dynamic_smem; + int gmem_threads_per_block; + int gmem_items_per_thread; + int gmem_vec_size; + BlockLoadAlgorithm gmem_load_algorithm; + CacheLoadModifier gmem_load_modifier; + bool gmem_rle_compress; + bool gmem_work_stealing; + + int static_smem_threads_per_block; + int static_smem_items_per_thread; + int static_smem_vec_size; + BlockLoadAlgorithm static_smem_load_algorithm; + CacheLoadModifier static_smem_load_modifier; + bool static_smem_rle_compress; + bool static_smem_work_stealing; + int static_smem_max_privatized_bytes; + int static_smem_min_blocks_per_sm; + + int dynamic_smem_threads_per_block; + int dynamic_smem_items_per_thread; + int dynamic_smem_vec_size; + BlockLoadAlgorithm dynamic_smem_load_algorithm; + CacheLoadModifier dynamic_smem_load_modifier; + bool dynamic_smem_rle_compress; + bool dynamic_smem_work_stealing; + int dynamic_smem_max_privatized_bytes; + int dynamic_smem_range_max_bins; + int dynamic_smem_even_2ch_max_bins; + int dynamic_smem_even_3ch_max_bins; + int dynamic_smem_even_4ch_max_bins; + int init_kernel_pdl_trigger_max_bins; //!< Common init-kernel PDL threshold, independent of accumulation tier [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(const HistogramPolicy& lhs, const HistogramPolicy& rhs) noexcept { - return lhs.gmem == rhs.gmem && lhs.static_smem == rhs.static_smem && lhs.dynamic_smem == rhs.dynamic_smem + return lhs.gmem_threads_per_block == rhs.gmem_threads_per_block + && lhs.gmem_items_per_thread == rhs.gmem_items_per_thread && lhs.gmem_vec_size == rhs.gmem_vec_size + && lhs.gmem_load_algorithm == rhs.gmem_load_algorithm && lhs.gmem_load_modifier == rhs.gmem_load_modifier + && lhs.gmem_rle_compress == rhs.gmem_rle_compress && lhs.gmem_work_stealing == rhs.gmem_work_stealing + && lhs.static_smem_threads_per_block == rhs.static_smem_threads_per_block + && lhs.static_smem_items_per_thread == rhs.static_smem_items_per_thread + && lhs.static_smem_vec_size == rhs.static_smem_vec_size + && lhs.static_smem_load_algorithm == rhs.static_smem_load_algorithm + && lhs.static_smem_load_modifier == rhs.static_smem_load_modifier + && lhs.static_smem_rle_compress == rhs.static_smem_rle_compress + && lhs.static_smem_work_stealing == rhs.static_smem_work_stealing + && lhs.static_smem_max_privatized_bytes == rhs.static_smem_max_privatized_bytes + && lhs.static_smem_min_blocks_per_sm == rhs.static_smem_min_blocks_per_sm + && lhs.dynamic_smem_threads_per_block == rhs.dynamic_smem_threads_per_block + && lhs.dynamic_smem_items_per_thread == rhs.dynamic_smem_items_per_thread + && lhs.dynamic_smem_vec_size == rhs.dynamic_smem_vec_size + && lhs.dynamic_smem_load_algorithm == rhs.dynamic_smem_load_algorithm + && lhs.dynamic_smem_load_modifier == rhs.dynamic_smem_load_modifier + && lhs.dynamic_smem_rle_compress == rhs.dynamic_smem_rle_compress + && lhs.dynamic_smem_work_stealing == rhs.dynamic_smem_work_stealing + && lhs.dynamic_smem_max_privatized_bytes == rhs.dynamic_smem_max_privatized_bytes + && lhs.dynamic_smem_range_max_bins == rhs.dynamic_smem_range_max_bins + && lhs.dynamic_smem_even_2ch_max_bins == rhs.dynamic_smem_even_2ch_max_bins + && lhs.dynamic_smem_even_3ch_max_bins == rhs.dynamic_smem_even_3ch_max_bins + && lhs.dynamic_smem_even_4ch_max_bins == rhs.dynamic_smem_even_4ch_max_bins && lhs.init_kernel_pdl_trigger_max_bins == rhs.init_kernel_pdl_trigger_max_bins; } @@ -104,26 +101,29 @@ struct HistogramPolicy #if _CCCL_HOSTED() friend ::std::ostream& operator<<(::std::ostream& os, const HistogramPolicy& p) { - const auto print_kernel = [&](const HistogramKernelConfig& kernel) -> ::std::ostream& { - return os - << "{ .threads_per_block = " << kernel.threads_per_block - << ", .items_per_thread = " << kernel.items_per_thread << ", .vec_size = " << kernel.vec_size - << ", .load_algorithm = " << kernel.load_algorithm << ", .load_modifier = " << kernel.load_modifier - << ", .rle_compress = " << kernel.rle_compress << ", .work_stealing = " << kernel.work_stealing << " }"; - }; - os << "HistogramPolicy { .gmem = "; - print_kernel(p.gmem); - os << ", .static_smem = { .kernel = "; - print_kernel(p.static_smem.kernel); - os << ", .max_privatized_smem_bytes = " << p.static_smem.max_privatized_smem_bytes - << ", .min_blocks_per_sm = " << p.static_smem.min_blocks_per_sm << " }, .dynamic_smem = { .kernel = "; - print_kernel(p.dynamic_smem.kernel); return os - << ", .max_privatized_smem_bytes = " << p.dynamic_smem.max_privatized_smem_bytes << ", .range_max_bins = " - << p.dynamic_smem.range_max_bins << ", .even_2ch_max_bins = " << p.dynamic_smem.even_2ch_max_bins - << ", .even_3ch_max_bins = " << p.dynamic_smem.even_3ch_max_bins - << ", .even_4ch_max_bins = " << p.dynamic_smem.even_4ch_max_bins - << " }, .init_kernel_pdl_trigger_max_bins = " << p.init_kernel_pdl_trigger_max_bins << " }"; + << "HistogramPolicy { .gmem_threads_per_block = " << p.gmem_threads_per_block + << ", .gmem_items_per_thread = " << p.gmem_items_per_thread << ", .gmem_vec_size = " << p.gmem_vec_size + << ", .gmem_load_algorithm = " << p.gmem_load_algorithm << ", .gmem_load_modifier = " << p.gmem_load_modifier + << ", .gmem_rle_compress = " << p.gmem_rle_compress << ", .gmem_work_stealing = " << p.gmem_work_stealing + << ", .static_smem_threads_per_block = " << p.static_smem_threads_per_block + << ", .static_smem_items_per_thread = " << p.static_smem_items_per_thread << ", .static_smem_vec_size = " + << p.static_smem_vec_size << ", .static_smem_load_algorithm = " << p.static_smem_load_algorithm + << ", .static_smem_load_modifier = " << p.static_smem_load_modifier << ", .static_smem_rle_compress = " + << p.static_smem_rle_compress << ", .static_smem_work_stealing = " << p.static_smem_work_stealing + << ", .static_smem_max_privatized_bytes = " << p.static_smem_max_privatized_bytes + << ", .static_smem_min_blocks_per_sm = " << p.static_smem_min_blocks_per_sm + << ", .dynamic_smem_threads_per_block = " << p.dynamic_smem_threads_per_block + << ", .dynamic_smem_items_per_thread = " << p.dynamic_smem_items_per_thread << ", .dynamic_smem_vec_size = " + << p.dynamic_smem_vec_size << ", .dynamic_smem_load_algorithm = " << p.dynamic_smem_load_algorithm + << ", .dynamic_smem_load_modifier = " << p.dynamic_smem_load_modifier << ", .dynamic_smem_rle_compress = " + << p.dynamic_smem_rle_compress << ", .dynamic_smem_work_stealing = " << p.dynamic_smem_work_stealing + << ", .dynamic_smem_max_privatized_bytes = " << p.dynamic_smem_max_privatized_bytes + << ", .dynamic_smem_range_max_bins = " << p.dynamic_smem_range_max_bins + << ", .dynamic_smem_even_2ch_max_bins = " << p.dynamic_smem_even_2ch_max_bins + << ", .dynamic_smem_even_3ch_max_bins = " << p.dynamic_smem_even_3ch_max_bins + << ", .dynamic_smem_even_4ch_max_bins = " << p.dynamic_smem_even_4ch_max_bins + << ", .init_kernel_pdl_trigger_max_bins = " << p.init_kernel_pdl_trigger_max_bins << " }"; } #endif }; @@ -142,24 +142,51 @@ inline constexpr int sm100_even_3ch_dynamic_smem_max_bins = 19029; inline constexpr int sm100_even_4ch_dynamic_smem_max_bins = 8192; template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const HistogramKernelConfig& -kernel_config(const HistogramPolicy& policy, PrivatizationMode) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int threads_per_block(const HistogramPolicy& policy, PrivatizationMode) { if constexpr (is_privatized_static_smem_v) { - return policy.static_smem.kernel; + return policy.static_smem_threads_per_block; } else if constexpr (is_privatized_dynamic_smem_v) { - return policy.dynamic_smem.kernel; + return policy.dynamic_smem_threads_per_block; } else { static_assert(is_privatized_gmem_v); - return policy.gmem; + return policy.gmem_threads_per_block; } } +#define CUB_DETAIL_HISTOGRAM_POLICY_FIELD_ACCESSOR(NAME, TYPE) \ + template \ + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr TYPE NAME(const HistogramPolicy& policy, PrivatizationMode) \ + { \ + if constexpr (is_privatized_static_smem_v) \ + { \ + return policy.static_smem_##NAME; \ + } \ + else if constexpr (is_privatized_dynamic_smem_v) \ + { \ + return policy.dynamic_smem_##NAME; \ + } \ + else \ + { \ + static_assert(is_privatized_gmem_v); \ + return policy.gmem_##NAME; \ + } \ + } + +CUB_DETAIL_HISTOGRAM_POLICY_FIELD_ACCESSOR(items_per_thread, int) +CUB_DETAIL_HISTOGRAM_POLICY_FIELD_ACCESSOR(vec_size, int) +CUB_DETAIL_HISTOGRAM_POLICY_FIELD_ACCESSOR(load_algorithm, BlockLoadAlgorithm) +CUB_DETAIL_HISTOGRAM_POLICY_FIELD_ACCESSOR(load_modifier, CacheLoadModifier) +CUB_DETAIL_HISTOGRAM_POLICY_FIELD_ACCESSOR(rle_compress, bool) +CUB_DETAIL_HISTOGRAM_POLICY_FIELD_ACCESSOR(work_stealing, bool) + +#undef CUB_DETAIL_HISTOGRAM_POLICY_FIELD_ACCESSOR + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int max_privatized_smem_bins(int max_privatized_smem_bytes, int counter_size, int num_active_channels) { @@ -173,13 +200,13 @@ max_privatized_smem_bins(int max_privatized_smem_bytes, int counter_size, int nu [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int max_privatized_static_smem_bins(const HistogramPolicy& policy, int counter_size, int num_active_channels) { - return max_privatized_smem_bins(policy.static_smem.max_privatized_smem_bytes, counter_size, num_active_channels); + return max_privatized_smem_bins(policy.static_smem_max_privatized_bytes, counter_size, num_active_channels); } [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int max_privatized_dynamic_smem_bins(const HistogramPolicy& policy, int counter_size, int num_active_channels) { - return max_privatized_smem_bins(policy.dynamic_smem.max_privatized_smem_bytes, counter_size, num_active_channels); + return max_privatized_smem_bins(policy.dynamic_smem_max_privatized_bytes, counter_size, num_active_channels); } [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool @@ -210,14 +237,14 @@ should_use_dynamic_smem(const HistogramPolicy& policy, int num_bins, int counter { if constexpr (IsEven) { - max_bins = num_active_channels == 2 ? policy.dynamic_smem.even_2ch_max_bins + max_bins = num_active_channels == 2 ? policy.dynamic_smem_even_2ch_max_bins : num_active_channels == 3 - ? policy.dynamic_smem.even_3ch_max_bins - : policy.dynamic_smem.even_4ch_max_bins; + ? policy.dynamic_smem_even_3ch_max_bins + : policy.dynamic_smem_even_4ch_max_bins; } else { - max_bins = policy.dynamic_smem.range_max_bins; + max_bins = policy.dynamic_smem_range_max_bins; } } @@ -367,7 +394,8 @@ struct policy_hub struct Policy500 : detail::chained_policy<500, Policy500, Policy500> { // TODO This might be worth it to separate usual histogram and the multi one - using AgentHistogramPolicyT = agent_histogram_policy<384, t_scale(16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, false>; + using AgentHistogramPolicyT = + legacy_agent_histogram_policy<384, t_scale(16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, false>; static constexpr int init_kernel_pdl_trigger_max_bins = 0; }; @@ -377,12 +405,12 @@ struct policy_hub // Use values from tuning if a specialization exists, otherwise pick Policy500 template _CCCL_HOST_DEVICE_API static auto select_agent_policy(int) - -> agent_histogram_policy; + -> legacy_agent_histogram_policy; template _CCCL_HOST_DEVICE_API static auto select_agent_policy(long) -> typename Policy500::AgentHistogramPolicyT; @@ -402,14 +430,14 @@ struct policy_hub { // Use values from tuning if a specialization exists, otherwise pick Policy900 template - _CCCL_HOST_DEVICE_API static auto select_agent_policy(int) - -> agent_histogram_policy; + _CCCL_HOST_DEVICE_API static auto select_agent_policy(int) -> legacy_agent_histogram_policy< + Tuning::threads_per_block, + Tuning::items_per_thread, + Tuning::load_algorithm, + Tuning::load_modifier, + Tuning::rle_compress, + Tuning::work_stealing, + Tuning::vec_size>; template _CCCL_HOST_DEVICE_API static auto select_agent_policy(long) -> typename Policy900::AgentHistogramPolicyT; @@ -450,120 +478,202 @@ private: return (::cuda::std::max) (nominal_items_per_thread / num_active_channels / sample_scale, 1); } - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto default_kernel_config() const -> HistogramKernelConfig + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto make_policy( + int threads_per_block, + int items_per_thread, + int vec_size, + BlockLoadAlgorithm load_algorithm, + CacheLoadModifier load_modifier, + bool rle_compress, + bool work_stealing, + int static_smem_max_bytes, + int pdl_max_bins) const -> HistogramPolicy { - return {384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + return { + threads_per_block, + items_per_thread, + vec_size, + load_algorithm, + load_modifier, + rle_compress, + work_stealing, + threads_per_block, + items_per_thread, + vec_size, + load_algorithm, + load_modifier, + rle_compress, + work_stealing, + static_smem_max_bytes, + 0, + threads_per_block, + items_per_thread, + vec_size, + load_algorithm, + load_modifier, + rle_compress, + work_stealing, + 0, + 0, + 0, + 0, + 0, + pdl_max_bins}; } - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto sm90_kernel_config() const -> HistogramKernelConfig + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto default_policy() const -> HistogramPolicy { - if (num_channels == 1 && num_active_channels == 1 && counter_size_bytes == 4 && sample_is_primitive) + return make_policy( + 384, + t_scale(16), + 4, + BLOCK_LOAD_DIRECT, + LOAD_LDG, + true, + false, + pre_sm100_static_smem_max_bins * counter_size_bytes * num_active_channels, + 0); + } + + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto sm90_policy() const -> HistogramPolicy + { + const bool single_channel = num_channels == 1 && num_active_channels == 1; + const int pdl_max_bins = + single_channel && counter_size_bytes == 4 && sample_is_primitive + && (sample_size_bytes == 1 || sample_size_bytes == 2) + ? pdl_trigger_max_bins + : 0; + if (single_channel && counter_size_bytes == 4 && sample_is_primitive) { if (sample_size_bytes == 1) { - return {768, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, false, false}; + return make_policy( + 768, + 12, + 4, + BLOCK_LOAD_DIRECT, + LOAD_LDG, + false, + false, + pre_sm100_static_smem_max_bins * counter_size_bytes * num_active_channels, + pdl_max_bins); } if (sample_size_bytes == 2) { - return {960, 10, 4, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, false}; + return make_policy( + 960, + 10, + 4, + BLOCK_LOAD_DIRECT, + LOAD_DEFAULT, + true, + false, + pre_sm100_static_smem_max_bins * counter_size_bytes * num_active_channels, + pdl_max_bins); } } - return default_kernel_config(); + auto result = default_policy(); + result.init_kernel_pdl_trigger_max_bins = pdl_max_bins; + return result; } - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto sm100_kernel_config() const -> HistogramKernelConfig +public: + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy { - if (num_channels >= 2 && counter_size_bytes == 4 && sample_is_primitive) + if (cc < ::cuda::compute_capability{9, 0}) { - return {1024, t_scale(is_even ? 8 : 16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + return default_policy(); } - if (num_channels == 1 && num_active_channels == 1 && counter_size_bytes == 4 && sample_is_primitive) + if (cc < ::cuda::compute_capability{10, 0}) { - if (sample_size_bytes == 1) - { - return is_even ? HistogramKernelConfig{928, 12, 4, BLOCK_LOAD_DIRECT, LOAD_CA, false, false} - : HistogramKernelConfig{448, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, false, false}; - } - if (sample_size_bytes == 4) - { - return {768, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; - } - if (sample_size_bytes == 8) - { - return {768, 6, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; - } + return sm90_policy(); } - return sm90_kernel_config(); - } -public: - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy - { const bool single_channel = num_channels == 1 && num_active_channels == 1; - if (cc >= ::cuda::compute_capability{10, 0}) + auto result = sm90_policy(); + if (num_channels >= 2 && counter_size_bytes == 4 && sample_is_primitive) + { + result.gmem_threads_per_block = 1024; + result.gmem_items_per_thread = t_scale(is_even ? 8 : 16); + } + else if (single_channel && counter_size_bytes == 4 && sample_is_primitive) { - const HistogramKernelConfig kernel = sm100_kernel_config(); - const bool range_multi_static = !is_even && num_channels >= 2 && counter_size_bytes == 4 && sample_is_primitive; - const bool range_u32_static = - !is_even && single_channel && counter_size_bytes == 4 && sample_is_primitive && sample_size_bytes == 4; - const bool range_u64_static = - !is_even && single_channel && counter_size_bytes == 4 && sample_is_primitive && sample_size_bytes == 8; - HistogramKernelConfig static_kernel = kernel; - if (range_multi_static || range_u64_static) + if (sample_size_bytes == 1) { - static_kernel.threads_per_block = 384; + result.gmem_threads_per_block = is_even ? 928 : 448; + result.gmem_items_per_thread = 12; + result.gmem_load_modifier = is_even ? LOAD_CA : LOAD_LDG; + result.gmem_rle_compress = false; } - else if (range_u32_static) + else if (sample_size_bytes == 4) { - static_kernel.threads_per_block = 768; + result.gmem_threads_per_block = 768; + result.gmem_items_per_thread = 12; } - if (range_u64_static) + else if (sample_size_bytes == 8) { - static_kernel.items_per_thread = t_scale(16); + result.gmem_threads_per_block = 768; + result.gmem_items_per_thread = 6; } + } - const bool has_dynamic_smem_tuning = - counter_size_bytes == 4 && sample_is_primitive - && ((single_channel && (sample_size_bytes == 1 || sample_size_bytes == 4 || sample_size_bytes == 8)) - || num_channels >= 2); - const int dynamic_smem_bytes = has_dynamic_smem_tuning ? sm100_dynamic_smem_max_bytes : 0; - const int dynamic_range_bins = has_dynamic_smem_tuning ? sm100_range_dynamic_smem_max_bins : 0; - const int dynamic_even_2ch_bins = has_dynamic_smem_tuning ? sm100_even_2ch_dynamic_smem_max_bins : 0; - const int dynamic_even_3ch_bins = has_dynamic_smem_tuning ? sm100_even_3ch_dynamic_smem_max_bins : 0; - const int dynamic_even_4ch_bins = has_dynamic_smem_tuning ? sm100_even_4ch_dynamic_smem_max_bins : 0; - const int pdl_bins = - single_channel && counter_size_bytes == 4 && sample_is_primitive - && (sample_size_bytes == 1 || sample_size_bytes == 2 || sample_size_bytes == 4 || sample_size_bytes == 8) - ? pdl_trigger_max_bins - : 0; - return { - kernel, - {static_kernel, - sm100_static_smem_max_bins * counter_size_bytes * num_active_channels, - range_multi_static || range_u64_static ? 3 : 0}, - {kernel, - dynamic_smem_bytes, - dynamic_range_bins, - dynamic_even_2ch_bins, - dynamic_even_3ch_bins, - dynamic_even_4ch_bins}, - pdl_bins}; + result.static_smem_threads_per_block = result.gmem_threads_per_block; + result.static_smem_items_per_thread = result.gmem_items_per_thread; + result.static_smem_vec_size = result.gmem_vec_size; + result.static_smem_load_algorithm = result.gmem_load_algorithm; + result.static_smem_load_modifier = result.gmem_load_modifier; + result.static_smem_rle_compress = result.gmem_rle_compress; + result.static_smem_work_stealing = result.gmem_work_stealing; + + const bool range_multi_static = !is_even && num_channels >= 2 && counter_size_bytes == 4 && sample_is_primitive; + const bool range_u32_static = + !is_even && single_channel && counter_size_bytes == 4 && sample_is_primitive && sample_size_bytes == 4; + const bool range_u64_static = + !is_even && single_channel && counter_size_bytes == 4 && sample_is_primitive && sample_size_bytes == 8; + if (range_multi_static || range_u64_static) + { + result.static_smem_threads_per_block = 384; + } + else if (range_u32_static) + { + result.static_smem_threads_per_block = 768; + } + if (range_u64_static) + { + result.static_smem_items_per_thread = t_scale(16); + } + result.static_smem_max_privatized_bytes = sm100_static_smem_max_bins * counter_size_bytes * num_active_channels; + result.static_smem_min_blocks_per_sm = range_multi_static || range_u64_static ? 3 : 0; + + result.dynamic_smem_threads_per_block = result.gmem_threads_per_block; + result.dynamic_smem_items_per_thread = result.gmem_items_per_thread; + result.dynamic_smem_vec_size = result.gmem_vec_size; + result.dynamic_smem_load_algorithm = result.gmem_load_algorithm; + result.dynamic_smem_load_modifier = result.gmem_load_modifier; + result.dynamic_smem_rle_compress = result.gmem_rle_compress; + result.dynamic_smem_work_stealing = result.gmem_work_stealing; + + const bool has_dynamic_smem_tuning = + counter_size_bytes == 4 && sample_is_primitive + && ((single_channel && (sample_size_bytes == 1 || sample_size_bytes == 4 || sample_size_bytes == 8)) + || num_channels >= 2); + if (has_dynamic_smem_tuning) + { + result.dynamic_smem_max_privatized_bytes = sm100_dynamic_smem_max_bytes; + result.dynamic_smem_range_max_bins = sm100_range_dynamic_smem_max_bins; + result.dynamic_smem_even_2ch_max_bins = sm100_even_2ch_dynamic_smem_max_bins; + result.dynamic_smem_even_3ch_max_bins = sm100_even_3ch_dynamic_smem_max_bins; + result.dynamic_smem_even_4ch_max_bins = sm100_even_4ch_dynamic_smem_max_bins; } - const HistogramKernelConfig kernel = - cc >= ::cuda::compute_capability{9, 0} ? sm90_kernel_config() : default_kernel_config(); - const int pdl_bins = - cc >= ::cuda::compute_capability{9, 0} && single_channel && counter_size_bytes == 4 && sample_is_primitive - && (sample_size_bytes == 1 || sample_size_bytes == 2) + result.init_kernel_pdl_trigger_max_bins = + single_channel && counter_size_bytes == 4 && sample_is_primitive + && (sample_size_bytes == 1 || sample_size_bytes == 2 || sample_size_bytes == 4 || sample_size_bytes == 8) ? pdl_trigger_max_bins : 0; - return {kernel, - {kernel, pre_sm100_static_smem_max_bins * counter_size_bytes * num_active_channels, 0}, - {kernel, 0, 0, 0, 0, 0}, - pdl_bins}; + return result; } }; - #if _CCCL_HAS_CONCEPTS() static_assert(histogram_policy_selector); #endif // _CCCL_HAS_CONCEPTS() diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index ac488a59314d..7542d6037418 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1626,9 +1626,36 @@ struct histogram_tuning { _CCCL_HOST_DEVICE_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { - constexpr auto sweep = - cub::HistogramKernelConfig{BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; - return {sweep, {sweep, 256 * sizeof(unsigned int), 0}, {sweep, 0, 0, 0, 0, 0}, 0}; + return { + BlockThreads, + 1, + 1, + cub::BLOCK_LOAD_DIRECT, + cub::LOAD_DEFAULT, + false, + false, + BlockThreads, + 1, + 1, + cub::BLOCK_LOAD_DIRECT, + cub::LOAD_DEFAULT, + false, + false, + 256 * sizeof(unsigned int), + 0, + BlockThreads, + 1, + 1, + cub::BLOCK_LOAD_DIRECT, + cub::LOAD_DEFAULT, + false, + false, + 0, + 0, + 0, + 0, + 0, + 0}; } }; @@ -1644,9 +1671,36 @@ struct mixed_counter_histogram_tuning _CCCL_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { - constexpr auto sweep = - cub::HistogramKernelConfig{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; - return {sweep, {sweep, 512 * sizeof(unsigned int), 0}, {sweep, 228352, 2048, 28544, 19029, 8192}, 0}; + return { + 128, + 4, + 1, + cub::BLOCK_LOAD_DIRECT, + cub::LOAD_DEFAULT, + false, + false, + 128, + 4, + 1, + cub::BLOCK_LOAD_DIRECT, + cub::LOAD_DEFAULT, + false, + false, + 512 * sizeof(unsigned int), + 0, + 128, + 4, + 1, + cub::BLOCK_LOAD_DIRECT, + cub::LOAD_DEFAULT, + false, + false, + 228352, + 2048, + 28544, + 19029, + 8192, + 0}; } }; @@ -1806,72 +1860,80 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) // aggregate init constexpr auto p1 = cub::HistogramPolicy{ - {128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, - {{96, 3, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, 2052, 2}, - {{128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, 12345, 1024, 4096, 8192, 16384}, + 128, + 7, + 4, + cub::BLOCK_LOAD_DIRECT, + cub::CacheLoadModifier::LOAD_LDG, + false, + false, + 96, + 3, + 4, + cub::BLOCK_LOAD_DIRECT, + cub::CacheLoadModifier::LOAD_LDG, + false, + false, + 2052, + 2, + 128, + 7, + 4, + cub::BLOCK_LOAD_DIRECT, + cub::CacheLoadModifier::LOAD_LDG, + false, + false, + 12345, + 1024, + 4096, + 8192, + 16384, 2048}; # if _CCCL_STD_VER >= 2020 // designated init constexpr auto p2 = cub::HistogramPolicy{ - .gmem = {.threads_per_block = 128, - .items_per_thread = 7, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .static_smem = {.kernel = {.threads_per_block = 96, - .items_per_thread = 3, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .max_privatized_smem_bytes = 2052, - .min_blocks_per_sm = 2}, - .dynamic_smem = - {.kernel = {.threads_per_block = 128, - .items_per_thread = 7, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .max_privatized_smem_bytes = 12345, - .range_max_bins = 1024, - .even_2ch_max_bins = 4096, - .even_3ch_max_bins = 8192, - .even_4ch_max_bins = 16384}, - .init_kernel_pdl_trigger_max_bins = 2048}; + .gmem_threads_per_block = 128, + .gmem_items_per_thread = 7, + .gmem_vec_size = 4, + .gmem_load_algorithm = cub::BLOCK_LOAD_DIRECT, + .gmem_load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .gmem_rle_compress = false, + .gmem_work_stealing = false, + .static_smem_threads_per_block = 96, + .static_smem_items_per_thread = 3, + .static_smem_vec_size = 4, + .static_smem_load_algorithm = cub::BLOCK_LOAD_DIRECT, + .static_smem_load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .static_smem_rle_compress = false, + .static_smem_work_stealing = false, + .static_smem_max_privatized_bytes = 2052, + .static_smem_min_blocks_per_sm = 2, + .dynamic_smem_threads_per_block = 128, + .dynamic_smem_items_per_thread = 7, + .dynamic_smem_vec_size = 4, + .dynamic_smem_load_algorithm = cub::BLOCK_LOAD_DIRECT, + .dynamic_smem_load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .dynamic_smem_rle_compress = false, + .dynamic_smem_work_stealing = false, + .dynamic_smem_max_privatized_bytes = 12345, + .dynamic_smem_range_max_bins = 1024, + .dynamic_smem_even_2ch_max_bins = 4096, + .dynamic_smem_even_3ch_max_bins = 8192, + .dynamic_smem_even_4ch_max_bins = 16384, + .init_kernel_pdl_trigger_max_bins = 2048}; # else // _CCCL_STD_VER >= 2020 constexpr auto p2 = p1; # endif // _CCCL_STD_VER >= 2020 - // comparison STATIC_REQUIRE(p1 == p2); STATIC_REQUIRE_FALSE(p1 != p2); - auto to_string = [](const auto& p) { - std::ostringstream os; - os << p; - return os.str(); - }; - REQUIRE( - to_string(p1) - == "HistogramPolicy { .gmem = { .threads_per_block = 128, .items_per_thread = 7, .vec_size = 4" - ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" - ", .work_stealing = 0 }" - ", .static_smem = { .kernel = { .threads_per_block = 96, .items_per_thread = 3, .vec_size = 4" - ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" - ", .work_stealing = 0 }" - ", .max_privatized_smem_bytes = 2052, .min_blocks_per_sm = 2 }, .dynamic_smem = { .kernel = { " - ".threads_per_block " - "= 128" - ", .items_per_thread = 7, .vec_size = 4, .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG" - ", .rle_compress = 0, .work_stealing = 0 }, .max_privatized_smem_bytes = 12345" - ", .range_max_bins = 1024, .even_2ch_max_bins = 4096, .even_3ch_max_bins = 8192" - ", .even_4ch_max_bins = 16384 }, .init_kernel_pdl_trigger_max_bins = 2048 }"); + std::ostringstream os; + os << p1; + REQUIRE(os.str().find("HistogramPolicy { .gmem_threads_per_block = 128") == 0); + REQUIRE(os.str().find(".static_smem_max_privatized_bytes = 2052") != std::string::npos); + REQUIRE(os.str().find(".dynamic_smem_max_privatized_bytes = 12345") != std::string::npos); } C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget", "[histogram][device]") @@ -1887,36 +1949,36 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE(cub::detail::histogram::max_privatized_static_smem_bins(sm90_policy, 4, 1) == 256); STATIC_REQUIRE(cub::detail::histogram::max_privatized_static_smem_bins(sm100_policy, 4, 1) == 512); STATIC_REQUIRE(cub::detail::histogram::max_privatized_static_smem_bins(sm100_policy, 4, 4) == 128); - STATIC_REQUIRE(sm90_policy.dynamic_smem.max_privatized_smem_bytes == 0); - STATIC_REQUIRE(sm100_policy.dynamic_smem.max_privatized_smem_bytes == 228352); - STATIC_REQUIRE(sm100_wide_counter_policy.dynamic_smem.max_privatized_smem_bytes == 0); + STATIC_REQUIRE(sm90_policy.dynamic_smem_max_privatized_bytes == 0); + STATIC_REQUIRE(sm100_policy.dynamic_smem_max_privatized_bytes == 228352); + STATIC_REQUIRE(sm100_wide_counter_policy.dynamic_smem_max_privatized_bytes == 0); STATIC_REQUIRE(cub::detail::histogram::max_privatized_dynamic_smem_bins(sm100_policy, 4, 1) == 57088); STATIC_REQUIRE(cub::detail::histogram::max_privatized_dynamic_smem_bins(sm100_policy, 4, 4) == 14272); - STATIC_REQUIRE(sm100_policy.dynamic_smem.range_max_bins == 2048); - STATIC_REQUIRE(sm100_policy.dynamic_smem.even_2ch_max_bins == 28544); - STATIC_REQUIRE(sm100_policy.dynamic_smem.even_3ch_max_bins == 19029); - STATIC_REQUIRE(sm100_policy.dynamic_smem.even_4ch_max_bins == 8192); - STATIC_REQUIRE(sm100_policy.gmem.threads_per_block == 768); - STATIC_REQUIRE(sm100_policy.gmem.items_per_thread == 12); - STATIC_REQUIRE(sm100_policy.static_smem.kernel == sm100_policy.gmem); + STATIC_REQUIRE(sm100_policy.dynamic_smem_range_max_bins == 2048); + STATIC_REQUIRE(sm100_policy.dynamic_smem_even_2ch_max_bins == 28544); + STATIC_REQUIRE(sm100_policy.dynamic_smem_even_3ch_max_bins == 19029); + STATIC_REQUIRE(sm100_policy.dynamic_smem_even_4ch_max_bins == 8192); + STATIC_REQUIRE(sm100_policy.gmem_threads_per_block == 768); + STATIC_REQUIRE(sm100_policy.gmem_items_per_thread == 12); + STATIC_REQUIRE(sm100_policy.static_smem_threads_per_block == sm100_policy.gmem_threads_per_block); constexpr auto sm100_range_u64_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); - STATIC_REQUIRE(sm100_range_u64_policy.gmem.threads_per_block == 768); - STATIC_REQUIRE(sm100_range_u64_policy.gmem.items_per_thread == 6); - STATIC_REQUIRE(sm100_range_u64_policy.static_smem.kernel.threads_per_block == 384); - STATIC_REQUIRE(sm100_range_u64_policy.static_smem.kernel.items_per_thread == 8); - STATIC_REQUIRE(sm100_range_u64_policy.static_smem.min_blocks_per_sm == 3); + STATIC_REQUIRE(sm100_range_u64_policy.gmem_threads_per_block == 768); + STATIC_REQUIRE(sm100_range_u64_policy.gmem_items_per_thread == 6); + STATIC_REQUIRE(sm100_range_u64_policy.static_smem_threads_per_block == 384); + STATIC_REQUIRE(sm100_range_u64_policy.static_smem_items_per_thread == 8); + STATIC_REQUIRE(sm100_range_u64_policy.static_smem_min_blocks_per_sm == 3); constexpr auto sm100_multi_range_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); - STATIC_REQUIRE(sm100_multi_range_policy.gmem.threads_per_block == 1024); - STATIC_REQUIRE(sm100_multi_range_policy.gmem.items_per_thread == 5); - STATIC_REQUIRE(sm100_multi_range_policy.static_smem.kernel.threads_per_block == 384); - STATIC_REQUIRE(sm100_multi_range_policy.static_smem.kernel.items_per_thread == 5); - STATIC_REQUIRE(sm100_multi_range_policy.static_smem.min_blocks_per_sm == 3); + STATIC_REQUIRE(sm100_multi_range_policy.gmem_threads_per_block == 1024); + STATIC_REQUIRE(sm100_multi_range_policy.gmem_items_per_thread == 5); + STATIC_REQUIRE(sm100_multi_range_policy.static_smem_threads_per_block == 384); + STATIC_REQUIRE(sm100_multi_range_policy.static_smem_items_per_thread == 5); + STATIC_REQUIRE(sm100_multi_range_policy.static_smem_min_blocks_per_sm == 3); STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57088, 4, 1)); STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57089, 4, 1)); @@ -1932,7 +1994,7 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" using max_policy_t = typename cub::detail::histogram::policy_hub::MaxPolicy; const auto legacy_policy = cub::detail::histogram::policy_selector_from_hub{}(cuda::compute_capability{10, 0}); - REQUIRE(legacy_policy.static_smem.max_privatized_smem_bytes == 256 * sizeof(unsigned int)); - REQUIRE(legacy_policy.dynamic_smem.max_privatized_smem_bytes == 0); + REQUIRE(legacy_policy.static_smem_max_privatized_bytes == 256 * sizeof(unsigned int)); + REQUIRE(legacy_policy.dynamic_smem_max_privatized_bytes == 0); } #endif // _CCCL_COMPILER(GCC, >=, 8) diff --git a/cub/test/catch2_test_device_histogram_env_api.cu b/cub/test/catch2_test_device_histogram_env_api.cu index ad64e49eae51..c8ca0b6914be 100644 --- a/cub/test/catch2_test_device_histogram_env_api.cu +++ b/cub/test/catch2_test_device_histogram_env_api.cu @@ -418,18 +418,37 @@ struct HistogramPolicySelector { __host__ __device__ constexpr auto operator()(cuda::compute_capability cc) const -> cub::HistogramPolicy { - const auto sweep = cub::HistogramKernelConfig{ - 128, cc > cuda::compute_capability{9, 0} ? 16 : 7, 4, cub::BLOCK_LOAD_DIRECT, cub::LOAD_LDG, false, false}; + const int items_per_thread = cc > cuda::compute_capability{9, 0} ? 16 : 7; return { - .gmem = sweep, - .static_smem = {.kernel = sweep, .max_privatized_smem_bytes = 256 * sizeof(unsigned int), .min_blocks_per_sm = 0}, - .dynamic_smem = {.kernel = sweep, - .max_privatized_smem_bytes = 0, - .range_max_bins = 0, - .even_2ch_max_bins = 0, - .even_3ch_max_bins = 0, - .even_4ch_max_bins = 0}, - .init_kernel_pdl_trigger_max_bins = 2048}; + .gmem_threads_per_block = 128, + .gmem_items_per_thread = items_per_thread, + .gmem_vec_size = 4, + .gmem_load_algorithm = cub::BLOCK_LOAD_DIRECT, + .gmem_load_modifier = cub::LOAD_LDG, + .gmem_rle_compress = false, + .gmem_work_stealing = false, + .static_smem_threads_per_block = 128, + .static_smem_items_per_thread = items_per_thread, + .static_smem_vec_size = 4, + .static_smem_load_algorithm = cub::BLOCK_LOAD_DIRECT, + .static_smem_load_modifier = cub::LOAD_LDG, + .static_smem_rle_compress = false, + .static_smem_work_stealing = false, + .static_smem_max_privatized_bytes = 256 * sizeof(unsigned int), + .static_smem_min_blocks_per_sm = 0, + .dynamic_smem_threads_per_block = 128, + .dynamic_smem_items_per_thread = items_per_thread, + .dynamic_smem_vec_size = 4, + .dynamic_smem_load_algorithm = cub::BLOCK_LOAD_DIRECT, + .dynamic_smem_load_modifier = cub::LOAD_LDG, + .dynamic_smem_rle_compress = false, + .dynamic_smem_work_stealing = false, + .dynamic_smem_max_privatized_bytes = 0, + .dynamic_smem_range_max_bins = 0, + .dynamic_smem_even_2ch_max_bins = 0, + .dynamic_smem_even_3ch_max_bins = 0, + .dynamic_smem_even_4ch_max_bins = 0, + .init_kernel_pdl_trigger_max_bins = 2048}; } }; // example-end histogram-even-policy-selector From 47c719ca0f0290c52865a268ef3d0dc36b0d6557 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Wed, 5 Aug 2026 13:24:21 +0000 Subject: [PATCH 29/45] [cub] Remove the default legacy histogram policy chain --- cub/cub/agent/agent_histogram.cuh | 24 +- .../device/dispatch/dispatch_histogram.cuh | 58 +++-- .../dispatch/tuning/tuning_histogram.cuh | 206 ------------------ cub/test/catch2_test_device_histogram_env.cu | 6 - 4 files changed, 30 insertions(+), 264 deletions(-) diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index 550f471343a4..a2c71ee48f2f 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -30,8 +30,7 @@ CUB_NAMESPACE_BEGIN -namespace detail -{ +//! Deprecated [Since 3.5] template -struct legacy_agent_histogram_policy +struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") AgentHistogramPolicy { static constexpr int BLOCK_THREADS = ThreadsPerBlock; static constexpr int PIXELS_PER_THREAD = PixelsPerThread; @@ -50,25 +49,6 @@ struct legacy_agent_histogram_policy static constexpr BlockLoadAlgorithm LOAD_ALGORITHM = LoadAlgorithm; static constexpr CacheLoadModifier LOAD_MODIFIER = LoadModifier; }; -} // namespace detail - -//! Deprecated [Since 3.5] -template -using AgentHistogramPolicy - CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") = detail::legacy_agent_histogram_policy< - ThreadsPerBlock, - PixelsPerThread, - LoadAlgorithm, - LoadModifier, - RleCompress, - WorkStealing, - VecSize>; namespace detail::histogram { diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 515a93f4046a..5854e007ab2f 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -855,6 +855,18 @@ public: } }; +template +struct max_policy_from_hub +{ + using type = typename PolicyHub::MaxPolicy; +}; + +template <> +struct max_policy_from_hub +{ + using type = void; +}; + template 0. */ - template , - /* fallback_policy_hub */ - detail::histogram::policy_hub, - PolicyHub>::MaxPolicy, - bool IsByteSample> + template ::type, bool IsByteSample> CUB_RUNTIME_FUNCTION static cudaError_t DispatchRange( void* d_temp_storage, size_t& temp_storage_bytes, @@ -1450,16 +1457,14 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc ::cuda::std::bool_constant is_byte_sample, KernelSource kernel_source = {}, KernelLauncherFactory launcher_factory = {}, - [[maybe_unused]] MaxPolicyT max_policy = {}) + [[maybe_unused]] ::cuda::std::_If<::cuda::std::is_void_v, ::cuda::std::nullptr_t, MaxPolicyT> + max_policy = {}) { - using default_policy_hub = - detail::histogram::policy_hub; - static constexpr bool uses_default_policy = - ::cuda::std::is_void_v && ::cuda::std::is_same_v; - using policy_selector_t = ::cuda::std::_If< - uses_default_policy, - detail::histogram::policy_selector_from_types, - detail::histogram::policy_selector_from_hub>; + static constexpr bool uses_default_policy = ::cuda::std::is_void_v; + using policy_selector_t = ::cuda::std::_If< + uses_default_policy, + detail::histogram::policy_selector_from_types, + detail::histogram::policy_selector_from_hub>; return detail::histogram::dispatch_range( d_temp_storage, temp_storage_bytes, @@ -1524,12 +1529,7 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc * CUDA stream to launch kernels within. Default is stream0. * */ - template , - /* fallback_policy_hub */ - detail::histogram::policy_hub, - PolicyHub>::MaxPolicy, - bool IsByteSample> + template ::type, bool IsByteSample> CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t DispatchEven( void* d_temp_storage, size_t& temp_storage_bytes, @@ -1545,16 +1545,14 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc ::cuda::std::bool_constant is_byte_sample, KernelSource kernel_source = {}, KernelLauncherFactory launcher_factory = {}, - [[maybe_unused]] MaxPolicyT max_policy = {}) + [[maybe_unused]] ::cuda::std::_If<::cuda::std::is_void_v, ::cuda::std::nullptr_t, MaxPolicyT> + max_policy = {}) { - using default_policy_hub = - detail::histogram::policy_hub; - static constexpr bool uses_default_policy = - ::cuda::std::is_void_v && ::cuda::std::is_same_v; - using policy_selector_t = ::cuda::std::_If< - uses_default_policy, - detail::histogram::policy_selector_from_types, - detail::histogram::policy_selector_from_hub>; + static constexpr bool uses_default_policy = ::cuda::std::is_void_v; + using policy_selector_t = ::cuda::std::_If< + uses_default_policy, + detail::histogram::policy_selector_from_types, + detail::histogram::policy_selector_from_hub>; return detail::histogram::dispatch_even( d_temp_storage, temp_storage_bytes, diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 2c04a46b6c41..ab0d9283c6a5 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -251,212 +251,6 @@ should_use_dynamic_smem(const HistogramPolicy& policy, int num_bins, int counter return prefer_dynamic_smem && max_bins > 0 && num_bins <= max_bins; } -// TODO(bgruber): drop in CCCL 4.0 -enum class primitive_sample -{ - no, - yes -}; - -// TODO(bgruber): drop in CCCL 4.0 -enum class sample_size -{ - _1, - _2, - _4, - _8, - unknown -}; - -// TODO(bgruber): drop in CCCL 4.0 -enum class counter_size -{ - _4, - unknown -}; - -// TODO(bgruber): drop in CCCL 4.0 -template -_CCCL_HOST_DEVICE_API constexpr primitive_sample is_primitive_sample() -{ - return is_primitive::value ? primitive_sample::yes : primitive_sample::no; -} - -// TODO(bgruber): drop in CCCL 4.0 -template -_CCCL_HOST_DEVICE_API constexpr counter_size classify_counter_size() -{ - return sizeof(CounterT) == 4 ? counter_size::_4 : counter_size::unknown; -} - -// TODO(bgruber): drop in CCCL 4.0 -template -_CCCL_HOST_DEVICE_API constexpr sample_size classify_sample_size() -{ - return sizeof(SampleT) == 1 ? sample_size::_1 - : sizeof(SampleT) == 2 ? sample_size::_2 - : sizeof(SampleT) == 4 ? sample_size::_4 - : sizeof(SampleT) == 8 - ? sample_size::_8 - : sample_size::unknown; -} - -// TODO(bgruber): drop in CCCL 4.0 -template (), - sample_size SampleSize = classify_sample_size()> -struct sm90_tuning; - -template -struct sm90_tuning -{ - static constexpr int threads_per_block = 768; - static constexpr int items_per_thread = 12; - - static constexpr CacheLoadModifier load_modifier = LOAD_LDG; - - static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_DIRECT; - - static constexpr bool rle_compress = false; - static constexpr bool work_stealing = false; -}; - -template -struct sm90_tuning -{ - static constexpr int threads_per_block = 960; - static constexpr int items_per_thread = 10; - - static constexpr CacheLoadModifier load_modifier = LOAD_DEFAULT; - - static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_DIRECT; - - static constexpr bool rle_compress = true; - static constexpr bool work_stealing = false; -}; - -// TODO(bgruber): drop in CCCL 4.0 -template (), - sample_size SampleSize = classify_sample_size()> -struct sm100_tuning; - -// even -template -struct sm100_tuning -{ - // ipt_12.tpb_928.rle_0.ws_0.mem_1.ld_2.laid_0.vec_2 1.033332 0.940517 1.031835 1.195876 - static constexpr int items_per_thread = 12; - static constexpr int threads_per_block = 928; - static constexpr bool rle_compress = false; - static constexpr bool work_stealing = false; - static constexpr CacheLoadModifier load_modifier = LOAD_CA; - static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_DIRECT; - static constexpr int vec_size = 1 << 2; -}; - -// range -template -struct sm100_tuning -{ - // ipt_12.tpb_448.rle_0.ws_0.mem_1.ld_1.laid_0.vec_2 1.078987 0.985542 1.085118 1.175637 - static constexpr int items_per_thread = 12; - static constexpr int threads_per_block = 448; - static constexpr bool rle_compress = false; - static constexpr bool work_stealing = false; - static constexpr CacheLoadModifier load_modifier = LOAD_LDG; - static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_DIRECT; - static constexpr int vec_size = 1 << 2; -}; - -// sample_size 2/4/8 retain the SM90 launch shape in the legacy policy hub. - -// TODO(bgruber): drop in CCCL 4.0 -template -struct policy_hub -{ - // TODO(bgruber): move inside t_scale in C++14 - static constexpr int v_scale = (sizeof(SampleT) + sizeof(int) - 1) / sizeof(int); - - _CCCL_HOST_DEVICE_API static constexpr int t_scale(int nominalItemsPerThread) - { - return (::cuda::std::max) (nominalItemsPerThread / NumActiveChannels / v_scale, 1); - } - - // SM50 - struct Policy500 : detail::chained_policy<500, Policy500, Policy500> - { - // TODO This might be worth it to separate usual histogram and the multi one - using AgentHistogramPolicyT = - legacy_agent_histogram_policy<384, t_scale(16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, false>; - static constexpr int init_kernel_pdl_trigger_max_bins = 0; - }; - - // SM90 - struct Policy900 : detail::chained_policy<900, Policy900, Policy500> - { - // Use values from tuning if a specialization exists, otherwise pick Policy500 - template - _CCCL_HOST_DEVICE_API static auto select_agent_policy(int) - -> legacy_agent_histogram_policy; - - template - _CCCL_HOST_DEVICE_API static auto select_agent_policy(long) -> typename Policy500::AgentHistogramPolicyT; - - using AgentHistogramPolicyT = - decltype(select_agent_policy< - sm90_tuning()>>(0)); - - static constexpr int init_kernel_pdl_trigger_max_bins = - NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value - && (sizeof(SampleT) == 1 || sizeof(SampleT) == 2) - ? pdl_trigger_max_bins - : 0; - }; - - struct Policy1000 : detail::chained_policy<1000, Policy1000, Policy900> - { - // Use values from tuning if a specialization exists, otherwise pick Policy900 - template - _CCCL_HOST_DEVICE_API static auto select_agent_policy(int) -> legacy_agent_histogram_policy< - Tuning::threads_per_block, - Tuning::items_per_thread, - Tuning::load_algorithm, - Tuning::load_modifier, - Tuning::rle_compress, - Tuning::work_stealing, - Tuning::vec_size>; - - template - _CCCL_HOST_DEVICE_API static auto select_agent_policy(long) -> typename Policy900::AgentHistogramPolicyT; - - using AgentHistogramPolicyT = - decltype(select_agent_policy< - sm100_tuning()>>( - 0)); - - static constexpr int init_kernel_pdl_trigger_max_bins = - NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value - && (sizeof(SampleT) == 1 || sizeof(SampleT) == 2 || sizeof(SampleT) == 4 || sizeof(SampleT) == 8) - ? pdl_trigger_max_bins - : 0; - }; - - using MaxPolicy = Policy1000; -}; - #if _CCCL_HAS_CONCEPTS() template concept histogram_policy_selector = policy_selector; diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index 7542d6037418..bddd3bfeb054 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1990,11 +1990,5 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 28545, 4, 2)); STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 19029, 4, 3)); STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 19030, 4, 3)); - - using max_policy_t = typename cub::detail::histogram::policy_hub::MaxPolicy; - const auto legacy_policy = - cub::detail::histogram::policy_selector_from_hub{}(cuda::compute_capability{10, 0}); - REQUIRE(legacy_policy.static_smem_max_privatized_bytes == 256 * sizeof(unsigned int)); - REQUIRE(legacy_policy.dynamic_smem_max_privatized_bytes == 0); } #endif // _CCCL_COMPILER(GCC, >=, 8) From 3cb975e82c5962adc5ec81e41d35d81af640d2a0 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Wed, 5 Aug 2026 13:36:55 +0000 Subject: [PATCH 30/45] Revert "[cub] Remove the default legacy histogram policy chain" This reverts commit 3c48046c8242d470c747ab3f2050f543c64f14ed. --- cub/cub/agent/agent_histogram.cuh | 24 +- .../device/dispatch/dispatch_histogram.cuh | 58 ++--- .../dispatch/tuning/tuning_histogram.cuh | 206 ++++++++++++++++++ cub/test/catch2_test_device_histogram_env.cu | 6 + 4 files changed, 264 insertions(+), 30 deletions(-) diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index a2c71ee48f2f..550f471343a4 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -30,7 +30,8 @@ CUB_NAMESPACE_BEGIN -//! Deprecated [Since 3.5] +namespace detail +{ template -struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") AgentHistogramPolicy +struct legacy_agent_histogram_policy { static constexpr int BLOCK_THREADS = ThreadsPerBlock; static constexpr int PIXELS_PER_THREAD = PixelsPerThread; @@ -49,6 +50,25 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") AgentHi static constexpr BlockLoadAlgorithm LOAD_ALGORITHM = LoadAlgorithm; static constexpr CacheLoadModifier LOAD_MODIFIER = LoadModifier; }; +} // namespace detail + +//! Deprecated [Since 3.5] +template +using AgentHistogramPolicy + CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") = detail::legacy_agent_histogram_policy< + ThreadsPerBlock, + PixelsPerThread, + LoadAlgorithm, + LoadModifier, + RleCompress, + WorkStealing, + VecSize>; namespace detail::histogram { diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 5854e007ab2f..515a93f4046a 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -855,18 +855,6 @@ public: } }; -template -struct max_policy_from_hub -{ - using type = typename PolicyHub::MaxPolicy; -}; - -template <> -struct max_policy_from_hub -{ - using type = void; -}; - template 0. */ - template ::type, bool IsByteSample> + template , + /* fallback_policy_hub */ + detail::histogram::policy_hub, + PolicyHub>::MaxPolicy, + bool IsByteSample> CUB_RUNTIME_FUNCTION static cudaError_t DispatchRange( void* d_temp_storage, size_t& temp_storage_bytes, @@ -1457,14 +1450,16 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc ::cuda::std::bool_constant is_byte_sample, KernelSource kernel_source = {}, KernelLauncherFactory launcher_factory = {}, - [[maybe_unused]] ::cuda::std::_If<::cuda::std::is_void_v, ::cuda::std::nullptr_t, MaxPolicyT> - max_policy = {}) + [[maybe_unused]] MaxPolicyT max_policy = {}) { - static constexpr bool uses_default_policy = ::cuda::std::is_void_v; - using policy_selector_t = ::cuda::std::_If< - uses_default_policy, - detail::histogram::policy_selector_from_types, - detail::histogram::policy_selector_from_hub>; + using default_policy_hub = + detail::histogram::policy_hub; + static constexpr bool uses_default_policy = + ::cuda::std::is_void_v && ::cuda::std::is_same_v; + using policy_selector_t = ::cuda::std::_If< + uses_default_policy, + detail::histogram::policy_selector_from_types, + detail::histogram::policy_selector_from_hub>; return detail::histogram::dispatch_range( d_temp_storage, temp_storage_bytes, @@ -1529,7 +1524,12 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc * CUDA stream to launch kernels within. Default is stream0. * */ - template ::type, bool IsByteSample> + template , + /* fallback_policy_hub */ + detail::histogram::policy_hub, + PolicyHub>::MaxPolicy, + bool IsByteSample> CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t DispatchEven( void* d_temp_storage, size_t& temp_storage_bytes, @@ -1545,14 +1545,16 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc ::cuda::std::bool_constant is_byte_sample, KernelSource kernel_source = {}, KernelLauncherFactory launcher_factory = {}, - [[maybe_unused]] ::cuda::std::_If<::cuda::std::is_void_v, ::cuda::std::nullptr_t, MaxPolicyT> - max_policy = {}) + [[maybe_unused]] MaxPolicyT max_policy = {}) { - static constexpr bool uses_default_policy = ::cuda::std::is_void_v; - using policy_selector_t = ::cuda::std::_If< - uses_default_policy, - detail::histogram::policy_selector_from_types, - detail::histogram::policy_selector_from_hub>; + using default_policy_hub = + detail::histogram::policy_hub; + static constexpr bool uses_default_policy = + ::cuda::std::is_void_v && ::cuda::std::is_same_v; + using policy_selector_t = ::cuda::std::_If< + uses_default_policy, + detail::histogram::policy_selector_from_types, + detail::histogram::policy_selector_from_hub>; return detail::histogram::dispatch_even( d_temp_storage, temp_storage_bytes, diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index ab0d9283c6a5..2c04a46b6c41 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -251,6 +251,212 @@ should_use_dynamic_smem(const HistogramPolicy& policy, int num_bins, int counter return prefer_dynamic_smem && max_bins > 0 && num_bins <= max_bins; } +// TODO(bgruber): drop in CCCL 4.0 +enum class primitive_sample +{ + no, + yes +}; + +// TODO(bgruber): drop in CCCL 4.0 +enum class sample_size +{ + _1, + _2, + _4, + _8, + unknown +}; + +// TODO(bgruber): drop in CCCL 4.0 +enum class counter_size +{ + _4, + unknown +}; + +// TODO(bgruber): drop in CCCL 4.0 +template +_CCCL_HOST_DEVICE_API constexpr primitive_sample is_primitive_sample() +{ + return is_primitive::value ? primitive_sample::yes : primitive_sample::no; +} + +// TODO(bgruber): drop in CCCL 4.0 +template +_CCCL_HOST_DEVICE_API constexpr counter_size classify_counter_size() +{ + return sizeof(CounterT) == 4 ? counter_size::_4 : counter_size::unknown; +} + +// TODO(bgruber): drop in CCCL 4.0 +template +_CCCL_HOST_DEVICE_API constexpr sample_size classify_sample_size() +{ + return sizeof(SampleT) == 1 ? sample_size::_1 + : sizeof(SampleT) == 2 ? sample_size::_2 + : sizeof(SampleT) == 4 ? sample_size::_4 + : sizeof(SampleT) == 8 + ? sample_size::_8 + : sample_size::unknown; +} + +// TODO(bgruber): drop in CCCL 4.0 +template (), + sample_size SampleSize = classify_sample_size()> +struct sm90_tuning; + +template +struct sm90_tuning +{ + static constexpr int threads_per_block = 768; + static constexpr int items_per_thread = 12; + + static constexpr CacheLoadModifier load_modifier = LOAD_LDG; + + static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_DIRECT; + + static constexpr bool rle_compress = false; + static constexpr bool work_stealing = false; +}; + +template +struct sm90_tuning +{ + static constexpr int threads_per_block = 960; + static constexpr int items_per_thread = 10; + + static constexpr CacheLoadModifier load_modifier = LOAD_DEFAULT; + + static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_DIRECT; + + static constexpr bool rle_compress = true; + static constexpr bool work_stealing = false; +}; + +// TODO(bgruber): drop in CCCL 4.0 +template (), + sample_size SampleSize = classify_sample_size()> +struct sm100_tuning; + +// even +template +struct sm100_tuning +{ + // ipt_12.tpb_928.rle_0.ws_0.mem_1.ld_2.laid_0.vec_2 1.033332 0.940517 1.031835 1.195876 + static constexpr int items_per_thread = 12; + static constexpr int threads_per_block = 928; + static constexpr bool rle_compress = false; + static constexpr bool work_stealing = false; + static constexpr CacheLoadModifier load_modifier = LOAD_CA; + static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_DIRECT; + static constexpr int vec_size = 1 << 2; +}; + +// range +template +struct sm100_tuning +{ + // ipt_12.tpb_448.rle_0.ws_0.mem_1.ld_1.laid_0.vec_2 1.078987 0.985542 1.085118 1.175637 + static constexpr int items_per_thread = 12; + static constexpr int threads_per_block = 448; + static constexpr bool rle_compress = false; + static constexpr bool work_stealing = false; + static constexpr CacheLoadModifier load_modifier = LOAD_LDG; + static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_DIRECT; + static constexpr int vec_size = 1 << 2; +}; + +// sample_size 2/4/8 retain the SM90 launch shape in the legacy policy hub. + +// TODO(bgruber): drop in CCCL 4.0 +template +struct policy_hub +{ + // TODO(bgruber): move inside t_scale in C++14 + static constexpr int v_scale = (sizeof(SampleT) + sizeof(int) - 1) / sizeof(int); + + _CCCL_HOST_DEVICE_API static constexpr int t_scale(int nominalItemsPerThread) + { + return (::cuda::std::max) (nominalItemsPerThread / NumActiveChannels / v_scale, 1); + } + + // SM50 + struct Policy500 : detail::chained_policy<500, Policy500, Policy500> + { + // TODO This might be worth it to separate usual histogram and the multi one + using AgentHistogramPolicyT = + legacy_agent_histogram_policy<384, t_scale(16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, false>; + static constexpr int init_kernel_pdl_trigger_max_bins = 0; + }; + + // SM90 + struct Policy900 : detail::chained_policy<900, Policy900, Policy500> + { + // Use values from tuning if a specialization exists, otherwise pick Policy500 + template + _CCCL_HOST_DEVICE_API static auto select_agent_policy(int) + -> legacy_agent_histogram_policy; + + template + _CCCL_HOST_DEVICE_API static auto select_agent_policy(long) -> typename Policy500::AgentHistogramPolicyT; + + using AgentHistogramPolicyT = + decltype(select_agent_policy< + sm90_tuning()>>(0)); + + static constexpr int init_kernel_pdl_trigger_max_bins = + NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value + && (sizeof(SampleT) == 1 || sizeof(SampleT) == 2) + ? pdl_trigger_max_bins + : 0; + }; + + struct Policy1000 : detail::chained_policy<1000, Policy1000, Policy900> + { + // Use values from tuning if a specialization exists, otherwise pick Policy900 + template + _CCCL_HOST_DEVICE_API static auto select_agent_policy(int) -> legacy_agent_histogram_policy< + Tuning::threads_per_block, + Tuning::items_per_thread, + Tuning::load_algorithm, + Tuning::load_modifier, + Tuning::rle_compress, + Tuning::work_stealing, + Tuning::vec_size>; + + template + _CCCL_HOST_DEVICE_API static auto select_agent_policy(long) -> typename Policy900::AgentHistogramPolicyT; + + using AgentHistogramPolicyT = + decltype(select_agent_policy< + sm100_tuning()>>( + 0)); + + static constexpr int init_kernel_pdl_trigger_max_bins = + NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value + && (sizeof(SampleT) == 1 || sizeof(SampleT) == 2 || sizeof(SampleT) == 4 || sizeof(SampleT) == 8) + ? pdl_trigger_max_bins + : 0; + }; + + using MaxPolicy = Policy1000; +}; + #if _CCCL_HAS_CONCEPTS() template concept histogram_policy_selector = policy_selector; diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index bddd3bfeb054..7542d6037418 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1990,5 +1990,11 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 28545, 4, 2)); STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 19029, 4, 3)); STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 19030, 4, 3)); + + using max_policy_t = typename cub::detail::histogram::policy_hub::MaxPolicy; + const auto legacy_policy = + cub::detail::histogram::policy_selector_from_hub{}(cuda::compute_capability{10, 0}); + REQUIRE(legacy_policy.static_smem_max_privatized_bytes == 256 * sizeof(unsigned int)); + REQUIRE(legacy_policy.dynamic_smem_max_privatized_bytes == 0); } #endif // _CCCL_COMPILER(GCC, >=, 8) From b1529673db823f3b2c48753f544d5442da3870f7 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Wed, 5 Aug 2026 13:36:55 +0000 Subject: [PATCH 31/45] Revert "[cub] Collapse histogram tuning into one policy" This reverts commit 7638db911ed440397c6f13eab98536131e1908ba. --- .../bench/histogram/histogram_common.cuh | 35 +- cub/cub/agent/agent_histogram.cuh | 100 ++-- .../device/dispatch/dispatch_histogram.cuh | 36 +- .../dispatch/kernels/kernel_histogram.cuh | 134 ++--- .../dispatch/tuning/tuning_histogram.cuh | 476 +++++++----------- cub/test/catch2_test_device_histogram_env.cu | 226 +++------ .../catch2_test_device_histogram_env_api.cu | 41 +- 7 files changed, 411 insertions(+), 637 deletions(-) diff --git a/cub/benchmarks/bench/histogram/histogram_common.cuh b/cub/benchmarks/bench/histogram/histogram_common.cuh index ab5f23c8524e..55c67f27688c 100644 --- a/cub/benchmarks/bench/histogram/histogram_common.cuh +++ b/cub/benchmarks/bench/histogram/histogram_common.cuh @@ -37,36 +37,23 @@ struct bench_policy_selector ? (NUM_CHANNELS == 1 ? cub::BLOCK_LOAD_STRIPED : cub::BLOCK_LOAD_DIRECT) : TUNE_LOAD_ALGORITHM; - return { + constexpr auto sweep = cub::HistogramKernelConfig{ TUNE_THREADS, TUNE_ITEMS, TUNE_VEC_SIZE, load_algorithm, TUNE_LOAD_MODIFIER, TUNE_RLE_COMPRESS, - TUNE_WORK_STEALING, - TUNE_THREADS, - TUNE_ITEMS, - TUNE_VEC_SIZE, - load_algorithm, - TUNE_LOAD_MODIFIER, - TUNE_RLE_COMPRESS, - TUNE_WORK_STEALING, - TUNE_STATIC_SMEM_MAX_BYTES, - TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM, - TUNE_THREADS, - TUNE_ITEMS, - TUNE_VEC_SIZE, - load_algorithm, - TUNE_LOAD_MODIFIER, - TUNE_RLE_COMPRESS, - TUNE_WORK_STEALING, - TUNE_DYNAMIC_SMEM_MAX_BYTES, - TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS, - TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS, - TUNE_DYNAMIC_SMEM_EVEN_3CH_MAX_BINS, - TUNE_DYNAMIC_SMEM_EVEN_4CH_MAX_BINS, - TUNE_INIT_KERNEL_PDL_TRIGGER_MAX_BINS}; + TUNE_WORK_STEALING}; + return {sweep, + {sweep, TUNE_STATIC_SMEM_MAX_BYTES, TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM}, + {sweep, + TUNE_DYNAMIC_SMEM_MAX_BYTES, + TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS, + TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS, + TUNE_DYNAMIC_SMEM_EVEN_3CH_MAX_BINS, + TUNE_DYNAMIC_SMEM_EVEN_4CH_MAX_BINS}, + TUNE_INIT_KERNEL_PDL_TRIGGER_MAX_BINS}; } }; #endif // !TUNE_BASE diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index 550f471343a4..e7ef066d9c95 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -32,23 +32,40 @@ CUB_NAMESPACE_BEGIN namespace detail { +//! Parameterizable tuning policy type for AgentHistogram template -struct legacy_agent_histogram_policy + int VecSize = 4, + int PrivatizedStaticSmemBytes = 0> +struct agent_histogram_policy { - static constexpr int BLOCK_THREADS = ThreadsPerBlock; + /// Threads per thread block + static constexpr int BLOCK_THREADS = ThreadsPerBlock; + /// Pixels per thread (per tile of input) static constexpr int PIXELS_PER_THREAD = PixelsPerThread; - static constexpr bool IS_RLE_COMPRESS = RleCompress; + + /// Whether to perform localized RLE to compress samples before histogramming + static constexpr bool IS_RLE_COMPRESS = RleCompress; + + /// Whether to dequeue tiles from a global work queue static constexpr bool IS_WORK_STEALING = WorkStealing; - static constexpr int VEC_SIZE = VecSize; + + /// Maximum compile-time-sized shared-memory allocation for privatized bins + static constexpr int PRIVATIZED_STATIC_SMEM_BYTES = PrivatizedStaticSmemBytes; + + /// Vector size for samples loading (1, 2, 4) + static constexpr int VEC_SIZE = VecSize; static_assert(VEC_SIZE == 1 || VEC_SIZE == 2 || VEC_SIZE == 4); + + ///< The BlockLoad algorithm to use static constexpr BlockLoadAlgorithm LOAD_ALGORITHM = LoadAlgorithm; - static constexpr CacheLoadModifier LOAD_MODIFIER = LoadModifier; + + ///< Cache load modifier for reading input elements + static constexpr CacheLoadModifier LOAD_MODIFIER = LoadModifier; }; } // namespace detail @@ -60,15 +77,8 @@ template -using AgentHistogramPolicy - CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") = detail::legacy_agent_histogram_policy< - ThreadsPerBlock, - PixelsPerThread, - LoadAlgorithm, - LoadModifier, - RleCompress, - WorkStealing, - VecSize>; +using AgentHistogramPolicy CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") = detail:: + agent_histogram_policy; namespace detail::histogram { @@ -113,29 +123,8 @@ _CCCL_DEVICE _CCCL_FORCEINLINE auto NativePointer(IteratorT itr) //! @brief AgentHistogram implements a stateful abstraction of CUDA thread blocks for participating //! in device-wide histogram . //! -//! @tparam ThreadsPerBlock -//! Number of threads in a thread block. -//! -//! @tparam PixelsPerThread -//! Number of pixels processed per thread. -//! -//! @tparam LoadAlgorithm -//! BlockLoad algorithm used to load samples. -//! -//! @tparam LoadModifier -//! Cache modifier used to load samples. -//! -//! @tparam RleCompress -//! Whether to locally run-length encode samples. -//! -//! @tparam WorkStealing -//! Whether blocks dequeue work from a global queue. -//! -//! @tparam VecSize -//! Vector width used to load samples. -//! -//! @tparam PrivatizedStaticSmemBytes -//! Compile-time-sized shared-memory allocation, or zero for another storage mode. +//! @tparam AgentHistogramPolicyT +//! Parameterized AgentHistogramPolicy tuning policy type //! //! @tparam PrivatizationMode //! Storage mode for the privatized histogram. @@ -165,14 +154,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE auto NativePointer(IteratorT itr) //! //! @tparam OutputCounterT //! Integer type for final output histogram bins. May be wider than `CounterT`. -template ; static constexpr bool uses_gmem = is_privatized_gmem_v; static constexpr int privatized_static_smem_bins = - uses_static_smem ? PrivatizedStaticSmemBytes / int{sizeof(CounterT)} / NumActiveChannels : 0; + uses_static_smem ? AgentHistogramPolicyT::PRIVATIZED_STATIC_SMEM_BYTES / int{sizeof(CounterT)} / NumActiveChannels + : 0; static_assert(!uses_static_smem || privatized_static_smem_bins > 0, "Static-SMEM privatization requires room for at least one bin"); - static constexpr int vec_size = VecSize; - static constexpr int threads_per_block = ThreadsPerBlock; - static constexpr int pixels_per_thread = PixelsPerThread; + static constexpr int vec_size = AgentHistogramPolicyT::VEC_SIZE; + static constexpr int threads_per_block = AgentHistogramPolicyT::BLOCK_THREADS; + static constexpr int pixels_per_thread = AgentHistogramPolicyT::PIXELS_PER_THREAD; static constexpr int samples_per_thread = pixels_per_thread * NumChannels; static constexpr int vecs_per_thread = samples_per_thread / vec_size; static constexpr int tile_pixels = pixels_per_thread * threads_per_block; static constexpr int tile_samples = samples_per_thread * threads_per_block; - static constexpr bool is_rle_compress = RleCompress; - static constexpr bool is_work_stealing = WorkStealing; - static constexpr CacheLoadModifier load_modifier = LoadModifier; - static_assert(vec_size == 1 || vec_size == 2 || vec_size == 4); + static constexpr bool is_rle_compress = AgentHistogramPolicyT::IS_RLE_COMPRESS; + static constexpr bool is_work_stealing = AgentHistogramPolicyT::IS_WORK_STEALING; + static constexpr CacheLoadModifier load_modifier = AgentHistogramPolicyT::LOAD_MODIFIER; using SampleT = it_value_t; using PixelT = typename CubVector::Type; @@ -220,9 +202,11 @@ struct AgentHistogram SampleIteratorT>; using WrappedPixelIteratorT = CacheModifiedInputIterator; using WrappedVecsIteratorT = CacheModifiedInputIterator; - using BlockLoadSampleT = BlockLoad; - using BlockLoadPixelT = BlockLoad; - using BlockLoadVecT = BlockLoad; + using BlockLoadSampleT = + BlockLoad; + using BlockLoadPixelT = + BlockLoad; + using BlockLoadVecT = BlockLoad; struct _TempStorage { @@ -445,7 +429,7 @@ struct AgentHistogram bool is_valid[pixels_per_thread]; LoadTile(block_offset, valid_samples, samples); - MarkValid(is_valid, valid_samples); + MarkValid(is_valid, valid_samples); AccumulatePixels(samples, is_valid, ::cuda::std::bool_constant{}); } diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 515a93f4046a..acb492b60cf0 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -276,8 +276,9 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( } }(); - const int threads_per_block = detail::histogram::threads_per_block(active_policy, PrivatizationMode{}); - const int items_per_thread = detail::histogram::items_per_thread(active_policy, PrivatizationMode{}); + const HistogramKernelConfig sweep = kernel_config(active_policy, PrivatizationMode{}); + const int threads_per_block = sweep.threads_per_block; + const int items_per_thread = sweep.items_per_thread; int dynamic_smem_bytes = 0; if constexpr (is_privatized_dynamic_smem_v) @@ -288,7 +289,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( } NV_IF_TARGET(NV_IS_HOST, ({ if (const auto error = CubDebug(launcher_factory.set_max_dynamic_smem_size_for( - sweep_kernel, active_policy.dynamic_smem_max_privatized_bytes))) + sweep_kernel, active_policy.dynamic_smem.max_privatized_smem_bytes))) { return error; } @@ -791,37 +792,16 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device template _CCCL_HOST_DEVICE_API constexpr auto convert_legacy_policy() -> HistogramPolicy { - using sweep = typename ActivePolicy::AgentHistogramPolicyT; - return { + using sweep = typename ActivePolicy::AgentHistogramPolicyT; + const auto kernel_config = HistogramKernelConfig{ sweep::BLOCK_THREADS, sweep::PIXELS_PER_THREAD, sweep::VEC_SIZE, sweep::LOAD_ALGORITHM, sweep::LOAD_MODIFIER, sweep::IS_RLE_COMPRESS, - sweep::IS_WORK_STEALING, - sweep::BLOCK_THREADS, - sweep::PIXELS_PER_THREAD, - sweep::VEC_SIZE, - sweep::LOAD_ALGORITHM, - sweep::LOAD_MODIFIER, - sweep::IS_RLE_COMPRESS, - sweep::IS_WORK_STEALING, - 256 * sizeof(unsigned int), - 0, - sweep::BLOCK_THREADS, - sweep::PIXELS_PER_THREAD, - sweep::VEC_SIZE, - sweep::LOAD_ALGORITHM, - sweep::LOAD_MODIFIER, - sweep::IS_RLE_COMPRESS, - sweep::IS_WORK_STEALING, - 0, - 0, - 0, - 0, - 0, - 0}; + sweep::IS_WORK_STEALING}; + return {kernel_config, {kernel_config, 256 * sizeof(unsigned int), 0}, {kernel_config, 0, 0, 0, 0, 0}, 0}; } // TODO(bgruber): drop in CCCL 4.0 diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index 82508a4993c1..fef7f13a2342 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -704,9 +704,9 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(threads_per_block(current_policy(), PrivatizationMode{})), +__launch_bounds__(int(kernel_config(current_policy(), PrivatizationMode{}).threads_per_block), int(is_privatized_static_smem_v - ? current_policy().static_smem_min_blocks_per_sm + ? current_policy().static_smem.min_blocks_per_sm : 0)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepKernel( const SampleIteratorT d_samples, @@ -722,25 +722,30 @@ __launch_bounds__(int(threads_per_block(current_policy(), Privat const int tiles_per_row, GridQueue tile_queue) { - using AgentHistogramT = AgentHistogram< - threads_per_block(current_policy(), PrivatizationMode{}), - items_per_thread(current_policy(), PrivatizationMode{}), - load_algorithm(current_policy(), PrivatizationMode{}), - load_modifier(current_policy(), PrivatizationMode{}), - rle_compress(current_policy(), PrivatizationMode{}), - work_stealing(current_policy(), PrivatizationMode{}), - vec_size(current_policy(), PrivatizationMode{}), - is_privatized_static_smem_v ? current_policy().static_smem_max_privatized_bytes - : 0, - PrivatizationMode, - NumChannels, - NumActiveChannels, - SampleIteratorT, - CounterT, - PrivatizedDecodeOpT, - OutputDecodeOpT, - OffsetT, - OutputCounterT>; + static constexpr HistogramPolicy hp = current_policy(); + static constexpr auto sweep = kernel_config(hp, PrivatizationMode{}); + + // Thread block type for compositing input tiles + using AgentHistogramPolicyT = agent_histogram_policy< + sweep.threads_per_block, + sweep.items_per_thread, + sweep.load_algorithm, + sweep.load_modifier, + sweep.rle_compress, + sweep.work_stealing, + sweep.vec_size, + is_privatized_static_smem_v ? hp.static_smem.max_privatized_smem_bytes : 0>; + using AgentHistogramT = + AgentHistogram; // Shared memory for AgentHistogram __shared__ typename AgentHistogramT::TempStorage static_smem_storage; @@ -785,7 +790,7 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(current_policy().dynamic_smem_threads_per_block)) +__launch_bounds__(int(current_policy().dynamic_smem.kernel.threads_per_block)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDynamicSmemKernel( const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, @@ -800,24 +805,28 @@ __launch_bounds__(int(current_policy().dynamic_smem_threads_per_ const int tiles_per_row, GridQueue tile_queue) { - using AgentHistogramT = AgentHistogram< - current_policy().dynamic_smem_threads_per_block, - current_policy().dynamic_smem_items_per_thread, - current_policy().dynamic_smem_load_algorithm, - current_policy().dynamic_smem_load_modifier, - current_policy().dynamic_smem_rle_compress, - current_policy().dynamic_smem_work_stealing, - current_policy().dynamic_smem_vec_size, - 0, - HistogramPrivatizedDynamicSmem, - NumChannels, - NumActiveChannels, - SampleIteratorT, - CounterT, - PrivatizedDecodeOpT, - OutputDecodeOpT, - OffsetT, - OutputCounterT>; + static constexpr HistogramPolicy hp = current_policy(); + static constexpr HistogramKernelConfig sweep = hp.dynamic_smem.kernel; + + using AgentHistogramPolicyT = + agent_histogram_policy; + using AgentHistogramT = + AgentHistogram; __shared__ typename AgentHistogramT::TempStorage static_smem_storage; extern __shared__ __align__(16) unsigned char dynamic_smem[]; @@ -949,7 +958,7 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(threads_per_block(current_policy(), PrivatizationMode{}))) +__launch_bounds__(int(kernel_config(current_policy(), PrivatizationMode{}).threads_per_block)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDeviceInitKernel( const SampleIteratorT d_samples, ::cuda::std::array num_output_bins_wrapper, @@ -964,6 +973,9 @@ __launch_bounds__(int(threads_per_block(current_policy(), Privat const int tiles_per_row, const GridQueue tile_queue) { + static constexpr HistogramPolicy hp = current_policy(); + static constexpr auto sweep = kernel_config(hp, PrivatizationMode{}); + OutputDecodeOpT output_decode_op[NumActiveChannels]; PrivatizedDecodeOpT privatized_decode_op[NumActiveChannels]; if constexpr (IsEven) @@ -990,25 +1002,27 @@ __launch_bounds__(int(threads_per_block(current_policy(), Privat } } - using AgentHistogramT = AgentHistogram< - threads_per_block(current_policy(), PrivatizationMode{}), - items_per_thread(current_policy(), PrivatizationMode{}), - load_algorithm(current_policy(), PrivatizationMode{}), - load_modifier(current_policy(), PrivatizationMode{}), - rle_compress(current_policy(), PrivatizationMode{}), - work_stealing(current_policy(), PrivatizationMode{}), - vec_size(current_policy(), PrivatizationMode{}), - is_privatized_static_smem_v ? current_policy().static_smem_max_privatized_bytes - : 0, - PrivatizationMode, - NumChannels, - NumActiveChannels, - SampleIteratorT, - CounterT, - PrivatizedDecodeOpT, - OutputDecodeOpT, - OffsetT, - OutputCounterT>; + // Thread block type for compositing input tiles + using AgentHistogramPolicyT = agent_histogram_policy< + sweep.threads_per_block, + sweep.items_per_thread, + sweep.load_algorithm, + sweep.load_modifier, + sweep.rle_compress, + sweep.work_stealing, + sweep.vec_size, + is_privatized_static_smem_v ? hp.static_smem.max_privatized_smem_bytes : 0>; + using AgentHistogramT = + AgentHistogram; // Shared memory for AgentHistogram __shared__ typename AgentHistogramT::TempStorage static_smem_storage; diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 2c04a46b6c41..ecb3f71e69a6 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -25,70 +25,73 @@ CUB_NAMESPACE_BEGIN +//! Runtime launch configuration for one DeviceHistogram kernel. +struct HistogramKernelConfig +{ + int threads_per_block; //!< Number of threads in a CUDA block + int items_per_thread; //!< Number of items processed per thread + int vec_size; //!< Vectorization size for loading samples + BlockLoadAlgorithm load_algorithm; //!< Algorithm used for loading samples + CacheLoadModifier load_modifier; //!< Cache modifier used for loading samples + bool rle_compress; //!< Whether to locally run-length encode samples + bool work_stealing; //!< Whether blocks dequeue tiles from a global queue + + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool + operator==(const HistogramKernelConfig& lhs, const HistogramKernelConfig& rhs) noexcept + { + return lhs.threads_per_block == rhs.threads_per_block && lhs.items_per_thread == rhs.items_per_thread + && lhs.vec_size == rhs.vec_size && lhs.load_algorithm == rhs.load_algorithm + && lhs.load_modifier == rhs.load_modifier && lhs.rle_compress == rhs.rle_compress + && lhs.work_stealing == rhs.work_stealing; + } +}; + +//! Tuning policy for the compile-time-sized shared-memory histogram kernel. +struct HistogramStaticSmemPolicy +{ + HistogramKernelConfig kernel; + int max_privatized_smem_bytes; //!< Maximum compile-time-sized shared-memory allocation + int min_blocks_per_sm; //!< Minimum blocks per SM requested through launch bounds + + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool + operator==(const HistogramStaticSmemPolicy& lhs, const HistogramStaticSmemPolicy& rhs) noexcept + { + return lhs.kernel == rhs.kernel && lhs.max_privatized_smem_bytes == rhs.max_privatized_smem_bytes + && lhs.min_blocks_per_sm == rhs.min_blocks_per_sm; + } +}; + +//! Tuning policy for the runtime-sized shared-memory histogram kernel. +struct HistogramDynamicSmemPolicy +{ + HistogramKernelConfig kernel; + int max_privatized_smem_bytes; //!< Maximum runtime-sized shared-memory allocation + int range_max_bins; //!< Maximum bins per channel for multi-channel HistogramRange + int even_2ch_max_bins; //!< Maximum bins per channel for two-channel HistogramEven + int even_3ch_max_bins; //!< Maximum bins per channel for three-channel HistogramEven + int even_4ch_max_bins; //!< Maximum bins per channel for four-channel HistogramEven + + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool + operator==(const HistogramDynamicSmemPolicy& lhs, const HistogramDynamicSmemPolicy& rhs) noexcept + { + return lhs.kernel == rhs.kernel && lhs.max_privatized_smem_bytes == rhs.max_privatized_smem_bytes + && lhs.range_max_bins == rhs.range_max_bins && lhs.even_2ch_max_bins == rhs.even_2ch_max_bins + && lhs.even_3ch_max_bins == rhs.even_3ch_max_bins && lhs.even_4ch_max_bins == rhs.even_4ch_max_bins; + } +}; + //! The tuning policy for all DeviceHistogram kernel variants. struct HistogramPolicy { - int gmem_threads_per_block; - int gmem_items_per_thread; - int gmem_vec_size; - BlockLoadAlgorithm gmem_load_algorithm; - CacheLoadModifier gmem_load_modifier; - bool gmem_rle_compress; - bool gmem_work_stealing; - - int static_smem_threads_per_block; - int static_smem_items_per_thread; - int static_smem_vec_size; - BlockLoadAlgorithm static_smem_load_algorithm; - CacheLoadModifier static_smem_load_modifier; - bool static_smem_rle_compress; - bool static_smem_work_stealing; - int static_smem_max_privatized_bytes; - int static_smem_min_blocks_per_sm; - - int dynamic_smem_threads_per_block; - int dynamic_smem_items_per_thread; - int dynamic_smem_vec_size; - BlockLoadAlgorithm dynamic_smem_load_algorithm; - CacheLoadModifier dynamic_smem_load_modifier; - bool dynamic_smem_rle_compress; - bool dynamic_smem_work_stealing; - int dynamic_smem_max_privatized_bytes; - int dynamic_smem_range_max_bins; - int dynamic_smem_even_2ch_max_bins; - int dynamic_smem_even_3ch_max_bins; - int dynamic_smem_even_4ch_max_bins; - + HistogramKernelConfig gmem; + HistogramStaticSmemPolicy static_smem; + HistogramDynamicSmemPolicy dynamic_smem; int init_kernel_pdl_trigger_max_bins; //!< Common init-kernel PDL threshold, independent of accumulation tier [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(const HistogramPolicy& lhs, const HistogramPolicy& rhs) noexcept { - return lhs.gmem_threads_per_block == rhs.gmem_threads_per_block - && lhs.gmem_items_per_thread == rhs.gmem_items_per_thread && lhs.gmem_vec_size == rhs.gmem_vec_size - && lhs.gmem_load_algorithm == rhs.gmem_load_algorithm && lhs.gmem_load_modifier == rhs.gmem_load_modifier - && lhs.gmem_rle_compress == rhs.gmem_rle_compress && lhs.gmem_work_stealing == rhs.gmem_work_stealing - && lhs.static_smem_threads_per_block == rhs.static_smem_threads_per_block - && lhs.static_smem_items_per_thread == rhs.static_smem_items_per_thread - && lhs.static_smem_vec_size == rhs.static_smem_vec_size - && lhs.static_smem_load_algorithm == rhs.static_smem_load_algorithm - && lhs.static_smem_load_modifier == rhs.static_smem_load_modifier - && lhs.static_smem_rle_compress == rhs.static_smem_rle_compress - && lhs.static_smem_work_stealing == rhs.static_smem_work_stealing - && lhs.static_smem_max_privatized_bytes == rhs.static_smem_max_privatized_bytes - && lhs.static_smem_min_blocks_per_sm == rhs.static_smem_min_blocks_per_sm - && lhs.dynamic_smem_threads_per_block == rhs.dynamic_smem_threads_per_block - && lhs.dynamic_smem_items_per_thread == rhs.dynamic_smem_items_per_thread - && lhs.dynamic_smem_vec_size == rhs.dynamic_smem_vec_size - && lhs.dynamic_smem_load_algorithm == rhs.dynamic_smem_load_algorithm - && lhs.dynamic_smem_load_modifier == rhs.dynamic_smem_load_modifier - && lhs.dynamic_smem_rle_compress == rhs.dynamic_smem_rle_compress - && lhs.dynamic_smem_work_stealing == rhs.dynamic_smem_work_stealing - && lhs.dynamic_smem_max_privatized_bytes == rhs.dynamic_smem_max_privatized_bytes - && lhs.dynamic_smem_range_max_bins == rhs.dynamic_smem_range_max_bins - && lhs.dynamic_smem_even_2ch_max_bins == rhs.dynamic_smem_even_2ch_max_bins - && lhs.dynamic_smem_even_3ch_max_bins == rhs.dynamic_smem_even_3ch_max_bins - && lhs.dynamic_smem_even_4ch_max_bins == rhs.dynamic_smem_even_4ch_max_bins + return lhs.gmem == rhs.gmem && lhs.static_smem == rhs.static_smem && lhs.dynamic_smem == rhs.dynamic_smem && lhs.init_kernel_pdl_trigger_max_bins == rhs.init_kernel_pdl_trigger_max_bins; } @@ -101,29 +104,26 @@ struct HistogramPolicy #if _CCCL_HOSTED() friend ::std::ostream& operator<<(::std::ostream& os, const HistogramPolicy& p) { + const auto print_kernel = [&](const HistogramKernelConfig& kernel) -> ::std::ostream& { + return os + << "{ .threads_per_block = " << kernel.threads_per_block + << ", .items_per_thread = " << kernel.items_per_thread << ", .vec_size = " << kernel.vec_size + << ", .load_algorithm = " << kernel.load_algorithm << ", .load_modifier = " << kernel.load_modifier + << ", .rle_compress = " << kernel.rle_compress << ", .work_stealing = " << kernel.work_stealing << " }"; + }; + os << "HistogramPolicy { .gmem = "; + print_kernel(p.gmem); + os << ", .static_smem = { .kernel = "; + print_kernel(p.static_smem.kernel); + os << ", .max_privatized_smem_bytes = " << p.static_smem.max_privatized_smem_bytes + << ", .min_blocks_per_sm = " << p.static_smem.min_blocks_per_sm << " }, .dynamic_smem = { .kernel = "; + print_kernel(p.dynamic_smem.kernel); return os - << "HistogramPolicy { .gmem_threads_per_block = " << p.gmem_threads_per_block - << ", .gmem_items_per_thread = " << p.gmem_items_per_thread << ", .gmem_vec_size = " << p.gmem_vec_size - << ", .gmem_load_algorithm = " << p.gmem_load_algorithm << ", .gmem_load_modifier = " << p.gmem_load_modifier - << ", .gmem_rle_compress = " << p.gmem_rle_compress << ", .gmem_work_stealing = " << p.gmem_work_stealing - << ", .static_smem_threads_per_block = " << p.static_smem_threads_per_block - << ", .static_smem_items_per_thread = " << p.static_smem_items_per_thread << ", .static_smem_vec_size = " - << p.static_smem_vec_size << ", .static_smem_load_algorithm = " << p.static_smem_load_algorithm - << ", .static_smem_load_modifier = " << p.static_smem_load_modifier << ", .static_smem_rle_compress = " - << p.static_smem_rle_compress << ", .static_smem_work_stealing = " << p.static_smem_work_stealing - << ", .static_smem_max_privatized_bytes = " << p.static_smem_max_privatized_bytes - << ", .static_smem_min_blocks_per_sm = " << p.static_smem_min_blocks_per_sm - << ", .dynamic_smem_threads_per_block = " << p.dynamic_smem_threads_per_block - << ", .dynamic_smem_items_per_thread = " << p.dynamic_smem_items_per_thread << ", .dynamic_smem_vec_size = " - << p.dynamic_smem_vec_size << ", .dynamic_smem_load_algorithm = " << p.dynamic_smem_load_algorithm - << ", .dynamic_smem_load_modifier = " << p.dynamic_smem_load_modifier << ", .dynamic_smem_rle_compress = " - << p.dynamic_smem_rle_compress << ", .dynamic_smem_work_stealing = " << p.dynamic_smem_work_stealing - << ", .dynamic_smem_max_privatized_bytes = " << p.dynamic_smem_max_privatized_bytes - << ", .dynamic_smem_range_max_bins = " << p.dynamic_smem_range_max_bins - << ", .dynamic_smem_even_2ch_max_bins = " << p.dynamic_smem_even_2ch_max_bins - << ", .dynamic_smem_even_3ch_max_bins = " << p.dynamic_smem_even_3ch_max_bins - << ", .dynamic_smem_even_4ch_max_bins = " << p.dynamic_smem_even_4ch_max_bins - << ", .init_kernel_pdl_trigger_max_bins = " << p.init_kernel_pdl_trigger_max_bins << " }"; + << ", .max_privatized_smem_bytes = " << p.dynamic_smem.max_privatized_smem_bytes << ", .range_max_bins = " + << p.dynamic_smem.range_max_bins << ", .even_2ch_max_bins = " << p.dynamic_smem.even_2ch_max_bins + << ", .even_3ch_max_bins = " << p.dynamic_smem.even_3ch_max_bins + << ", .even_4ch_max_bins = " << p.dynamic_smem.even_4ch_max_bins + << " }, .init_kernel_pdl_trigger_max_bins = " << p.init_kernel_pdl_trigger_max_bins << " }"; } #endif }; @@ -142,51 +142,24 @@ inline constexpr int sm100_even_3ch_dynamic_smem_max_bins = 19029; inline constexpr int sm100_even_4ch_dynamic_smem_max_bins = 8192; template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int threads_per_block(const HistogramPolicy& policy, PrivatizationMode) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const HistogramKernelConfig& +kernel_config(const HistogramPolicy& policy, PrivatizationMode) { if constexpr (is_privatized_static_smem_v) { - return policy.static_smem_threads_per_block; + return policy.static_smem.kernel; } else if constexpr (is_privatized_dynamic_smem_v) { - return policy.dynamic_smem_threads_per_block; + return policy.dynamic_smem.kernel; } else { static_assert(is_privatized_gmem_v); - return policy.gmem_threads_per_block; + return policy.gmem; } } -#define CUB_DETAIL_HISTOGRAM_POLICY_FIELD_ACCESSOR(NAME, TYPE) \ - template \ - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr TYPE NAME(const HistogramPolicy& policy, PrivatizationMode) \ - { \ - if constexpr (is_privatized_static_smem_v) \ - { \ - return policy.static_smem_##NAME; \ - } \ - else if constexpr (is_privatized_dynamic_smem_v) \ - { \ - return policy.dynamic_smem_##NAME; \ - } \ - else \ - { \ - static_assert(is_privatized_gmem_v); \ - return policy.gmem_##NAME; \ - } \ - } - -CUB_DETAIL_HISTOGRAM_POLICY_FIELD_ACCESSOR(items_per_thread, int) -CUB_DETAIL_HISTOGRAM_POLICY_FIELD_ACCESSOR(vec_size, int) -CUB_DETAIL_HISTOGRAM_POLICY_FIELD_ACCESSOR(load_algorithm, BlockLoadAlgorithm) -CUB_DETAIL_HISTOGRAM_POLICY_FIELD_ACCESSOR(load_modifier, CacheLoadModifier) -CUB_DETAIL_HISTOGRAM_POLICY_FIELD_ACCESSOR(rle_compress, bool) -CUB_DETAIL_HISTOGRAM_POLICY_FIELD_ACCESSOR(work_stealing, bool) - -#undef CUB_DETAIL_HISTOGRAM_POLICY_FIELD_ACCESSOR - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int max_privatized_smem_bins(int max_privatized_smem_bytes, int counter_size, int num_active_channels) { @@ -200,13 +173,13 @@ max_privatized_smem_bins(int max_privatized_smem_bytes, int counter_size, int nu [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int max_privatized_static_smem_bins(const HistogramPolicy& policy, int counter_size, int num_active_channels) { - return max_privatized_smem_bins(policy.static_smem_max_privatized_bytes, counter_size, num_active_channels); + return max_privatized_smem_bins(policy.static_smem.max_privatized_smem_bytes, counter_size, num_active_channels); } [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int max_privatized_dynamic_smem_bins(const HistogramPolicy& policy, int counter_size, int num_active_channels) { - return max_privatized_smem_bins(policy.dynamic_smem_max_privatized_bytes, counter_size, num_active_channels); + return max_privatized_smem_bins(policy.dynamic_smem.max_privatized_smem_bytes, counter_size, num_active_channels); } [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool @@ -237,14 +210,14 @@ should_use_dynamic_smem(const HistogramPolicy& policy, int num_bins, int counter { if constexpr (IsEven) { - max_bins = num_active_channels == 2 ? policy.dynamic_smem_even_2ch_max_bins + max_bins = num_active_channels == 2 ? policy.dynamic_smem.even_2ch_max_bins : num_active_channels == 3 - ? policy.dynamic_smem_even_3ch_max_bins - : policy.dynamic_smem_even_4ch_max_bins; + ? policy.dynamic_smem.even_3ch_max_bins + : policy.dynamic_smem.even_4ch_max_bins; } else { - max_bins = policy.dynamic_smem_range_max_bins; + max_bins = policy.dynamic_smem.range_max_bins; } } @@ -394,8 +367,7 @@ struct policy_hub struct Policy500 : detail::chained_policy<500, Policy500, Policy500> { // TODO This might be worth it to separate usual histogram and the multi one - using AgentHistogramPolicyT = - legacy_agent_histogram_policy<384, t_scale(16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, false>; + using AgentHistogramPolicyT = agent_histogram_policy<384, t_scale(16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, false>; static constexpr int init_kernel_pdl_trigger_max_bins = 0; }; @@ -405,12 +377,12 @@ struct policy_hub // Use values from tuning if a specialization exists, otherwise pick Policy500 template _CCCL_HOST_DEVICE_API static auto select_agent_policy(int) - -> legacy_agent_histogram_policy; + -> agent_histogram_policy; template _CCCL_HOST_DEVICE_API static auto select_agent_policy(long) -> typename Policy500::AgentHistogramPolicyT; @@ -430,14 +402,14 @@ struct policy_hub { // Use values from tuning if a specialization exists, otherwise pick Policy900 template - _CCCL_HOST_DEVICE_API static auto select_agent_policy(int) -> legacy_agent_histogram_policy< - Tuning::threads_per_block, - Tuning::items_per_thread, - Tuning::load_algorithm, - Tuning::load_modifier, - Tuning::rle_compress, - Tuning::work_stealing, - Tuning::vec_size>; + _CCCL_HOST_DEVICE_API static auto select_agent_policy(int) + -> agent_histogram_policy; template _CCCL_HOST_DEVICE_API static auto select_agent_policy(long) -> typename Policy900::AgentHistogramPolicyT; @@ -478,202 +450,120 @@ private: return (::cuda::std::max) (nominal_items_per_thread / num_active_channels / sample_scale, 1); } - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto make_policy( - int threads_per_block, - int items_per_thread, - int vec_size, - BlockLoadAlgorithm load_algorithm, - CacheLoadModifier load_modifier, - bool rle_compress, - bool work_stealing, - int static_smem_max_bytes, - int pdl_max_bins) const -> HistogramPolicy + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto default_kernel_config() const -> HistogramKernelConfig { - return { - threads_per_block, - items_per_thread, - vec_size, - load_algorithm, - load_modifier, - rle_compress, - work_stealing, - threads_per_block, - items_per_thread, - vec_size, - load_algorithm, - load_modifier, - rle_compress, - work_stealing, - static_smem_max_bytes, - 0, - threads_per_block, - items_per_thread, - vec_size, - load_algorithm, - load_modifier, - rle_compress, - work_stealing, - 0, - 0, - 0, - 0, - 0, - pdl_max_bins}; + return {384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; } - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto default_policy() const -> HistogramPolicy + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto sm90_kernel_config() const -> HistogramKernelConfig { - return make_policy( - 384, - t_scale(16), - 4, - BLOCK_LOAD_DIRECT, - LOAD_LDG, - true, - false, - pre_sm100_static_smem_max_bins * counter_size_bytes * num_active_channels, - 0); - } - - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto sm90_policy() const -> HistogramPolicy - { - const bool single_channel = num_channels == 1 && num_active_channels == 1; - const int pdl_max_bins = - single_channel && counter_size_bytes == 4 && sample_is_primitive - && (sample_size_bytes == 1 || sample_size_bytes == 2) - ? pdl_trigger_max_bins - : 0; - if (single_channel && counter_size_bytes == 4 && sample_is_primitive) + if (num_channels == 1 && num_active_channels == 1 && counter_size_bytes == 4 && sample_is_primitive) { if (sample_size_bytes == 1) { - return make_policy( - 768, - 12, - 4, - BLOCK_LOAD_DIRECT, - LOAD_LDG, - false, - false, - pre_sm100_static_smem_max_bins * counter_size_bytes * num_active_channels, - pdl_max_bins); + return {768, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, false, false}; } if (sample_size_bytes == 2) { - return make_policy( - 960, - 10, - 4, - BLOCK_LOAD_DIRECT, - LOAD_DEFAULT, - true, - false, - pre_sm100_static_smem_max_bins * counter_size_bytes * num_active_channels, - pdl_max_bins); + return {960, 10, 4, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, false}; } } - auto result = default_policy(); - result.init_kernel_pdl_trigger_max_bins = pdl_max_bins; - return result; + return default_kernel_config(); } -public: - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto sm100_kernel_config() const -> HistogramKernelConfig { - if (cc < ::cuda::compute_capability{9, 0}) - { - return default_policy(); - } - if (cc < ::cuda::compute_capability{10, 0}) - { - return sm90_policy(); - } - - const bool single_channel = num_channels == 1 && num_active_channels == 1; - auto result = sm90_policy(); if (num_channels >= 2 && counter_size_bytes == 4 && sample_is_primitive) { - result.gmem_threads_per_block = 1024; - result.gmem_items_per_thread = t_scale(is_even ? 8 : 16); + return {1024, t_scale(is_even ? 8 : 16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; } - else if (single_channel && counter_size_bytes == 4 && sample_is_primitive) + if (num_channels == 1 && num_active_channels == 1 && counter_size_bytes == 4 && sample_is_primitive) { if (sample_size_bytes == 1) { - result.gmem_threads_per_block = is_even ? 928 : 448; - result.gmem_items_per_thread = 12; - result.gmem_load_modifier = is_even ? LOAD_CA : LOAD_LDG; - result.gmem_rle_compress = false; + return is_even ? HistogramKernelConfig{928, 12, 4, BLOCK_LOAD_DIRECT, LOAD_CA, false, false} + : HistogramKernelConfig{448, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, false, false}; } - else if (sample_size_bytes == 4) + if (sample_size_bytes == 4) { - result.gmem_threads_per_block = 768; - result.gmem_items_per_thread = 12; + return {768, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; } - else if (sample_size_bytes == 8) + if (sample_size_bytes == 8) { - result.gmem_threads_per_block = 768; - result.gmem_items_per_thread = 6; + return {768, 6, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; } } + return sm90_kernel_config(); + } - result.static_smem_threads_per_block = result.gmem_threads_per_block; - result.static_smem_items_per_thread = result.gmem_items_per_thread; - result.static_smem_vec_size = result.gmem_vec_size; - result.static_smem_load_algorithm = result.gmem_load_algorithm; - result.static_smem_load_modifier = result.gmem_load_modifier; - result.static_smem_rle_compress = result.gmem_rle_compress; - result.static_smem_work_stealing = result.gmem_work_stealing; - - const bool range_multi_static = !is_even && num_channels >= 2 && counter_size_bytes == 4 && sample_is_primitive; - const bool range_u32_static = - !is_even && single_channel && counter_size_bytes == 4 && sample_is_primitive && sample_size_bytes == 4; - const bool range_u64_static = - !is_even && single_channel && counter_size_bytes == 4 && sample_is_primitive && sample_size_bytes == 8; - if (range_multi_static || range_u64_static) - { - result.static_smem_threads_per_block = 384; - } - else if (range_u32_static) - { - result.static_smem_threads_per_block = 768; - } - if (range_u64_static) - { - result.static_smem_items_per_thread = t_scale(16); - } - result.static_smem_max_privatized_bytes = sm100_static_smem_max_bins * counter_size_bytes * num_active_channels; - result.static_smem_min_blocks_per_sm = range_multi_static || range_u64_static ? 3 : 0; - - result.dynamic_smem_threads_per_block = result.gmem_threads_per_block; - result.dynamic_smem_items_per_thread = result.gmem_items_per_thread; - result.dynamic_smem_vec_size = result.gmem_vec_size; - result.dynamic_smem_load_algorithm = result.gmem_load_algorithm; - result.dynamic_smem_load_modifier = result.gmem_load_modifier; - result.dynamic_smem_rle_compress = result.gmem_rle_compress; - result.dynamic_smem_work_stealing = result.gmem_work_stealing; - - const bool has_dynamic_smem_tuning = - counter_size_bytes == 4 && sample_is_primitive - && ((single_channel && (sample_size_bytes == 1 || sample_size_bytes == 4 || sample_size_bytes == 8)) - || num_channels >= 2); - if (has_dynamic_smem_tuning) +public: + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy + { + const bool single_channel = num_channels == 1 && num_active_channels == 1; + if (cc >= ::cuda::compute_capability{10, 0}) { - result.dynamic_smem_max_privatized_bytes = sm100_dynamic_smem_max_bytes; - result.dynamic_smem_range_max_bins = sm100_range_dynamic_smem_max_bins; - result.dynamic_smem_even_2ch_max_bins = sm100_even_2ch_dynamic_smem_max_bins; - result.dynamic_smem_even_3ch_max_bins = sm100_even_3ch_dynamic_smem_max_bins; - result.dynamic_smem_even_4ch_max_bins = sm100_even_4ch_dynamic_smem_max_bins; + const HistogramKernelConfig kernel = sm100_kernel_config(); + const bool range_multi_static = !is_even && num_channels >= 2 && counter_size_bytes == 4 && sample_is_primitive; + const bool range_u32_static = + !is_even && single_channel && counter_size_bytes == 4 && sample_is_primitive && sample_size_bytes == 4; + const bool range_u64_static = + !is_even && single_channel && counter_size_bytes == 4 && sample_is_primitive && sample_size_bytes == 8; + HistogramKernelConfig static_kernel = kernel; + if (range_multi_static || range_u64_static) + { + static_kernel.threads_per_block = 384; + } + else if (range_u32_static) + { + static_kernel.threads_per_block = 768; + } + if (range_u64_static) + { + static_kernel.items_per_thread = t_scale(16); + } + + const bool has_dynamic_smem_tuning = + counter_size_bytes == 4 && sample_is_primitive + && ((single_channel && (sample_size_bytes == 1 || sample_size_bytes == 4 || sample_size_bytes == 8)) + || num_channels >= 2); + const int dynamic_smem_bytes = has_dynamic_smem_tuning ? sm100_dynamic_smem_max_bytes : 0; + const int dynamic_range_bins = has_dynamic_smem_tuning ? sm100_range_dynamic_smem_max_bins : 0; + const int dynamic_even_2ch_bins = has_dynamic_smem_tuning ? sm100_even_2ch_dynamic_smem_max_bins : 0; + const int dynamic_even_3ch_bins = has_dynamic_smem_tuning ? sm100_even_3ch_dynamic_smem_max_bins : 0; + const int dynamic_even_4ch_bins = has_dynamic_smem_tuning ? sm100_even_4ch_dynamic_smem_max_bins : 0; + const int pdl_bins = + single_channel && counter_size_bytes == 4 && sample_is_primitive + && (sample_size_bytes == 1 || sample_size_bytes == 2 || sample_size_bytes == 4 || sample_size_bytes == 8) + ? pdl_trigger_max_bins + : 0; + return { + kernel, + {static_kernel, + sm100_static_smem_max_bins * counter_size_bytes * num_active_channels, + range_multi_static || range_u64_static ? 3 : 0}, + {kernel, + dynamic_smem_bytes, + dynamic_range_bins, + dynamic_even_2ch_bins, + dynamic_even_3ch_bins, + dynamic_even_4ch_bins}, + pdl_bins}; } - result.init_kernel_pdl_trigger_max_bins = - single_channel && counter_size_bytes == 4 && sample_is_primitive - && (sample_size_bytes == 1 || sample_size_bytes == 2 || sample_size_bytes == 4 || sample_size_bytes == 8) + const HistogramKernelConfig kernel = + cc >= ::cuda::compute_capability{9, 0} ? sm90_kernel_config() : default_kernel_config(); + const int pdl_bins = + cc >= ::cuda::compute_capability{9, 0} && single_channel && counter_size_bytes == 4 && sample_is_primitive + && (sample_size_bytes == 1 || sample_size_bytes == 2) ? pdl_trigger_max_bins : 0; - return result; + return {kernel, + {kernel, pre_sm100_static_smem_max_bins * counter_size_bytes * num_active_channels, 0}, + {kernel, 0, 0, 0, 0, 0}, + pdl_bins}; } }; + #if _CCCL_HAS_CONCEPTS() static_assert(histogram_policy_selector); #endif // _CCCL_HAS_CONCEPTS() diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index 7542d6037418..ac488a59314d 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1626,36 +1626,9 @@ struct histogram_tuning { _CCCL_HOST_DEVICE_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { - return { - BlockThreads, - 1, - 1, - cub::BLOCK_LOAD_DIRECT, - cub::LOAD_DEFAULT, - false, - false, - BlockThreads, - 1, - 1, - cub::BLOCK_LOAD_DIRECT, - cub::LOAD_DEFAULT, - false, - false, - 256 * sizeof(unsigned int), - 0, - BlockThreads, - 1, - 1, - cub::BLOCK_LOAD_DIRECT, - cub::LOAD_DEFAULT, - false, - false, - 0, - 0, - 0, - 0, - 0, - 0}; + constexpr auto sweep = + cub::HistogramKernelConfig{BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; + return {sweep, {sweep, 256 * sizeof(unsigned int), 0}, {sweep, 0, 0, 0, 0, 0}, 0}; } }; @@ -1671,36 +1644,9 @@ struct mixed_counter_histogram_tuning _CCCL_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { - return { - 128, - 4, - 1, - cub::BLOCK_LOAD_DIRECT, - cub::LOAD_DEFAULT, - false, - false, - 128, - 4, - 1, - cub::BLOCK_LOAD_DIRECT, - cub::LOAD_DEFAULT, - false, - false, - 512 * sizeof(unsigned int), - 0, - 128, - 4, - 1, - cub::BLOCK_LOAD_DIRECT, - cub::LOAD_DEFAULT, - false, - false, - 228352, - 2048, - 28544, - 19029, - 8192, - 0}; + constexpr auto sweep = + cub::HistogramKernelConfig{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; + return {sweep, {sweep, 512 * sizeof(unsigned int), 0}, {sweep, 228352, 2048, 28544, 19029, 8192}, 0}; } }; @@ -1860,80 +1806,72 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) // aggregate init constexpr auto p1 = cub::HistogramPolicy{ - 128, - 7, - 4, - cub::BLOCK_LOAD_DIRECT, - cub::CacheLoadModifier::LOAD_LDG, - false, - false, - 96, - 3, - 4, - cub::BLOCK_LOAD_DIRECT, - cub::CacheLoadModifier::LOAD_LDG, - false, - false, - 2052, - 2, - 128, - 7, - 4, - cub::BLOCK_LOAD_DIRECT, - cub::CacheLoadModifier::LOAD_LDG, - false, - false, - 12345, - 1024, - 4096, - 8192, - 16384, + {128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, + {{96, 3, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, 2052, 2}, + {{128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, 12345, 1024, 4096, 8192, 16384}, 2048}; # if _CCCL_STD_VER >= 2020 // designated init constexpr auto p2 = cub::HistogramPolicy{ - .gmem_threads_per_block = 128, - .gmem_items_per_thread = 7, - .gmem_vec_size = 4, - .gmem_load_algorithm = cub::BLOCK_LOAD_DIRECT, - .gmem_load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .gmem_rle_compress = false, - .gmem_work_stealing = false, - .static_smem_threads_per_block = 96, - .static_smem_items_per_thread = 3, - .static_smem_vec_size = 4, - .static_smem_load_algorithm = cub::BLOCK_LOAD_DIRECT, - .static_smem_load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .static_smem_rle_compress = false, - .static_smem_work_stealing = false, - .static_smem_max_privatized_bytes = 2052, - .static_smem_min_blocks_per_sm = 2, - .dynamic_smem_threads_per_block = 128, - .dynamic_smem_items_per_thread = 7, - .dynamic_smem_vec_size = 4, - .dynamic_smem_load_algorithm = cub::BLOCK_LOAD_DIRECT, - .dynamic_smem_load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .dynamic_smem_rle_compress = false, - .dynamic_smem_work_stealing = false, - .dynamic_smem_max_privatized_bytes = 12345, - .dynamic_smem_range_max_bins = 1024, - .dynamic_smem_even_2ch_max_bins = 4096, - .dynamic_smem_even_3ch_max_bins = 8192, - .dynamic_smem_even_4ch_max_bins = 16384, - .init_kernel_pdl_trigger_max_bins = 2048}; + .gmem = {.threads_per_block = 128, + .items_per_thread = 7, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .static_smem = {.kernel = {.threads_per_block = 96, + .items_per_thread = 3, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .max_privatized_smem_bytes = 2052, + .min_blocks_per_sm = 2}, + .dynamic_smem = + {.kernel = {.threads_per_block = 128, + .items_per_thread = 7, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .max_privatized_smem_bytes = 12345, + .range_max_bins = 1024, + .even_2ch_max_bins = 4096, + .even_3ch_max_bins = 8192, + .even_4ch_max_bins = 16384}, + .init_kernel_pdl_trigger_max_bins = 2048}; # else // _CCCL_STD_VER >= 2020 constexpr auto p2 = p1; # endif // _CCCL_STD_VER >= 2020 + // comparison STATIC_REQUIRE(p1 == p2); STATIC_REQUIRE_FALSE(p1 != p2); - std::ostringstream os; - os << p1; - REQUIRE(os.str().find("HistogramPolicy { .gmem_threads_per_block = 128") == 0); - REQUIRE(os.str().find(".static_smem_max_privatized_bytes = 2052") != std::string::npos); - REQUIRE(os.str().find(".dynamic_smem_max_privatized_bytes = 12345") != std::string::npos); + auto to_string = [](const auto& p) { + std::ostringstream os; + os << p; + return os.str(); + }; + REQUIRE( + to_string(p1) + == "HistogramPolicy { .gmem = { .threads_per_block = 128, .items_per_thread = 7, .vec_size = 4" + ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" + ", .work_stealing = 0 }" + ", .static_smem = { .kernel = { .threads_per_block = 96, .items_per_thread = 3, .vec_size = 4" + ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" + ", .work_stealing = 0 }" + ", .max_privatized_smem_bytes = 2052, .min_blocks_per_sm = 2 }, .dynamic_smem = { .kernel = { " + ".threads_per_block " + "= 128" + ", .items_per_thread = 7, .vec_size = 4, .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG" + ", .rle_compress = 0, .work_stealing = 0 }, .max_privatized_smem_bytes = 12345" + ", .range_max_bins = 1024, .even_2ch_max_bins = 4096, .even_3ch_max_bins = 8192" + ", .even_4ch_max_bins = 16384 }, .init_kernel_pdl_trigger_max_bins = 2048 }"); } C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget", "[histogram][device]") @@ -1949,36 +1887,36 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE(cub::detail::histogram::max_privatized_static_smem_bins(sm90_policy, 4, 1) == 256); STATIC_REQUIRE(cub::detail::histogram::max_privatized_static_smem_bins(sm100_policy, 4, 1) == 512); STATIC_REQUIRE(cub::detail::histogram::max_privatized_static_smem_bins(sm100_policy, 4, 4) == 128); - STATIC_REQUIRE(sm90_policy.dynamic_smem_max_privatized_bytes == 0); - STATIC_REQUIRE(sm100_policy.dynamic_smem_max_privatized_bytes == 228352); - STATIC_REQUIRE(sm100_wide_counter_policy.dynamic_smem_max_privatized_bytes == 0); + STATIC_REQUIRE(sm90_policy.dynamic_smem.max_privatized_smem_bytes == 0); + STATIC_REQUIRE(sm100_policy.dynamic_smem.max_privatized_smem_bytes == 228352); + STATIC_REQUIRE(sm100_wide_counter_policy.dynamic_smem.max_privatized_smem_bytes == 0); STATIC_REQUIRE(cub::detail::histogram::max_privatized_dynamic_smem_bins(sm100_policy, 4, 1) == 57088); STATIC_REQUIRE(cub::detail::histogram::max_privatized_dynamic_smem_bins(sm100_policy, 4, 4) == 14272); - STATIC_REQUIRE(sm100_policy.dynamic_smem_range_max_bins == 2048); - STATIC_REQUIRE(sm100_policy.dynamic_smem_even_2ch_max_bins == 28544); - STATIC_REQUIRE(sm100_policy.dynamic_smem_even_3ch_max_bins == 19029); - STATIC_REQUIRE(sm100_policy.dynamic_smem_even_4ch_max_bins == 8192); - STATIC_REQUIRE(sm100_policy.gmem_threads_per_block == 768); - STATIC_REQUIRE(sm100_policy.gmem_items_per_thread == 12); - STATIC_REQUIRE(sm100_policy.static_smem_threads_per_block == sm100_policy.gmem_threads_per_block); + STATIC_REQUIRE(sm100_policy.dynamic_smem.range_max_bins == 2048); + STATIC_REQUIRE(sm100_policy.dynamic_smem.even_2ch_max_bins == 28544); + STATIC_REQUIRE(sm100_policy.dynamic_smem.even_3ch_max_bins == 19029); + STATIC_REQUIRE(sm100_policy.dynamic_smem.even_4ch_max_bins == 8192); + STATIC_REQUIRE(sm100_policy.gmem.threads_per_block == 768); + STATIC_REQUIRE(sm100_policy.gmem.items_per_thread == 12); + STATIC_REQUIRE(sm100_policy.static_smem.kernel == sm100_policy.gmem); constexpr auto sm100_range_u64_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); - STATIC_REQUIRE(sm100_range_u64_policy.gmem_threads_per_block == 768); - STATIC_REQUIRE(sm100_range_u64_policy.gmem_items_per_thread == 6); - STATIC_REQUIRE(sm100_range_u64_policy.static_smem_threads_per_block == 384); - STATIC_REQUIRE(sm100_range_u64_policy.static_smem_items_per_thread == 8); - STATIC_REQUIRE(sm100_range_u64_policy.static_smem_min_blocks_per_sm == 3); + STATIC_REQUIRE(sm100_range_u64_policy.gmem.threads_per_block == 768); + STATIC_REQUIRE(sm100_range_u64_policy.gmem.items_per_thread == 6); + STATIC_REQUIRE(sm100_range_u64_policy.static_smem.kernel.threads_per_block == 384); + STATIC_REQUIRE(sm100_range_u64_policy.static_smem.kernel.items_per_thread == 8); + STATIC_REQUIRE(sm100_range_u64_policy.static_smem.min_blocks_per_sm == 3); constexpr auto sm100_multi_range_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); - STATIC_REQUIRE(sm100_multi_range_policy.gmem_threads_per_block == 1024); - STATIC_REQUIRE(sm100_multi_range_policy.gmem_items_per_thread == 5); - STATIC_REQUIRE(sm100_multi_range_policy.static_smem_threads_per_block == 384); - STATIC_REQUIRE(sm100_multi_range_policy.static_smem_items_per_thread == 5); - STATIC_REQUIRE(sm100_multi_range_policy.static_smem_min_blocks_per_sm == 3); + STATIC_REQUIRE(sm100_multi_range_policy.gmem.threads_per_block == 1024); + STATIC_REQUIRE(sm100_multi_range_policy.gmem.items_per_thread == 5); + STATIC_REQUIRE(sm100_multi_range_policy.static_smem.kernel.threads_per_block == 384); + STATIC_REQUIRE(sm100_multi_range_policy.static_smem.kernel.items_per_thread == 5); + STATIC_REQUIRE(sm100_multi_range_policy.static_smem.min_blocks_per_sm == 3); STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57088, 4, 1)); STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57089, 4, 1)); @@ -1994,7 +1932,7 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" using max_policy_t = typename cub::detail::histogram::policy_hub::MaxPolicy; const auto legacy_policy = cub::detail::histogram::policy_selector_from_hub{}(cuda::compute_capability{10, 0}); - REQUIRE(legacy_policy.static_smem_max_privatized_bytes == 256 * sizeof(unsigned int)); - REQUIRE(legacy_policy.dynamic_smem_max_privatized_bytes == 0); + REQUIRE(legacy_policy.static_smem.max_privatized_smem_bytes == 256 * sizeof(unsigned int)); + REQUIRE(legacy_policy.dynamic_smem.max_privatized_smem_bytes == 0); } #endif // _CCCL_COMPILER(GCC, >=, 8) diff --git a/cub/test/catch2_test_device_histogram_env_api.cu b/cub/test/catch2_test_device_histogram_env_api.cu index c8ca0b6914be..ad64e49eae51 100644 --- a/cub/test/catch2_test_device_histogram_env_api.cu +++ b/cub/test/catch2_test_device_histogram_env_api.cu @@ -418,37 +418,18 @@ struct HistogramPolicySelector { __host__ __device__ constexpr auto operator()(cuda::compute_capability cc) const -> cub::HistogramPolicy { - const int items_per_thread = cc > cuda::compute_capability{9, 0} ? 16 : 7; + const auto sweep = cub::HistogramKernelConfig{ + 128, cc > cuda::compute_capability{9, 0} ? 16 : 7, 4, cub::BLOCK_LOAD_DIRECT, cub::LOAD_LDG, false, false}; return { - .gmem_threads_per_block = 128, - .gmem_items_per_thread = items_per_thread, - .gmem_vec_size = 4, - .gmem_load_algorithm = cub::BLOCK_LOAD_DIRECT, - .gmem_load_modifier = cub::LOAD_LDG, - .gmem_rle_compress = false, - .gmem_work_stealing = false, - .static_smem_threads_per_block = 128, - .static_smem_items_per_thread = items_per_thread, - .static_smem_vec_size = 4, - .static_smem_load_algorithm = cub::BLOCK_LOAD_DIRECT, - .static_smem_load_modifier = cub::LOAD_LDG, - .static_smem_rle_compress = false, - .static_smem_work_stealing = false, - .static_smem_max_privatized_bytes = 256 * sizeof(unsigned int), - .static_smem_min_blocks_per_sm = 0, - .dynamic_smem_threads_per_block = 128, - .dynamic_smem_items_per_thread = items_per_thread, - .dynamic_smem_vec_size = 4, - .dynamic_smem_load_algorithm = cub::BLOCK_LOAD_DIRECT, - .dynamic_smem_load_modifier = cub::LOAD_LDG, - .dynamic_smem_rle_compress = false, - .dynamic_smem_work_stealing = false, - .dynamic_smem_max_privatized_bytes = 0, - .dynamic_smem_range_max_bins = 0, - .dynamic_smem_even_2ch_max_bins = 0, - .dynamic_smem_even_3ch_max_bins = 0, - .dynamic_smem_even_4ch_max_bins = 0, - .init_kernel_pdl_trigger_max_bins = 2048}; + .gmem = sweep, + .static_smem = {.kernel = sweep, .max_privatized_smem_bytes = 256 * sizeof(unsigned int), .min_blocks_per_sm = 0}, + .dynamic_smem = {.kernel = sweep, + .max_privatized_smem_bytes = 0, + .range_max_bins = 0, + .even_2ch_max_bins = 0, + .even_3ch_max_bins = 0, + .even_4ch_max_bins = 0}, + .init_kernel_pdl_trigger_max_bins = 2048}; } }; // example-end histogram-even-policy-selector From 8707bff539e042feb90adb7ee2fe160cfdafc9ae Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Wed, 5 Aug 2026 15:22:10 +0000 Subject: [PATCH 32/45] [cub] Simplify histogram runtime tuning policy --- .../bench/histogram/histogram_common.cuh | 2 +- cub/cub/agent/agent_histogram.cuh | 58 +- .../device/dispatch/dispatch_histogram.cuh | 98 +-- .../dispatch/kernels/kernel_histogram.cuh | 47 +- .../dispatch/tuning/tuning_histogram.cuh | 616 +++++++----------- ...test_device_histogram_custom_policy_hub.cu | 25 +- cub/test/catch2_test_device_histogram_env.cu | 10 +- .../catch2_test_device_histogram_env_api.cu | 2 +- 8 files changed, 361 insertions(+), 497 deletions(-) diff --git a/cub/benchmarks/bench/histogram/histogram_common.cuh b/cub/benchmarks/bench/histogram/histogram_common.cuh index 55c67f27688c..d71579b1f10f 100644 --- a/cub/benchmarks/bench/histogram/histogram_common.cuh +++ b/cub/benchmarks/bench/histogram/histogram_common.cuh @@ -37,7 +37,7 @@ struct bench_policy_selector ? (NUM_CHANNELS == 1 ? cub::BLOCK_LOAD_STRIPED : cub::BLOCK_LOAD_DIRECT) : TUNE_LOAD_ALGORITHM; - constexpr auto sweep = cub::HistogramKernelConfig{ + constexpr auto sweep = cub::HistogramPolicy::Kernel{ TUNE_THREADS, TUNE_ITEMS, TUNE_VEC_SIZE, diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index e7ef066d9c95..e5cf64918a54 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -30,18 +31,15 @@ CUB_NAMESPACE_BEGIN -namespace detail -{ -//! Parameterizable tuning policy type for AgentHistogram +//! Deprecated [Since 3.5] template -struct agent_histogram_policy + int VecSize = 4> +struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") AgentHistogramPolicy { /// Threads per thread block static constexpr int BLOCK_THREADS = ThreadsPerBlock; @@ -54,9 +52,6 @@ struct agent_histogram_policy /// Whether to dequeue tiles from a global work queue static constexpr bool IS_WORK_STEALING = WorkStealing; - /// Maximum compile-time-sized shared-memory allocation for privatized bins - static constexpr int PRIVATIZED_STATIC_SMEM_BYTES = PrivatizedStaticSmemBytes; - /// Vector size for samples loading (1, 2, 4) static constexpr int VEC_SIZE = VecSize; static_assert(VEC_SIZE == 1 || VEC_SIZE == 2 || VEC_SIZE == 4); @@ -67,18 +62,6 @@ struct agent_histogram_policy ///< Cache load modifier for reading input elements static constexpr CacheLoadModifier LOAD_MODIFIER = LoadModifier; }; -} // namespace detail - -//! Deprecated [Since 3.5] -template -using AgentHistogramPolicy CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") = detail:: - agent_histogram_policy; namespace detail::histogram { @@ -123,8 +106,8 @@ _CCCL_DEVICE _CCCL_FORCEINLINE auto NativePointer(IteratorT itr) //! @brief AgentHistogram implements a stateful abstraction of CUDA thread blocks for participating //! in device-wide histogram . //! -//! @tparam AgentHistogramPolicyT -//! Parameterized AgentHistogramPolicy tuning policy type +//! @tparam PolicySelector +//! Selector that returns the active HistogramPolicy. //! //! @tparam PrivatizationMode //! Storage mode for the privatized histogram. @@ -154,7 +137,7 @@ _CCCL_DEVICE _CCCL_FORCEINLINE auto NativePointer(IteratorT itr) //! //! @tparam OutputCounterT //! Integer type for final output histogram bins. May be wider than `CounterT`. -template ; static constexpr bool uses_dynamic_smem = is_privatized_dynamic_smem_v; static constexpr bool uses_gmem = is_privatized_gmem_v; + static constexpr auto policy = current_policy(); + static constexpr auto sweep = policy.kernel(PrivatizationMode{}); static constexpr int privatized_static_smem_bins = - uses_static_smem ? AgentHistogramPolicyT::PRIVATIZED_STATIC_SMEM_BYTES / int{sizeof(CounterT)} / NumActiveChannels - : 0; + uses_static_smem ? policy.static_smem.max_privatized_smem_bytes / int{sizeof(CounterT)} / NumActiveChannels : 0; static_assert(!uses_static_smem || privatized_static_smem_bins > 0, "Static-SMEM privatization requires room for at least one bin"); - static constexpr int vec_size = AgentHistogramPolicyT::VEC_SIZE; - static constexpr int threads_per_block = AgentHistogramPolicyT::BLOCK_THREADS; - static constexpr int pixels_per_thread = AgentHistogramPolicyT::PIXELS_PER_THREAD; + static constexpr int vec_size = sweep.vec_size; + static constexpr int threads_per_block = sweep.threads_per_block; + static constexpr int pixels_per_thread = sweep.items_per_thread; static constexpr int samples_per_thread = pixels_per_thread * NumChannels; static constexpr int vecs_per_thread = samples_per_thread / vec_size; static constexpr int tile_pixels = pixels_per_thread * threads_per_block; static constexpr int tile_samples = samples_per_thread * threads_per_block; - static constexpr bool is_rle_compress = AgentHistogramPolicyT::IS_RLE_COMPRESS; - static constexpr bool is_work_stealing = AgentHistogramPolicyT::IS_WORK_STEALING; - static constexpr CacheLoadModifier load_modifier = AgentHistogramPolicyT::LOAD_MODIFIER; + static constexpr bool is_rle_compress = sweep.rle_compress; + static constexpr bool is_work_stealing = sweep.work_stealing; + static constexpr CacheLoadModifier load_modifier = sweep.load_modifier; using SampleT = it_value_t; using PixelT = typename CubVector::Type; @@ -202,11 +186,9 @@ struct AgentHistogram SampleIteratorT>; using WrappedPixelIteratorT = CacheModifiedInputIterator; using WrappedVecsIteratorT = CacheModifiedInputIterator; - using BlockLoadSampleT = - BlockLoad; - using BlockLoadPixelT = - BlockLoad; - using BlockLoadVecT = BlockLoad; + using BlockLoadSampleT = BlockLoad; + using BlockLoadPixelT = BlockLoad; + using BlockLoadVecT = BlockLoad; struct _TempStorage { @@ -429,7 +411,7 @@ struct AgentHistogram bool is_valid[pixels_per_thread]; LoadTile(block_offset, valid_samples, samples); - MarkValid(is_valid, valid_samples); + MarkValid(is_valid, valid_samples); AccumulatePixels(samples, is_valid, ::cuda::std::bool_constant{}); } diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index acb492b60cf0..1dccefd81d45 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -276,9 +276,9 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( } }(); - const HistogramKernelConfig sweep = kernel_config(active_policy, PrivatizationMode{}); - const int threads_per_block = sweep.threads_per_block; - const int items_per_thread = sweep.items_per_thread; + const HistogramPolicy::Kernel sweep = active_policy.kernel(PrivatizationMode{}); + const int threads_per_block = sweep.threads_per_block; + const int items_per_thread = sweep.items_per_thread; int dynamic_smem_bytes = 0; if constexpr (is_privatized_dynamic_smem_v) @@ -385,7 +385,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( num_privatized_levels.begin(), num_privatized_levels.end(), num_privatized_bins_wrapper.begin(), minus_one); ::cuda::std::transform(num_output_levels.begin(), num_output_levels.end(), num_output_bins_wrapper.begin(), minus_one); - constexpr int histogram_init_threads_per_block = 256; + constexpr int histogram_init_threads_per_block = init_threads_per_block; int histogram_init_grid_dims = (max_num_output_bins + histogram_init_threads_per_block - 1) / histogram_init_threads_per_block; @@ -403,7 +403,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( histogram_init_threads_per_block, 0, stream, - /* dependent_launch */ cc >= ::cuda::compute_capability{9, 0}) + /* dependent_launch */ supports_dependent_launch(cc)) .doit(init_kernel, num_output_bins_wrapper, d_output_histograms, tile_queue))) { return error; @@ -434,7 +434,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( threads_per_block, dynamic_smem_bytes, stream, - /* dependent_launch */ cc >= ::cuda::compute_capability{9, 0}) + /* dependent_launch */ supports_dependent_launch(cc)) .doit(sweep_kernel, d_samples, num_output_bins_wrapper, @@ -736,7 +736,7 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel) { - num_privatized_levels[channel] = 257; + num_privatized_levels[channel] = byte_sample_privatized_levels; int num_levels = num_output_levels[channel]; if (kernel_source.MayOverflow(num_levels - 1, upper_level, lower_level, channel)) @@ -788,12 +788,27 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device return cudaSuccess; } +// TODO(bgruber): drop in CCCL 4.0 +template +_CCCL_HOST_DEVICE_API constexpr auto convert_pdl_trigger(int) + -> decltype(ActivePolicy::init_kernel_pdl_trigger_max_bins) +{ + return ActivePolicy::init_kernel_pdl_trigger_max_bins; +} + +// TODO(bgruber): drop in CCCL 4.0 +template +_CCCL_HOST_DEVICE_API constexpr auto convert_pdl_trigger(long) +{ + return 0; +} + // TODO(bgruber): drop in CCCL 4.0 template _CCCL_HOST_DEVICE_API constexpr auto convert_legacy_policy() -> HistogramPolicy { using sweep = typename ActivePolicy::AgentHistogramPolicyT; - const auto kernel_config = HistogramKernelConfig{ + const auto kernel_config = HistogramPolicy::Kernel{ sweep::BLOCK_THREADS, sweep::PIXELS_PER_THREAD, sweep::VEC_SIZE, @@ -801,7 +816,10 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_legacy_policy() -> HistogramPolicy sweep::LOAD_MODIFIER, sweep::IS_RLE_COMPRESS, sweep::IS_WORK_STEALING}; - return {kernel_config, {kernel_config, 256 * sizeof(unsigned int), 0}, {kernel_config, 0, 0, 0, 0, 0}, 0}; + return {kernel_config, + {kernel_config, legacy_privatized_smem_bins * supported_counter_bytes, 0}, + {kernel_config, 0, 0, 0, 0, 0}, + convert_pdl_trigger(0)}; } // TODO(bgruber): drop in CCCL 4.0 @@ -835,6 +853,18 @@ public: } }; +template +struct max_policy_from_hub +{ + using type = typename PolicyHub::MaxPolicy; +}; + +template <> +struct max_policy_from_hub +{ + using type = void; +}; + template max_levels) @@ -1122,7 +1152,7 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t dispatch_even( for (int channel = 0; channel < NUM_ACTIVE_CHANNELS; ++channel) { - num_privatized_levels[channel] = 257; + num_privatized_levels[channel] = byte_sample_privatized_levels; int num_levels = num_output_levels[channel]; if (kernel_source.MayOverflow(static_cast(num_levels - 1), upper_level, lower_level, channel)) @@ -1410,12 +1440,7 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc * @param stream * CUDA stream to launch kernels within. Default is stream0. */ - template , - /* fallback_policy_hub */ - detail::histogram::policy_hub, - PolicyHub>::MaxPolicy, - bool IsByteSample> + template ::type, bool IsByteSample> CUB_RUNTIME_FUNCTION static cudaError_t DispatchRange( void* d_temp_storage, size_t& temp_storage_bytes, @@ -1430,16 +1455,14 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc ::cuda::std::bool_constant is_byte_sample, KernelSource kernel_source = {}, KernelLauncherFactory launcher_factory = {}, - [[maybe_unused]] MaxPolicyT max_policy = {}) + [[maybe_unused]] ::cuda::std::_If<::cuda::std::is_void_v, ::cuda::std::nullptr_t, MaxPolicyT> + max_policy = {}) { - using default_policy_hub = - detail::histogram::policy_hub; - static constexpr bool uses_default_policy = - ::cuda::std::is_void_v && ::cuda::std::is_same_v; - using policy_selector_t = ::cuda::std::_If< - uses_default_policy, - detail::histogram::policy_selector_from_types, - detail::histogram::policy_selector_from_hub>; + static constexpr bool uses_default_policy = ::cuda::std::is_void_v; + using policy_selector_t = ::cuda::std::_If< + uses_default_policy, + detail::histogram::policy_selector_from_types, + detail::histogram::policy_selector_from_hub>; return detail::histogram::dispatch_range( d_temp_storage, temp_storage_bytes, @@ -1504,12 +1527,7 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc * CUDA stream to launch kernels within. Default is stream0. * */ - template , - /* fallback_policy_hub */ - detail::histogram::policy_hub, - PolicyHub>::MaxPolicy, - bool IsByteSample> + template ::type, bool IsByteSample> CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t DispatchEven( void* d_temp_storage, size_t& temp_storage_bytes, @@ -1525,16 +1543,14 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc ::cuda::std::bool_constant is_byte_sample, KernelSource kernel_source = {}, KernelLauncherFactory launcher_factory = {}, - [[maybe_unused]] MaxPolicyT max_policy = {}) + [[maybe_unused]] ::cuda::std::_If<::cuda::std::is_void_v, ::cuda::std::nullptr_t, MaxPolicyT> + max_policy = {}) { - using default_policy_hub = - detail::histogram::policy_hub; - static constexpr bool uses_default_policy = - ::cuda::std::is_void_v && ::cuda::std::is_same_v; - using policy_selector_t = ::cuda::std::_If< - uses_default_policy, - detail::histogram::policy_selector_from_types, - detail::histogram::policy_selector_from_hub>; + static constexpr bool uses_default_policy = ::cuda::std::is_void_v; + using policy_selector_t = ::cuda::std::_If< + uses_default_policy, + detail::histogram::policy_selector_from_types, + detail::histogram::policy_selector_from_hub>; return detail::histogram::dispatch_even( d_temp_storage, temp_storage_bytes, diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index fef7f13a2342..acd7d5d2cbf4 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -704,7 +704,7 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(kernel_config(current_policy(), PrivatizationMode{}).threads_per_block), +__launch_bounds__(int(current_policy().kernel(PrivatizationMode{}).threads_per_block), int(is_privatized_static_smem_v ? current_policy().static_smem.min_blocks_per_sm : 0)) @@ -722,21 +722,8 @@ __launch_bounds__(int(kernel_config(current_policy(), Privatizat const int tiles_per_row, GridQueue tile_queue) { - static constexpr HistogramPolicy hp = current_policy(); - static constexpr auto sweep = kernel_config(hp, PrivatizationMode{}); - - // Thread block type for compositing input tiles - using AgentHistogramPolicyT = agent_histogram_policy< - sweep.threads_per_block, - sweep.items_per_thread, - sweep.load_algorithm, - sweep.load_modifier, - sweep.rle_compress, - sweep.work_stealing, - sweep.vec_size, - is_privatized_static_smem_v ? hp.static_smem.max_privatized_smem_bytes : 0>; using AgentHistogramT = - AgentHistogram().dynamic_smem.kernel.threa const int tiles_per_row, GridQueue tile_queue) { - static constexpr HistogramPolicy hp = current_policy(); - static constexpr HistogramKernelConfig sweep = hp.dynamic_smem.kernel; - - using AgentHistogramPolicyT = - agent_histogram_policy; using AgentHistogramT = - AgentHistogram #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(kernel_config(current_policy(), PrivatizationMode{}).threads_per_block)) +__launch_bounds__(int(current_policy().kernel(PrivatizationMode{}).threads_per_block)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDeviceInitKernel( const SampleIteratorT d_samples, ::cuda::std::array num_output_bins_wrapper, @@ -973,9 +949,6 @@ __launch_bounds__(int(kernel_config(current_policy(), Privatizat const int tiles_per_row, const GridQueue tile_queue) { - static constexpr HistogramPolicy hp = current_policy(); - static constexpr auto sweep = kernel_config(hp, PrivatizationMode{}); - OutputDecodeOpT output_decode_op[NumActiveChannels]; PrivatizedDecodeOpT privatized_decode_op[NumActiveChannels]; if constexpr (IsEven) @@ -1002,18 +975,8 @@ __launch_bounds__(int(kernel_config(current_policy(), Privatizat } } - // Thread block type for compositing input tiles - using AgentHistogramPolicyT = agent_histogram_policy< - sweep.threads_per_block, - sweep.items_per_thread, - sweep.load_algorithm, - sweep.load_modifier, - sweep.rle_compress, - sweep.work_stealing, - sweep.vec_size, - is_privatized_static_smem_v ? hp.static_smem.max_privatized_smem_bytes : 0>; using AgentHistogramT = - AgentHistogram + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const Kernel& kernel(PrivatizationMode) const { - return lhs.kernel == rhs.kernel && lhs.max_privatized_smem_bytes == rhs.max_privatized_smem_bytes - && lhs.range_max_bins == rhs.range_max_bins && lhs.even_2ch_max_bins == rhs.even_2ch_max_bins - && lhs.even_3ch_max_bins == rhs.even_3ch_max_bins && lhs.even_4ch_max_bins == rhs.even_4ch_max_bins; + if constexpr (detail::histogram::is_privatized_static_smem_v) + { + return static_smem.kernel; + } + else if constexpr (detail::histogram::is_privatized_dynamic_smem_v) + { + return dynamic_smem.kernel; + } + else + { + static_assert(detail::histogram::is_privatized_gmem_v); + return gmem; + } } -}; - -//! The tuning policy for all DeviceHistogram kernel variants. -struct HistogramPolicy -{ - HistogramKernelConfig gmem; - HistogramStaticSmemPolicy static_smem; - HistogramDynamicSmemPolicy dynamic_smem; - int init_kernel_pdl_trigger_max_bins; //!< Common init-kernel PDL threshold, independent of accumulation tier [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(const HistogramPolicy& lhs, const HistogramPolicy& rhs) noexcept @@ -104,7 +118,7 @@ struct HistogramPolicy #if _CCCL_HOSTED() friend ::std::ostream& operator<<(::std::ostream& os, const HistogramPolicy& p) { - const auto print_kernel = [&](const HistogramKernelConfig& kernel) -> ::std::ostream& { + const auto print_kernel = [&](const Kernel& kernel) -> ::std::ostream& { return os << "{ .threads_per_block = " << kernel.threads_per_block << ", .items_per_thread = " << kernel.items_per_thread << ", .vec_size = " << kernel.vec_size @@ -130,6 +144,20 @@ struct HistogramPolicy namespace detail::histogram { +// All DeviceHistogram tuning values live in this block. Keep architecture +// selection and policy construction below free of unexplained numeric values. +inline constexpr auto sm90 = ::cuda::compute_capability{9, 0}; +inline constexpr auto sm100 = ::cuda::compute_capability{10, 0}; +inline constexpr int supported_counter_bytes = 4; +inline constexpr int sample_u8_bytes = 1; +inline constexpr int sample_u16_bytes = 2; +inline constexpr int sample_u32_bytes = 4; +inline constexpr int sample_u64_bytes = 8; +inline constexpr int single_channel_count = 1; +inline constexpr int first_multi_channel_count = 2; +inline constexpr int two_active_channels = 2; +inline constexpr int three_active_channels = 3; +inline constexpr int four_active_channels = 4; inline constexpr int pre_sm100_static_smem_max_bins = 256; inline constexpr int sm100_static_smem_max_bins = 512; inline constexpr int pdl_trigger_max_bins = 2048; @@ -140,24 +168,34 @@ inline constexpr int sm100_range_dynamic_smem_max_bins = 2048; inline constexpr int sm100_even_2ch_dynamic_smem_max_bins = 28544; inline constexpr int sm100_even_3ch_dynamic_smem_max_bins = 19029; inline constexpr int sm100_even_4ch_dynamic_smem_max_bins = 8192; - -template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const HistogramKernelConfig& -kernel_config(const HistogramPolicy& policy, PrivatizationMode) +inline constexpr int legacy_privatized_smem_bins = 256; +inline constexpr int byte_sample_privatized_levels = legacy_privatized_smem_bins + 1; +inline constexpr int init_threads_per_block = 256; +inline constexpr int default_threads_per_block = 384; +inline constexpr int default_nominal_items_per_thread = 16; +inline constexpr int default_vec_size = 4; +inline constexpr int sm90_u8_threads_per_block = 768; +inline constexpr int sm90_u8_items_per_thread = 12; +inline constexpr int sm90_u16_threads_per_block = 960; +inline constexpr int sm90_u16_items_per_thread = 10; +inline constexpr int sm100_multi_threads_per_block = 1024; +inline constexpr int sm100_multi_even_nominal_items = 8; +inline constexpr int sm100_multi_range_nominal_items = 16; +inline constexpr int sm100_u8_even_threads_per_block = 928; +inline constexpr int sm100_u8_range_threads_per_block = 448; +inline constexpr int sm100_u8_items_per_thread = 12; +inline constexpr int sm100_u32_threads_per_block = 768; +inline constexpr int sm100_u32_items_per_thread = 12; +inline constexpr int sm100_u64_threads_per_block = 768; +inline constexpr int sm100_u64_items_per_thread = 6; +inline constexpr int sm100_range_static_threads_per_block = 384; +inline constexpr int sm100_range_u32_threads_per_block = 768; +inline constexpr int sm100_range_u64_nominal_items = 16; +inline constexpr int sm100_range_static_min_blocks_per_sm = 3; + +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool supports_dependent_launch(::cuda::compute_capability cc) { - if constexpr (is_privatized_static_smem_v) - { - return policy.static_smem.kernel; - } - else if constexpr (is_privatized_dynamic_smem_v) - { - return policy.dynamic_smem.kernel; - } - else - { - static_assert(is_privatized_gmem_v); - return policy.gmem; - } + return cc >= sm90; } [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int @@ -202,18 +240,19 @@ should_use_dynamic_smem(const HistogramPolicy& policy, int num_bins, int counter return false; } - const bool prefer_dynamic_smem = counter_size > int{sizeof(unsigned int)} + const bool prefer_dynamic_smem = counter_size > supported_counter_bytes || !should_use_static_smem(policy, num_bins, counter_size, num_active_channels); int max_bins = max_privatized_dynamic_smem_bins(policy, counter_size, num_active_channels); - if (num_active_channels > 1) + if (num_active_channels >= first_multi_channel_count) { if constexpr (IsEven) { - max_bins = num_active_channels == 2 ? policy.dynamic_smem.even_2ch_max_bins - : num_active_channels == 3 - ? policy.dynamic_smem.even_3ch_max_bins - : policy.dynamic_smem.even_4ch_max_bins; + max_bins = num_active_channels == two_active_channels ? policy.dynamic_smem.even_2ch_max_bins + : num_active_channels == three_active_channels ? policy.dynamic_smem.even_3ch_max_bins + : num_active_channels == four_active_channels + ? policy.dynamic_smem.even_4ch_max_bins + : 0; } else { @@ -224,211 +263,6 @@ should_use_dynamic_smem(const HistogramPolicy& policy, int num_bins, int counter return prefer_dynamic_smem && max_bins > 0 && num_bins <= max_bins; } -// TODO(bgruber): drop in CCCL 4.0 -enum class primitive_sample -{ - no, - yes -}; - -// TODO(bgruber): drop in CCCL 4.0 -enum class sample_size -{ - _1, - _2, - _4, - _8, - unknown -}; - -// TODO(bgruber): drop in CCCL 4.0 -enum class counter_size -{ - _4, - unknown -}; - -// TODO(bgruber): drop in CCCL 4.0 -template -_CCCL_HOST_DEVICE_API constexpr primitive_sample is_primitive_sample() -{ - return is_primitive::value ? primitive_sample::yes : primitive_sample::no; -} - -// TODO(bgruber): drop in CCCL 4.0 -template -_CCCL_HOST_DEVICE_API constexpr counter_size classify_counter_size() -{ - return sizeof(CounterT) == 4 ? counter_size::_4 : counter_size::unknown; -} - -// TODO(bgruber): drop in CCCL 4.0 -template -_CCCL_HOST_DEVICE_API constexpr sample_size classify_sample_size() -{ - return sizeof(SampleT) == 1 ? sample_size::_1 - : sizeof(SampleT) == 2 ? sample_size::_2 - : sizeof(SampleT) == 4 ? sample_size::_4 - : sizeof(SampleT) == 8 - ? sample_size::_8 - : sample_size::unknown; -} - -// TODO(bgruber): drop in CCCL 4.0 -template (), - sample_size SampleSize = classify_sample_size()> -struct sm90_tuning; - -template -struct sm90_tuning -{ - static constexpr int threads_per_block = 768; - static constexpr int items_per_thread = 12; - - static constexpr CacheLoadModifier load_modifier = LOAD_LDG; - - static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_DIRECT; - - static constexpr bool rle_compress = false; - static constexpr bool work_stealing = false; -}; - -template -struct sm90_tuning -{ - static constexpr int threads_per_block = 960; - static constexpr int items_per_thread = 10; - - static constexpr CacheLoadModifier load_modifier = LOAD_DEFAULT; - - static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_DIRECT; - - static constexpr bool rle_compress = true; - static constexpr bool work_stealing = false; -}; - -// TODO(bgruber): drop in CCCL 4.0 -template (), - sample_size SampleSize = classify_sample_size()> -struct sm100_tuning; - -// even -template -struct sm100_tuning -{ - // ipt_12.tpb_928.rle_0.ws_0.mem_1.ld_2.laid_0.vec_2 1.033332 0.940517 1.031835 1.195876 - static constexpr int items_per_thread = 12; - static constexpr int threads_per_block = 928; - static constexpr bool rle_compress = false; - static constexpr bool work_stealing = false; - static constexpr CacheLoadModifier load_modifier = LOAD_CA; - static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_DIRECT; - static constexpr int vec_size = 1 << 2; -}; - -// range -template -struct sm100_tuning -{ - // ipt_12.tpb_448.rle_0.ws_0.mem_1.ld_1.laid_0.vec_2 1.078987 0.985542 1.085118 1.175637 - static constexpr int items_per_thread = 12; - static constexpr int threads_per_block = 448; - static constexpr bool rle_compress = false; - static constexpr bool work_stealing = false; - static constexpr CacheLoadModifier load_modifier = LOAD_LDG; - static constexpr BlockLoadAlgorithm load_algorithm = BLOCK_LOAD_DIRECT; - static constexpr int vec_size = 1 << 2; -}; - -// sample_size 2/4/8 retain the SM90 launch shape in the legacy policy hub. - -// TODO(bgruber): drop in CCCL 4.0 -template -struct policy_hub -{ - // TODO(bgruber): move inside t_scale in C++14 - static constexpr int v_scale = (sizeof(SampleT) + sizeof(int) - 1) / sizeof(int); - - _CCCL_HOST_DEVICE_API static constexpr int t_scale(int nominalItemsPerThread) - { - return (::cuda::std::max) (nominalItemsPerThread / NumActiveChannels / v_scale, 1); - } - - // SM50 - struct Policy500 : detail::chained_policy<500, Policy500, Policy500> - { - // TODO This might be worth it to separate usual histogram and the multi one - using AgentHistogramPolicyT = agent_histogram_policy<384, t_scale(16), BLOCK_LOAD_DIRECT, LOAD_LDG, true, false>; - static constexpr int init_kernel_pdl_trigger_max_bins = 0; - }; - - // SM90 - struct Policy900 : detail::chained_policy<900, Policy900, Policy500> - { - // Use values from tuning if a specialization exists, otherwise pick Policy500 - template - _CCCL_HOST_DEVICE_API static auto select_agent_policy(int) - -> agent_histogram_policy; - - template - _CCCL_HOST_DEVICE_API static auto select_agent_policy(long) -> typename Policy500::AgentHistogramPolicyT; - - using AgentHistogramPolicyT = - decltype(select_agent_policy< - sm90_tuning()>>(0)); - - static constexpr int init_kernel_pdl_trigger_max_bins = - NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value - && (sizeof(SampleT) == 1 || sizeof(SampleT) == 2) - ? pdl_trigger_max_bins - : 0; - }; - - struct Policy1000 : detail::chained_policy<1000, Policy1000, Policy900> - { - // Use values from tuning if a specialization exists, otherwise pick Policy900 - template - _CCCL_HOST_DEVICE_API static auto select_agent_policy(int) - -> agent_histogram_policy; - - template - _CCCL_HOST_DEVICE_API static auto select_agent_policy(long) -> typename Policy900::AgentHistogramPolicyT; - - using AgentHistogramPolicyT = - decltype(select_agent_policy< - sm100_tuning()>>( - 0)); - - static constexpr int init_kernel_pdl_trigger_max_bins = - NumChannels == 1 && NumActiveChannels == 1 && sizeof(CounterT) == 4 && is_primitive::value - && (sizeof(SampleT) == 1 || sizeof(SampleT) == 2 || sizeof(SampleT) == 4 || sizeof(SampleT) == 8) - ? pdl_trigger_max_bins - : 0; - }; - - using MaxPolicy = Policy1000; -}; - #if _CCCL_HAS_CONCEPTS() template concept histogram_policy_selector = policy_selector; @@ -450,117 +284,171 @@ private: return (::cuda::std::max) (nominal_items_per_thread / num_active_channels / sample_scale, 1); } - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto default_kernel_config() const -> HistogramKernelConfig + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_sm50_tuning() const -> HistogramPolicy { - return {384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + const auto kernel = HistogramPolicy::Kernel{ + default_threads_per_block, + t_scale(default_nominal_items_per_thread), + default_vec_size, + BLOCK_LOAD_DIRECT, + LOAD_LDG, + true, + false}; + return {kernel, + {kernel, pre_sm100_static_smem_max_bins * counter_size_bytes * num_active_channels, 0}, + {kernel, 0, 0, 0, 0, 0}, + 0}; } - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto sm90_kernel_config() const -> HistogramKernelConfig + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_sm90_tuning() const -> HistogramPolicy { - if (num_channels == 1 && num_active_channels == 1 && counter_size_bytes == 4 && sample_is_primitive) + auto result = get_sm50_tuning(); + if (num_channels == single_channel_count && num_active_channels == single_channel_count + && counter_size_bytes == supported_counter_bytes && sample_is_primitive) { - if (sample_size_bytes == 1) + if (sample_size_bytes == sample_u8_bytes) { - return {768, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, false, false}; + result.gmem = { + sm90_u8_threads_per_block, + sm90_u8_items_per_thread, + default_vec_size, + BLOCK_LOAD_DIRECT, + LOAD_LDG, + false, + false}; } - if (sample_size_bytes == 2) + else if (sample_size_bytes == sample_u16_bytes) { - return {960, 10, 4, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, false}; + result.gmem = { + sm90_u16_threads_per_block, + sm90_u16_items_per_thread, + default_vec_size, + BLOCK_LOAD_DIRECT, + LOAD_DEFAULT, + true, + false}; } + result.static_smem.kernel = result.gmem; + result.dynamic_smem.kernel = result.gmem; + result.init_kernel_pdl_trigger_max_bins = + sample_size_bytes == sample_u8_bytes || sample_size_bytes == sample_u16_bytes ? pdl_trigger_max_bins : 0; } - return default_kernel_config(); + return result; } - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto sm100_kernel_config() const -> HistogramKernelConfig + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_sm100_tuning() const -> HistogramPolicy { - if (num_channels >= 2 && counter_size_bytes == 4 && sample_is_primitive) + auto result = get_sm90_tuning(); + const bool single_channel = num_channels == single_channel_count && num_active_channels == single_channel_count; + if (num_channels >= first_multi_channel_count && counter_size_bytes == supported_counter_bytes + && sample_is_primitive) { - return {1024, t_scale(is_even ? 8 : 16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + result.gmem = { + sm100_multi_threads_per_block, + t_scale(is_even ? sm100_multi_even_nominal_items : sm100_multi_range_nominal_items), + default_vec_size, + BLOCK_LOAD_DIRECT, + LOAD_LDG, + true, + false}; } - if (num_channels == 1 && num_active_channels == 1 && counter_size_bytes == 4 && sample_is_primitive) + else if (single_channel && counter_size_bytes == supported_counter_bytes && sample_is_primitive) { - if (sample_size_bytes == 1) + if (sample_size_bytes == sample_u8_bytes) { - return is_even ? HistogramKernelConfig{928, 12, 4, BLOCK_LOAD_DIRECT, LOAD_CA, false, false} - : HistogramKernelConfig{448, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, false, false}; + result.gmem = + is_even + ? HistogramPolicy::Kernel{sm100_u8_even_threads_per_block, + sm100_u8_items_per_thread, + default_vec_size, + BLOCK_LOAD_DIRECT, + LOAD_CA, + false, + false} + : HistogramPolicy::Kernel{ + sm100_u8_range_threads_per_block, + sm100_u8_items_per_thread, + default_vec_size, + BLOCK_LOAD_DIRECT, + LOAD_LDG, + false, + false}; } - if (sample_size_bytes == 4) + else if (sample_size_bytes == sample_u32_bytes) { - return {768, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + result.gmem = { + sm100_u32_threads_per_block, + sm100_u32_items_per_thread, + default_vec_size, + BLOCK_LOAD_DIRECT, + LOAD_LDG, + true, + false}; } - if (sample_size_bytes == 8) + else if (sample_size_bytes == sample_u64_bytes) { - return {768, 6, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + result.gmem = { + sm100_u64_threads_per_block, + sm100_u64_items_per_thread, + default_vec_size, + BLOCK_LOAD_DIRECT, + LOAD_LDG, + true, + false}; } } - return sm90_kernel_config(); - } -public: - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy - { - const bool single_channel = num_channels == 1 && num_active_channels == 1; - if (cc >= ::cuda::compute_capability{10, 0}) + result.static_smem.kernel = result.gmem; + result.dynamic_smem.kernel = result.gmem; + const bool range_multi_static = !is_even && num_channels >= first_multi_channel_count + && counter_size_bytes == supported_counter_bytes && sample_is_primitive; + const bool range_u32_static = !is_even && single_channel && counter_size_bytes == supported_counter_bytes + && sample_is_primitive && sample_size_bytes == sample_u32_bytes; + const bool range_u64_static = !is_even && single_channel && counter_size_bytes == supported_counter_bytes + && sample_is_primitive && sample_size_bytes == sample_u64_bytes; + if (range_multi_static || range_u64_static) { - const HistogramKernelConfig kernel = sm100_kernel_config(); - const bool range_multi_static = !is_even && num_channels >= 2 && counter_size_bytes == 4 && sample_is_primitive; - const bool range_u32_static = - !is_even && single_channel && counter_size_bytes == 4 && sample_is_primitive && sample_size_bytes == 4; - const bool range_u64_static = - !is_even && single_channel && counter_size_bytes == 4 && sample_is_primitive && sample_size_bytes == 8; - HistogramKernelConfig static_kernel = kernel; - if (range_multi_static || range_u64_static) - { - static_kernel.threads_per_block = 384; - } - else if (range_u32_static) - { - static_kernel.threads_per_block = 768; - } - if (range_u64_static) - { - static_kernel.items_per_thread = t_scale(16); - } - - const bool has_dynamic_smem_tuning = - counter_size_bytes == 4 && sample_is_primitive - && ((single_channel && (sample_size_bytes == 1 || sample_size_bytes == 4 || sample_size_bytes == 8)) - || num_channels >= 2); - const int dynamic_smem_bytes = has_dynamic_smem_tuning ? sm100_dynamic_smem_max_bytes : 0; - const int dynamic_range_bins = has_dynamic_smem_tuning ? sm100_range_dynamic_smem_max_bins : 0; - const int dynamic_even_2ch_bins = has_dynamic_smem_tuning ? sm100_even_2ch_dynamic_smem_max_bins : 0; - const int dynamic_even_3ch_bins = has_dynamic_smem_tuning ? sm100_even_3ch_dynamic_smem_max_bins : 0; - const int dynamic_even_4ch_bins = has_dynamic_smem_tuning ? sm100_even_4ch_dynamic_smem_max_bins : 0; - const int pdl_bins = - single_channel && counter_size_bytes == 4 && sample_is_primitive - && (sample_size_bytes == 1 || sample_size_bytes == 2 || sample_size_bytes == 4 || sample_size_bytes == 8) - ? pdl_trigger_max_bins - : 0; - return { - kernel, - {static_kernel, - sm100_static_smem_max_bins * counter_size_bytes * num_active_channels, - range_multi_static || range_u64_static ? 3 : 0}, - {kernel, - dynamic_smem_bytes, - dynamic_range_bins, - dynamic_even_2ch_bins, - dynamic_even_3ch_bins, - dynamic_even_4ch_bins}, - pdl_bins}; + result.static_smem.kernel.threads_per_block = sm100_range_static_threads_per_block; } - - const HistogramKernelConfig kernel = - cc >= ::cuda::compute_capability{9, 0} ? sm90_kernel_config() : default_kernel_config(); - const int pdl_bins = - cc >= ::cuda::compute_capability{9, 0} && single_channel && counter_size_bytes == 4 && sample_is_primitive - && (sample_size_bytes == 1 || sample_size_bytes == 2) + else if (range_u32_static) + { + result.static_smem.kernel.threads_per_block = sm100_range_u32_threads_per_block; + } + if (range_u64_static) + { + result.static_smem.kernel.items_per_thread = t_scale(sm100_range_u64_nominal_items); + } + result.static_smem.max_privatized_smem_bytes = + sm100_static_smem_max_bins * counter_size_bytes * num_active_channels; + result.static_smem.min_blocks_per_sm = + range_multi_static || range_u64_static ? sm100_range_static_min_blocks_per_sm : 0; + + const bool has_dynamic_smem_tuning = + counter_size_bytes == supported_counter_bytes && sample_is_primitive + && ((single_channel + && (sample_size_bytes == sample_u8_bytes || sample_size_bytes == sample_u32_bytes + || sample_size_bytes == sample_u64_bytes)) + || num_channels >= first_multi_channel_count); + result.dynamic_smem = { + result.gmem, + has_dynamic_smem_tuning ? sm100_dynamic_smem_max_bytes : 0, + has_dynamic_smem_tuning ? sm100_range_dynamic_smem_max_bins : 0, + has_dynamic_smem_tuning ? sm100_even_2ch_dynamic_smem_max_bins : 0, + has_dynamic_smem_tuning ? sm100_even_3ch_dynamic_smem_max_bins : 0, + has_dynamic_smem_tuning ? sm100_even_4ch_dynamic_smem_max_bins : 0}; + result.init_kernel_pdl_trigger_max_bins = + single_channel && counter_size_bytes == supported_counter_bytes && sample_is_primitive + && (sample_size_bytes == sample_u8_bytes || sample_size_bytes == sample_u16_bytes + || sample_size_bytes == sample_u32_bytes || sample_size_bytes == sample_u64_bytes) ? pdl_trigger_max_bins : 0; - return {kernel, - {kernel, pre_sm100_static_smem_max_bins * counter_size_bytes * num_active_channels, 0}, - {kernel, 0, 0, 0, 0, 0}, - pdl_bins}; + return result; + } + +public: + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy + { + return cc >= sm100 ? get_sm100_tuning() : cc >= sm90 ? get_sm90_tuning() : get_sm50_tuning(); } }; diff --git a/cub/test/catch2_test_device_histogram_custom_policy_hub.cu b/cub/test/catch2_test_device_histogram_custom_policy_hub.cu index 8dc8649f7ff8..6b9f41c218e3 100644 --- a/cub/test/catch2_test_device_histogram_custom_policy_hub.cu +++ b/cub/test/catch2_test_device_histogram_custom_policy_hub.cu @@ -20,16 +20,37 @@ using namespace cub; template struct my_policy_hub { - // simplified from Policy500 of the CUB histogram tunings - struct MaxPolicy : cub::detail::chained_policy<500, MaxPolicy, MaxPolicy> + struct Policy500 : cub::detail::chained_policy<500, Policy500, Policy500> { using AgentHistogramPolicyT = AgentHistogramPolicy<384, 16, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false>; + static constexpr int init_kernel_pdl_trigger_max_bins = 0; + }; + + struct Policy900 : cub::detail::chained_policy<900, Policy900, Policy500> + { + using AgentHistogramPolicyT = AgentHistogramPolicy<256, 8, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, false, false>; static constexpr int init_kernel_pdl_trigger_max_bins = 2048; }; + + using MaxPolicy = Policy900; }; CUB_TEST("DispatchHistogram::DispatchEven: custom policy hub", "[histogram][device]", CUB_SMALL) { + using custom_max_policy_t = typename my_policy_hub::MaxPolicy; + const auto custom_sm75_policy = + cub::detail::histogram::policy_selector_from_hub{}(cuda::compute_capability{7, 5}); + const auto custom_sm90_policy = + cub::detail::histogram::policy_selector_from_hub{}(cuda::compute_capability{9, 0}); + REQUIRE(custom_sm75_policy.gmem.threads_per_block == 384); + REQUIRE(custom_sm75_policy.gmem.items_per_thread == 16); + REQUIRE(custom_sm90_policy.gmem.threads_per_block == 256); + REQUIRE(custom_sm90_policy.gmem.items_per_thread == 8); + REQUIRE(custom_sm75_policy.static_smem.max_privatized_smem_bytes == 256 * sizeof(unsigned int)); + REQUIRE(custom_sm90_policy.dynamic_smem.max_privatized_smem_bytes == 0); + REQUIRE(custom_sm75_policy.init_kernel_pdl_trigger_max_bins == 0); + REQUIRE(custom_sm90_policy.init_kernel_pdl_trigger_max_bins == 2048); + using sample_t = cuda::std::uint8_t; using counter_t = int; using level_t = int; diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index ac488a59314d..3784a634fa9b 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1627,7 +1627,7 @@ struct histogram_tuning _CCCL_HOST_DEVICE_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { constexpr auto sweep = - cub::HistogramKernelConfig{BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; + cub::HistogramPolicy::Kernel{BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; return {sweep, {sweep, 256 * sizeof(unsigned int), 0}, {sweep, 0, 0, 0, 0, 0}, 0}; } }; @@ -1645,7 +1645,7 @@ struct mixed_counter_histogram_tuning _CCCL_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { constexpr auto sweep = - cub::HistogramKernelConfig{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; + cub::HistogramPolicy::Kernel{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; return {sweep, {sweep, 512 * sizeof(unsigned int), 0}, {sweep, 228352, 2048, 28544, 19029, 8192}, 0}; } }; @@ -1928,11 +1928,5 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 28545, 4, 2)); STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 19029, 4, 3)); STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 19030, 4, 3)); - - using max_policy_t = typename cub::detail::histogram::policy_hub::MaxPolicy; - const auto legacy_policy = - cub::detail::histogram::policy_selector_from_hub{}(cuda::compute_capability{10, 0}); - REQUIRE(legacy_policy.static_smem.max_privatized_smem_bytes == 256 * sizeof(unsigned int)); - REQUIRE(legacy_policy.dynamic_smem.max_privatized_smem_bytes == 0); } #endif // _CCCL_COMPILER(GCC, >=, 8) diff --git a/cub/test/catch2_test_device_histogram_env_api.cu b/cub/test/catch2_test_device_histogram_env_api.cu index ad64e49eae51..8bdbfd7980cf 100644 --- a/cub/test/catch2_test_device_histogram_env_api.cu +++ b/cub/test/catch2_test_device_histogram_env_api.cu @@ -418,7 +418,7 @@ struct HistogramPolicySelector { __host__ __device__ constexpr auto operator()(cuda::compute_capability cc) const -> cub::HistogramPolicy { - const auto sweep = cub::HistogramKernelConfig{ + const auto sweep = cub::HistogramPolicy::Kernel{ 128, cc > cuda::compute_capability{9, 0} ? 16 : 7, 4, cub::BLOCK_LOAD_DIRECT, cub::LOAD_LDG, false, false}; return { .gmem = sweep, From 005058896f0d0d8d36f706b67a0a942819713d55 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Fri, 7 Aug 2026 10:45:41 +0000 Subject: [PATCH 33/45] [cub] Follow established histogram tuning conventions --- .../bench/histogram/histogram_common.cuh | 23 +- cub/cub/agent/agent_histogram.cuh | 8 +- .../device/dispatch/dispatch_histogram.cuh | 39 +- .../dispatch/kernels/kernel_histogram.cuh | 20 +- .../dispatch/tuning/tuning_histogram.cuh | 475 ++++++------------ ...test_device_histogram_custom_policy_hub.cu | 4 +- cub/test/catch2_test_device_histogram_env.cu | 141 +++--- .../catch2_test_device_histogram_env_api.cu | 22 +- 8 files changed, 308 insertions(+), 424 deletions(-) diff --git a/cub/benchmarks/bench/histogram/histogram_common.cuh b/cub/benchmarks/bench/histogram/histogram_common.cuh index d71579b1f10f..d59b870070c0 100644 --- a/cub/benchmarks/bench/histogram/histogram_common.cuh +++ b/cub/benchmarks/bench/histogram/histogram_common.cuh @@ -37,7 +37,7 @@ struct bench_policy_selector ? (NUM_CHANNELS == 1 ? cub::BLOCK_LOAD_STRIPED : cub::BLOCK_LOAD_DIRECT) : TUNE_LOAD_ALGORITHM; - constexpr auto sweep = cub::HistogramPolicy::Kernel{ + constexpr auto sweep = cub::HistogramSweepPolicy{ TUNE_THREADS, TUNE_ITEMS, TUNE_VEC_SIZE, @@ -45,15 +45,18 @@ struct bench_policy_selector TUNE_LOAD_MODIFIER, TUNE_RLE_COMPRESS, TUNE_WORK_STEALING}; - return {sweep, - {sweep, TUNE_STATIC_SMEM_MAX_BYTES, TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM}, - {sweep, - TUNE_DYNAMIC_SMEM_MAX_BYTES, - TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS, - TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS, - TUNE_DYNAMIC_SMEM_EVEN_3CH_MAX_BINS, - TUNE_DYNAMIC_SMEM_EVEN_4CH_MAX_BINS}, - TUNE_INIT_KERNEL_PDL_TRIGGER_MAX_BINS}; + return { + sweep, + sweep, + sweep, + TUNE_STATIC_SMEM_MAX_BYTES, + TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM, + TUNE_DYNAMIC_SMEM_MAX_BYTES, + TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS, + TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS, + TUNE_DYNAMIC_SMEM_EVEN_3CH_MAX_BINS, + TUNE_DYNAMIC_SMEM_EVEN_4CH_MAX_BINS, + TUNE_INIT_KERNEL_PDL_TRIGGER_MAX_BINS}; } }; #endif // !TUNE_BASE diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index e5cf64918a54..8a5726789abe 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -157,9 +157,13 @@ struct AgentHistogram static constexpr bool uses_dynamic_smem = is_privatized_dynamic_smem_v; static constexpr bool uses_gmem = is_privatized_gmem_v; static constexpr auto policy = current_policy(); - static constexpr auto sweep = policy.kernel(PrivatizationMode{}); + static constexpr auto sweep = + uses_static_smem ? policy.static_smem + : uses_dynamic_smem + ? policy.dynamic_smem + : policy.gmem; static constexpr int privatized_static_smem_bins = - uses_static_smem ? policy.static_smem.max_privatized_smem_bytes / int{sizeof(CounterT)} / NumActiveChannels : 0; + uses_static_smem ? policy.max_privatized_static_smem_bytes / int{sizeof(CounterT)} / NumActiveChannels : 0; static_assert(!uses_static_smem || privatized_static_smem_bins > 0, "Static-SMEM privatization requires room for at least one bin"); static constexpr int vec_size = sweep.vec_size; diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 1dccefd81d45..646eac045166 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -54,6 +54,9 @@ CUB_NAMESPACE_BEGIN namespace detail::histogram { +inline constexpr int byte_sample_privatized_levels = + static_cast(::cuda::std::numeric_limits::max()) + 2; + template struct local_counter { @@ -276,9 +279,13 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( } }(); - const HistogramPolicy::Kernel sweep = active_policy.kernel(PrivatizationMode{}); - const int threads_per_block = sweep.threads_per_block; - const int items_per_thread = sweep.items_per_thread; + const HistogramSweepPolicy sweep = + is_privatized_static_smem_v ? active_policy.static_smem + : is_privatized_dynamic_smem_v + ? active_policy.dynamic_smem + : active_policy.gmem; + const int threads_per_block = sweep.threads_per_block; + const int items_per_thread = sweep.items_per_thread; int dynamic_smem_bytes = 0; if constexpr (is_privatized_dynamic_smem_v) @@ -289,7 +296,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( } NV_IF_TARGET(NV_IS_HOST, ({ if (const auto error = CubDebug(launcher_factory.set_max_dynamic_smem_size_for( - sweep_kernel, active_policy.dynamic_smem.max_privatized_smem_bytes))) + sweep_kernel, active_policy.max_privatized_dynamic_smem_bytes))) { return error; } @@ -385,7 +392,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( num_privatized_levels.begin(), num_privatized_levels.end(), num_privatized_bins_wrapper.begin(), minus_one); ::cuda::std::transform(num_output_levels.begin(), num_output_levels.end(), num_output_bins_wrapper.begin(), minus_one); - constexpr int histogram_init_threads_per_block = init_threads_per_block; + constexpr int histogram_init_threads_per_block = 256; int histogram_init_grid_dims = (max_num_output_bins + histogram_init_threads_per_block - 1) / histogram_init_threads_per_block; @@ -403,7 +410,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( histogram_init_threads_per_block, 0, stream, - /* dependent_launch */ supports_dependent_launch(cc)) + /* dependent_launch */ cc >= ::cuda::compute_capability{9, 0}) .doit(init_kernel, num_output_bins_wrapper, d_output_histograms, tile_queue))) { return error; @@ -434,7 +441,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( threads_per_block, dynamic_smem_bytes, stream, - /* dependent_launch */ supports_dependent_launch(cc)) + /* dependent_launch */ cc >= ::cuda::compute_capability{9, 0}) .doit(sweep_kernel, d_samples, num_output_bins_wrapper, @@ -808,7 +815,7 @@ template _CCCL_HOST_DEVICE_API constexpr auto convert_legacy_policy() -> HistogramPolicy { using sweep = typename ActivePolicy::AgentHistogramPolicyT; - const auto kernel_config = HistogramPolicy::Kernel{ + const auto kernel_config = HistogramSweepPolicy{ sweep::BLOCK_THREADS, sweep::PIXELS_PER_THREAD, sweep::VEC_SIZE, @@ -816,10 +823,18 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_legacy_policy() -> HistogramPolicy sweep::LOAD_MODIFIER, sweep::IS_RLE_COMPRESS, sweep::IS_WORK_STEALING}; - return {kernel_config, - {kernel_config, legacy_privatized_smem_bins * supported_counter_bytes, 0}, - {kernel_config, 0, 0, 0, 0, 0}, - convert_pdl_trigger(0)}; + return { + kernel_config, + kernel_config, + kernel_config, + 256 * int{sizeof(unsigned int)}, + 0, + 0, + 0, + 0, + 0, + 0, + convert_pdl_trigger(0)}; } // TODO(bgruber): drop in CCCL 4.0 diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index acd7d5d2cbf4..602dc17d63b2 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -704,10 +704,14 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(current_policy().kernel(PrivatizationMode{}).threads_per_block), - int(is_privatized_static_smem_v - ? current_policy().static_smem.min_blocks_per_sm - : 0)) +__launch_bounds__( + int(is_privatized_static_smem_v ? current_policy().static_smem.threads_per_block + : is_privatized_dynamic_smem_v + ? current_policy().dynamic_smem.threads_per_block + : current_policy().gmem.threads_per_block), + int(is_privatized_static_smem_v + ? current_policy().static_smem_min_blocks_per_sm + : 0)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepKernel( const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, @@ -777,7 +781,7 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(current_policy().dynamic_smem.kernel.threads_per_block)) +__launch_bounds__(int(current_policy().dynamic_smem.threads_per_block)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDynamicSmemKernel( const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, @@ -934,7 +938,11 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(current_policy().kernel(PrivatizationMode{}).threads_per_block)) +__launch_bounds__(int( + is_privatized_static_smem_v ? current_policy().static_smem.threads_per_block + : is_privatized_dynamic_smem_v + ? current_policy().dynamic_smem.threads_per_block + : current_policy().gmem.threads_per_block)) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDeviceInitKernel( const SampleIteratorT d_samples, ::cuda::std::array num_output_bins_wrapper, diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 20910f0a8575..f5bc98bd1286 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -25,87 +25,64 @@ CUB_NAMESPACE_BEGIN -//! The tuning policy for all DeviceHistogram kernel variants. -struct HistogramPolicy +//! The tuning policy for one DeviceHistogram sweep pass. +struct HistogramSweepPolicy { - struct Kernel - { - int threads_per_block; //!< Number of threads in a CUDA block - int items_per_thread; //!< Number of items processed per thread - int vec_size; //!< Vectorization size for loading samples - BlockLoadAlgorithm load_algorithm; //!< Algorithm used for loading samples - CacheLoadModifier load_modifier; //!< Cache modifier used for loading samples - bool rle_compress; //!< Whether to locally run-length encode samples - bool work_stealing; //!< Whether blocks dequeue tiles from a global queue - - [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(const Kernel& lhs, const Kernel& rhs) noexcept - { - return lhs.threads_per_block == rhs.threads_per_block && lhs.items_per_thread == rhs.items_per_thread - && lhs.vec_size == rhs.vec_size && lhs.load_algorithm == rhs.load_algorithm - && lhs.load_modifier == rhs.load_modifier && lhs.rle_compress == rhs.rle_compress - && lhs.work_stealing == rhs.work_stealing; - } - }; + int threads_per_block; //!< Number of threads in a CUDA block + int items_per_thread; //!< Number of items processed per thread + int vec_size; //!< Vectorization size for loading samples + BlockLoadAlgorithm load_algorithm; //!< Algorithm used for loading samples + CacheLoadModifier load_modifier; //!< Cache modifier used for loading samples + bool rle_compress; //!< Whether to locally run-length encode samples + bool work_stealing; //!< Whether blocks dequeue tiles from a global work queue - struct StaticSmem + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool + operator==(const HistogramSweepPolicy& lhs, const HistogramSweepPolicy& rhs) noexcept { - Kernel kernel; - int max_privatized_smem_bytes; //!< Maximum compile-time-sized shared-memory allocation - int min_blocks_per_sm; //!< Minimum blocks per SM requested through launch bounds - - [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool - operator==(const StaticSmem& lhs, const StaticSmem& rhs) noexcept - { - return lhs.kernel == rhs.kernel && lhs.max_privatized_smem_bytes == rhs.max_privatized_smem_bytes - && lhs.min_blocks_per_sm == rhs.min_blocks_per_sm; - } - }; + return lhs.threads_per_block == rhs.threads_per_block && lhs.items_per_thread == rhs.items_per_thread + && lhs.vec_size == rhs.vec_size && lhs.load_algorithm == rhs.load_algorithm + && lhs.load_modifier == rhs.load_modifier && lhs.rle_compress == rhs.rle_compress + && lhs.work_stealing == rhs.work_stealing; + } - struct DynamicSmem +#if _CCCL_HOSTED() + friend ::std::ostream& operator<<(::std::ostream& os, const HistogramSweepPolicy& p) { - Kernel kernel; - int max_privatized_smem_bytes; //!< Maximum runtime-sized shared-memory allocation - int range_max_bins; //!< Maximum bins per channel for multi-channel HistogramRange - int even_2ch_max_bins; //!< Maximum bins per channel for two-channel HistogramEven - int even_3ch_max_bins; //!< Maximum bins per channel for three-channel HistogramEven - int even_4ch_max_bins; //!< Maximum bins per channel for four-channel HistogramEven - - [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool - operator==(const DynamicSmem& lhs, const DynamicSmem& rhs) noexcept - { - return lhs.kernel == rhs.kernel && lhs.max_privatized_smem_bytes == rhs.max_privatized_smem_bytes - && lhs.range_max_bins == rhs.range_max_bins && lhs.even_2ch_max_bins == rhs.even_2ch_max_bins - && lhs.even_3ch_max_bins == rhs.even_3ch_max_bins && lhs.even_4ch_max_bins == rhs.even_4ch_max_bins; - } - }; + return os + << "HistogramSweepPolicy { .threads_per_block = " << p.threads_per_block + << ", .items_per_thread = " << p.items_per_thread << ", .vec_size = " << p.vec_size + << ", .load_algorithm = " << p.load_algorithm << ", .load_modifier = " << p.load_modifier + << ", .rle_compress = " << p.rle_compress << ", .work_stealing = " << p.work_stealing << " }"; + } +#endif // _CCCL_HOSTED() +}; - Kernel gmem; - StaticSmem static_smem; - DynamicSmem dynamic_smem; +//! The tuning policy for all DeviceHistogram sweep passes. +struct HistogramPolicy +{ + HistogramSweepPolicy gmem; //!< Policy for global-memory privatization + HistogramSweepPolicy static_smem; //!< Policy for compile-time-sized shared-memory privatization + HistogramSweepPolicy dynamic_smem; //!< Policy for runtime-sized shared-memory privatization + int max_privatized_static_smem_bytes; //!< Maximum compile-time-sized shared-memory allocation + int static_smem_min_blocks_per_sm; //!< Minimum blocks per SM requested by the static-SMEM launch bounds + int max_privatized_dynamic_smem_bytes; //!< Maximum runtime-sized shared-memory allocation + int dynamic_smem_range_max_bins; //!< Multi-channel HistogramRange limit, in bins per channel + int dynamic_smem_even_2ch_max_bins; //!< Two-channel HistogramEven limit, in bins per channel + int dynamic_smem_even_3ch_max_bins; //!< Three-channel HistogramEven limit, in bins per channel + int dynamic_smem_even_4ch_max_bins; //!< Four-channel HistogramEven limit, in bins per channel int init_kernel_pdl_trigger_max_bins; //!< Common init-kernel PDL threshold, independent of accumulation tier - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr const Kernel& kernel(PrivatizationMode) const - { - if constexpr (detail::histogram::is_privatized_static_smem_v) - { - return static_smem.kernel; - } - else if constexpr (detail::histogram::is_privatized_dynamic_smem_v) - { - return dynamic_smem.kernel; - } - else - { - static_assert(detail::histogram::is_privatized_gmem_v); - return gmem; - } - } - [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(const HistogramPolicy& lhs, const HistogramPolicy& rhs) noexcept { return lhs.gmem == rhs.gmem && lhs.static_smem == rhs.static_smem && lhs.dynamic_smem == rhs.dynamic_smem + && lhs.max_privatized_static_smem_bytes == rhs.max_privatized_static_smem_bytes + && lhs.static_smem_min_blocks_per_sm == rhs.static_smem_min_blocks_per_sm + && lhs.max_privatized_dynamic_smem_bytes == rhs.max_privatized_dynamic_smem_bytes + && lhs.dynamic_smem_range_max_bins == rhs.dynamic_smem_range_max_bins + && lhs.dynamic_smem_even_2ch_max_bins == rhs.dynamic_smem_even_2ch_max_bins + && lhs.dynamic_smem_even_3ch_max_bins == rhs.dynamic_smem_even_3ch_max_bins + && lhs.dynamic_smem_even_4ch_max_bins == rhs.dynamic_smem_even_4ch_max_bins && lhs.init_kernel_pdl_trigger_max_bins == rhs.init_kernel_pdl_trigger_max_bins; } @@ -118,86 +95,22 @@ struct HistogramPolicy #if _CCCL_HOSTED() friend ::std::ostream& operator<<(::std::ostream& os, const HistogramPolicy& p) { - const auto print_kernel = [&](const Kernel& kernel) -> ::std::ostream& { - return os - << "{ .threads_per_block = " << kernel.threads_per_block - << ", .items_per_thread = " << kernel.items_per_thread << ", .vec_size = " << kernel.vec_size - << ", .load_algorithm = " << kernel.load_algorithm << ", .load_modifier = " << kernel.load_modifier - << ", .rle_compress = " << kernel.rle_compress << ", .work_stealing = " << kernel.work_stealing << " }"; - }; - os << "HistogramPolicy { .gmem = "; - print_kernel(p.gmem); - os << ", .static_smem = { .kernel = "; - print_kernel(p.static_smem.kernel); - os << ", .max_privatized_smem_bytes = " << p.static_smem.max_privatized_smem_bytes - << ", .min_blocks_per_sm = " << p.static_smem.min_blocks_per_sm << " }, .dynamic_smem = { .kernel = "; - print_kernel(p.dynamic_smem.kernel); return os - << ", .max_privatized_smem_bytes = " << p.dynamic_smem.max_privatized_smem_bytes << ", .range_max_bins = " - << p.dynamic_smem.range_max_bins << ", .even_2ch_max_bins = " << p.dynamic_smem.even_2ch_max_bins - << ", .even_3ch_max_bins = " << p.dynamic_smem.even_3ch_max_bins - << ", .even_4ch_max_bins = " << p.dynamic_smem.even_4ch_max_bins - << " }, .init_kernel_pdl_trigger_max_bins = " << p.init_kernel_pdl_trigger_max_bins << " }"; + << "HistogramPolicy { .gmem = " << p.gmem << ", .static_smem = " << p.static_smem << ", .dynamic_smem = " + << p.dynamic_smem << ", .max_privatized_static_smem_bytes = " << p.max_privatized_static_smem_bytes + << ", .static_smem_min_blocks_per_sm = " << p.static_smem_min_blocks_per_sm + << ", .max_privatized_dynamic_smem_bytes = " << p.max_privatized_dynamic_smem_bytes + << ", .dynamic_smem_range_max_bins = " << p.dynamic_smem_range_max_bins + << ", .dynamic_smem_even_2ch_max_bins = " << p.dynamic_smem_even_2ch_max_bins + << ", .dynamic_smem_even_3ch_max_bins = " << p.dynamic_smem_even_3ch_max_bins + << ", .dynamic_smem_even_4ch_max_bins = " << p.dynamic_smem_even_4ch_max_bins + << ", .init_kernel_pdl_trigger_max_bins = " << p.init_kernel_pdl_trigger_max_bins << " }"; } #endif }; namespace detail::histogram { -// All DeviceHistogram tuning values live in this block. Keep architecture -// selection and policy construction below free of unexplained numeric values. -inline constexpr auto sm90 = ::cuda::compute_capability{9, 0}; -inline constexpr auto sm100 = ::cuda::compute_capability{10, 0}; -inline constexpr int supported_counter_bytes = 4; -inline constexpr int sample_u8_bytes = 1; -inline constexpr int sample_u16_bytes = 2; -inline constexpr int sample_u32_bytes = 4; -inline constexpr int sample_u64_bytes = 8; -inline constexpr int single_channel_count = 1; -inline constexpr int first_multi_channel_count = 2; -inline constexpr int two_active_channels = 2; -inline constexpr int three_active_channels = 3; -inline constexpr int four_active_channels = 4; -inline constexpr int pre_sm100_static_smem_max_bins = 256; -inline constexpr int sm100_static_smem_max_bins = 512; -inline constexpr int pdl_trigger_max_bins = 2048; -inline constexpr int sm100_opt_in_smem_bytes = 232448; -inline constexpr int sm100_non_histogram_smem_reserve = 4096; -inline constexpr int sm100_dynamic_smem_max_bytes = sm100_opt_in_smem_bytes - sm100_non_histogram_smem_reserve; -inline constexpr int sm100_range_dynamic_smem_max_bins = 2048; -inline constexpr int sm100_even_2ch_dynamic_smem_max_bins = 28544; -inline constexpr int sm100_even_3ch_dynamic_smem_max_bins = 19029; -inline constexpr int sm100_even_4ch_dynamic_smem_max_bins = 8192; -inline constexpr int legacy_privatized_smem_bins = 256; -inline constexpr int byte_sample_privatized_levels = legacy_privatized_smem_bins + 1; -inline constexpr int init_threads_per_block = 256; -inline constexpr int default_threads_per_block = 384; -inline constexpr int default_nominal_items_per_thread = 16; -inline constexpr int default_vec_size = 4; -inline constexpr int sm90_u8_threads_per_block = 768; -inline constexpr int sm90_u8_items_per_thread = 12; -inline constexpr int sm90_u16_threads_per_block = 960; -inline constexpr int sm90_u16_items_per_thread = 10; -inline constexpr int sm100_multi_threads_per_block = 1024; -inline constexpr int sm100_multi_even_nominal_items = 8; -inline constexpr int sm100_multi_range_nominal_items = 16; -inline constexpr int sm100_u8_even_threads_per_block = 928; -inline constexpr int sm100_u8_range_threads_per_block = 448; -inline constexpr int sm100_u8_items_per_thread = 12; -inline constexpr int sm100_u32_threads_per_block = 768; -inline constexpr int sm100_u32_items_per_thread = 12; -inline constexpr int sm100_u64_threads_per_block = 768; -inline constexpr int sm100_u64_items_per_thread = 6; -inline constexpr int sm100_range_static_threads_per_block = 384; -inline constexpr int sm100_range_u32_threads_per_block = 768; -inline constexpr int sm100_range_u64_nominal_items = 16; -inline constexpr int sm100_range_static_min_blocks_per_sm = 3; - -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool supports_dependent_launch(::cuda::compute_capability cc) -{ - return cc >= sm90; -} - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int max_privatized_smem_bins(int max_privatized_smem_bytes, int counter_size, int num_active_channels) { @@ -208,22 +121,12 @@ max_privatized_smem_bins(int max_privatized_smem_bytes, int counter_size, int nu return max_privatized_smem_bytes / counter_size / num_active_channels; } -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int -max_privatized_static_smem_bins(const HistogramPolicy& policy, int counter_size, int num_active_channels) -{ - return max_privatized_smem_bins(policy.static_smem.max_privatized_smem_bytes, counter_size, num_active_channels); -} - -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int -max_privatized_dynamic_smem_bins(const HistogramPolicy& policy, int counter_size, int num_active_channels) -{ - return max_privatized_smem_bins(policy.dynamic_smem.max_privatized_smem_bytes, counter_size, num_active_channels); -} - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool should_use_static_smem(const HistogramPolicy& policy, int num_bins, int counter_size, int num_active_channels) { - return num_bins > 0 && num_bins <= max_privatized_static_smem_bins(policy, counter_size, num_active_channels); + return num_bins > 0 + && num_bins + <= max_privatized_smem_bins(policy.max_privatized_static_smem_bytes, counter_size, num_active_channels); } template @@ -240,23 +143,23 @@ should_use_dynamic_smem(const HistogramPolicy& policy, int num_bins, int counter return false; } - const bool prefer_dynamic_smem = counter_size > supported_counter_bytes + const bool prefer_dynamic_smem = counter_size > int{sizeof(unsigned int)} || !should_use_static_smem(policy, num_bins, counter_size, num_active_channels); - int max_bins = max_privatized_dynamic_smem_bins(policy, counter_size, num_active_channels); - if (num_active_channels >= first_multi_channel_count) + int max_bins = max_privatized_smem_bins(policy.max_privatized_dynamic_smem_bytes, counter_size, num_active_channels); + if (num_active_channels > 1) { if constexpr (IsEven) { - max_bins = num_active_channels == two_active_channels ? policy.dynamic_smem.even_2ch_max_bins - : num_active_channels == three_active_channels ? policy.dynamic_smem.even_3ch_max_bins - : num_active_channels == four_active_channels - ? policy.dynamic_smem.even_4ch_max_bins + max_bins = num_active_channels == 2 ? policy.dynamic_smem_even_2ch_max_bins + : num_active_channels == 3 ? policy.dynamic_smem_even_3ch_max_bins + : num_active_channels == 4 + ? policy.dynamic_smem_even_4ch_max_bins : 0; } else { - max_bins = policy.dynamic_smem.range_max_bins; + max_bins = policy.dynamic_smem_range_max_bins; } } @@ -284,171 +187,123 @@ private: return (::cuda::std::max) (nominal_items_per_thread / num_active_channels / sample_scale, 1); } - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_sm50_tuning() const -> HistogramPolicy - { - const auto kernel = HistogramPolicy::Kernel{ - default_threads_per_block, - t_scale(default_nominal_items_per_thread), - default_vec_size, - BLOCK_LOAD_DIRECT, - LOAD_LDG, - true, - false}; - return {kernel, - {kernel, pre_sm100_static_smem_max_bins * counter_size_bytes * num_active_channels, 0}, - {kernel, 0, 0, 0, 0, 0}, - 0}; - } - - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_sm90_tuning() const -> HistogramPolicy +public: + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy { - auto result = get_sm50_tuning(); - if (num_channels == single_channel_count && num_active_channels == single_channel_count - && counter_size_bytes == supported_counter_bytes && sample_is_primitive) + if (cc >= ::cuda::compute_capability{10, 0}) { - if (sample_size_bytes == sample_u8_bytes) + const bool single_channel = num_channels == 1 && num_active_channels == 1; + auto gmem = HistogramSweepPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + + if (num_channels > 1 && counter_size_bytes == int{sizeof(unsigned int)} && sample_is_primitive) { - result.gmem = { - sm90_u8_threads_per_block, - sm90_u8_items_per_thread, - default_vec_size, - BLOCK_LOAD_DIRECT, - LOAD_LDG, - false, - false}; + gmem = HistogramSweepPolicy{1024, t_scale(is_even ? 8 : 16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; } - else if (sample_size_bytes == sample_u16_bytes) + else if (single_channel && counter_size_bytes == int{sizeof(unsigned int)} && sample_is_primitive) { - result.gmem = { - sm90_u16_threads_per_block, - sm90_u16_items_per_thread, - default_vec_size, - BLOCK_LOAD_DIRECT, - LOAD_DEFAULT, - true, - false}; + if (sample_size_bytes == 1) + { + gmem = is_even ? HistogramSweepPolicy{928, 12, 4, BLOCK_LOAD_DIRECT, LOAD_CA, false, false} + : HistogramSweepPolicy{448, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, false, false}; + } + else if (sample_size_bytes == 2) + { + // Retain the SM90 U16 sweep shape on SM100. + gmem = HistogramSweepPolicy{960, 10, 4, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, false}; + } + else if (sample_size_bytes == 4) + { + gmem = HistogramSweepPolicy{768, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + } + else if (sample_size_bytes == 8) + { + gmem = HistogramSweepPolicy{768, 6, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + } } - result.static_smem.kernel = result.gmem; - result.dynamic_smem.kernel = result.gmem; - result.init_kernel_pdl_trigger_max_bins = - sample_size_bytes == sample_u8_bytes || sample_size_bytes == sample_u16_bytes ? pdl_trigger_max_bins : 0; - } - return result; - } - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto get_sm100_tuning() const -> HistogramPolicy - { - auto result = get_sm90_tuning(); - const bool single_channel = num_channels == single_channel_count && num_active_channels == single_channel_count; - if (num_channels >= first_multi_channel_count && counter_size_bytes == supported_counter_bytes - && sample_is_primitive) - { - result.gmem = { - sm100_multi_threads_per_block, - t_scale(is_even ? sm100_multi_even_nominal_items : sm100_multi_range_nominal_items), - default_vec_size, - BLOCK_LOAD_DIRECT, - LOAD_LDG, - true, - false}; - } - else if (single_channel && counter_size_bytes == supported_counter_bytes && sample_is_primitive) - { - if (sample_size_bytes == sample_u8_bytes) + auto static_smem = gmem; + const bool range_multi_static = + !is_even && num_channels > 1 && counter_size_bytes == int{sizeof(unsigned int)} && sample_is_primitive; + const bool range_u32_static = !is_even && single_channel && counter_size_bytes == int{sizeof(unsigned int)} + && sample_is_primitive && sample_size_bytes == 4; + const bool range_u64_static = !is_even && single_channel && counter_size_bytes == int{sizeof(unsigned int)} + && sample_is_primitive && sample_size_bytes == 8; + if (range_multi_static || range_u64_static) { - result.gmem = - is_even - ? HistogramPolicy::Kernel{sm100_u8_even_threads_per_block, - sm100_u8_items_per_thread, - default_vec_size, - BLOCK_LOAD_DIRECT, - LOAD_CA, - false, - false} - : HistogramPolicy::Kernel{ - sm100_u8_range_threads_per_block, - sm100_u8_items_per_thread, - default_vec_size, - BLOCK_LOAD_DIRECT, - LOAD_LDG, - false, - false}; + static_smem.threads_per_block = 384; } - else if (sample_size_bytes == sample_u32_bytes) + else if (range_u32_static) { - result.gmem = { - sm100_u32_threads_per_block, - sm100_u32_items_per_thread, - default_vec_size, - BLOCK_LOAD_DIRECT, - LOAD_LDG, - true, - false}; + static_smem.threads_per_block = 768; } - else if (sample_size_bytes == sample_u64_bytes) + if (range_u64_static) { - result.gmem = { - sm100_u64_threads_per_block, - sm100_u64_items_per_thread, - default_vec_size, - BLOCK_LOAD_DIRECT, - LOAD_LDG, - true, - false}; + static_smem.items_per_thread = t_scale(16); } - } - result.static_smem.kernel = result.gmem; - result.dynamic_smem.kernel = result.gmem; - const bool range_multi_static = !is_even && num_channels >= first_multi_channel_count - && counter_size_bytes == supported_counter_bytes && sample_is_primitive; - const bool range_u32_static = !is_even && single_channel && counter_size_bytes == supported_counter_bytes - && sample_is_primitive && sample_size_bytes == sample_u32_bytes; - const bool range_u64_static = !is_even && single_channel && counter_size_bytes == supported_counter_bytes - && sample_is_primitive && sample_size_bytes == sample_u64_bytes; - if (range_multi_static || range_u64_static) - { - result.static_smem.kernel.threads_per_block = sm100_range_static_threads_per_block; - } - else if (range_u32_static) - { - result.static_smem.kernel.threads_per_block = sm100_range_u32_threads_per_block; + const bool has_dynamic_smem_tuning = + counter_size_bytes == int{sizeof(unsigned int)} && sample_is_primitive + && ((single_channel && (sample_size_bytes == 1 || sample_size_bytes == 4 || sample_size_bytes == 8)) + || num_channels > 1); + // B200 provides 232448 bytes of opt-in shared memory. Reserve 4096 bytes + // for the kernel's statically allocated shared-memory state. + const int max_privatized_dynamic_smem_bytes = has_dynamic_smem_tuning ? 232448 - 4096 : 0; + const int init_kernel_pdl_trigger_max_bins = + single_channel && counter_size_bytes == int{sizeof(unsigned int)} && sample_is_primitive + && (sample_size_bytes == 1 || sample_size_bytes == 2 || sample_size_bytes == 4 || sample_size_bytes == 8) + ? 2048 + : 0; + + return HistogramPolicy{ + gmem, + static_smem, + gmem, + 512 * counter_size_bytes * num_active_channels, + range_multi_static || range_u64_static ? 3 : 0, + max_privatized_dynamic_smem_bytes, + has_dynamic_smem_tuning ? 2048 : 0, + has_dynamic_smem_tuning ? 28544 : 0, + has_dynamic_smem_tuning ? 19029 : 0, + has_dynamic_smem_tuning ? 8192 : 0, + init_kernel_pdl_trigger_max_bins}; } - if (range_u64_static) + + if (cc >= ::cuda::compute_capability{9, 0}) { - result.static_smem.kernel.items_per_thread = t_scale(sm100_range_u64_nominal_items); + auto sweep = HistogramSweepPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + if (num_channels == 1 && num_active_channels == 1 && counter_size_bytes == int{sizeof(unsigned int)} + && sample_is_primitive) + { + if (sample_size_bytes == 1) + { + sweep = HistogramSweepPolicy{768, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, false, false}; + } + else if (sample_size_bytes == 2) + { + sweep = HistogramSweepPolicy{960, 10, 4, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, false}; + } + } + const int init_kernel_pdl_trigger_max_bins = + num_channels == 1 && num_active_channels == 1 && counter_size_bytes == int{sizeof(unsigned int)} + && sample_is_primitive && (sample_size_bytes == 1 || sample_size_bytes == 2) + ? 2048 + : 0; + return HistogramPolicy{ + sweep, + sweep, + sweep, + 256 * counter_size_bytes * num_active_channels, + 0, + 0, + 0, + 0, + 0, + 0, + init_kernel_pdl_trigger_max_bins}; } - result.static_smem.max_privatized_smem_bytes = - sm100_static_smem_max_bins * counter_size_bytes * num_active_channels; - result.static_smem.min_blocks_per_sm = - range_multi_static || range_u64_static ? sm100_range_static_min_blocks_per_sm : 0; - - const bool has_dynamic_smem_tuning = - counter_size_bytes == supported_counter_bytes && sample_is_primitive - && ((single_channel - && (sample_size_bytes == sample_u8_bytes || sample_size_bytes == sample_u32_bytes - || sample_size_bytes == sample_u64_bytes)) - || num_channels >= first_multi_channel_count); - result.dynamic_smem = { - result.gmem, - has_dynamic_smem_tuning ? sm100_dynamic_smem_max_bytes : 0, - has_dynamic_smem_tuning ? sm100_range_dynamic_smem_max_bins : 0, - has_dynamic_smem_tuning ? sm100_even_2ch_dynamic_smem_max_bins : 0, - has_dynamic_smem_tuning ? sm100_even_3ch_dynamic_smem_max_bins : 0, - has_dynamic_smem_tuning ? sm100_even_4ch_dynamic_smem_max_bins : 0}; - result.init_kernel_pdl_trigger_max_bins = - single_channel && counter_size_bytes == supported_counter_bytes && sample_is_primitive - && (sample_size_bytes == sample_u8_bytes || sample_size_bytes == sample_u16_bytes - || sample_size_bytes == sample_u32_bytes || sample_size_bytes == sample_u64_bytes) - ? pdl_trigger_max_bins - : 0; - return result; - } -public: - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy - { - return cc >= sm100 ? get_sm100_tuning() : cc >= sm90 ? get_sm90_tuning() : get_sm50_tuning(); + const auto sweep = HistogramSweepPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + return HistogramPolicy{sweep, sweep, sweep, 256 * counter_size_bytes * num_active_channels, 0, 0, 0, 0, 0, 0, 0}; } }; diff --git a/cub/test/catch2_test_device_histogram_custom_policy_hub.cu b/cub/test/catch2_test_device_histogram_custom_policy_hub.cu index 6b9f41c218e3..6773673b20b5 100644 --- a/cub/test/catch2_test_device_histogram_custom_policy_hub.cu +++ b/cub/test/catch2_test_device_histogram_custom_policy_hub.cu @@ -46,8 +46,8 @@ CUB_TEST("DispatchHistogram::DispatchEven: custom policy hub", "[histogram][devi REQUIRE(custom_sm75_policy.gmem.items_per_thread == 16); REQUIRE(custom_sm90_policy.gmem.threads_per_block == 256); REQUIRE(custom_sm90_policy.gmem.items_per_thread == 8); - REQUIRE(custom_sm75_policy.static_smem.max_privatized_smem_bytes == 256 * sizeof(unsigned int)); - REQUIRE(custom_sm90_policy.dynamic_smem.max_privatized_smem_bytes == 0); + REQUIRE(custom_sm75_policy.max_privatized_static_smem_bytes == 256 * sizeof(unsigned int)); + REQUIRE(custom_sm90_policy.max_privatized_dynamic_smem_bytes == 0); REQUIRE(custom_sm75_policy.init_kernel_pdl_trigger_max_bins == 0); REQUIRE(custom_sm90_policy.init_kernel_pdl_trigger_max_bins == 2048); diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index 3784a634fa9b..0e07f2f3d763 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1627,8 +1627,8 @@ struct histogram_tuning _CCCL_HOST_DEVICE_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { constexpr auto sweep = - cub::HistogramPolicy::Kernel{BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; - return {sweep, {sweep, 256 * sizeof(unsigned int), 0}, {sweep, 0, 0, 0, 0, 0}, 0}; + cub::HistogramSweepPolicy{BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; + return {sweep, sweep, sweep, 256 * sizeof(unsigned int), 0, 0, 0, 0, 0, 0, 0}; } }; @@ -1645,8 +1645,8 @@ struct mixed_counter_histogram_tuning _CCCL_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { constexpr auto sweep = - cub::HistogramPolicy::Kernel{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; - return {sweep, {sweep, 512 * sizeof(unsigned int), 0}, {sweep, 228352, 2048, 28544, 19029, 8192}, 0}; + cub::HistogramSweepPolicy{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; + return {sweep, sweep, sweep, 512 * sizeof(unsigned int), 0, 228352, 2048, 28544, 19029, 8192, 0}; } }; @@ -1657,7 +1657,7 @@ static_assert( static_assert(cuda::std::is_same_v, unsigned long long>, unsigned long long>); -C2H_TEST("DeviceHistogram supports narrower local counters than output counters", "[histogram][device]") +CUB_TEST("DeviceHistogram supports narrower local counters than output counters", "[histogram][device]", CUB_SMALL) { int current_device{}; REQUIRE(cudaSuccess == cudaGetDevice(¤t_device)); @@ -1807,43 +1807,49 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) // aggregate init constexpr auto p1 = cub::HistogramPolicy{ {128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, - {{96, 3, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, 2052, 2}, - {{128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, 12345, 1024, 4096, 8192, 16384}, + {96, 3, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, + {128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, + 2052, + 2, + 12345, + 1024, + 4096, + 8192, + 16384, 2048}; # if _CCCL_STD_VER >= 2020 // designated init constexpr auto p2 = cub::HistogramPolicy{ - .gmem = {.threads_per_block = 128, - .items_per_thread = 7, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .static_smem = {.kernel = {.threads_per_block = 96, - .items_per_thread = 3, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .max_privatized_smem_bytes = 2052, - .min_blocks_per_sm = 2}, - .dynamic_smem = - {.kernel = {.threads_per_block = 128, - .items_per_thread = 7, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .max_privatized_smem_bytes = 12345, - .range_max_bins = 1024, - .even_2ch_max_bins = 4096, - .even_3ch_max_bins = 8192, - .even_4ch_max_bins = 16384}, - .init_kernel_pdl_trigger_max_bins = 2048}; + .gmem = {.threads_per_block = 128, + .items_per_thread = 7, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .static_smem = {.threads_per_block = 96, + .items_per_thread = 3, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .dynamic_smem = {.threads_per_block = 128, + .items_per_thread = 7, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .max_privatized_static_smem_bytes = 2052, + .static_smem_min_blocks_per_sm = 2, + .max_privatized_dynamic_smem_bytes = 12345, + .dynamic_smem_range_max_bins = 1024, + .dynamic_smem_even_2ch_max_bins = 4096, + .dynamic_smem_even_3ch_max_bins = 8192, + .dynamic_smem_even_4ch_max_bins = 16384, + .init_kernel_pdl_trigger_max_bins = 2048}; # else // _CCCL_STD_VER >= 2020 constexpr auto p2 = p1; # endif // _CCCL_STD_VER >= 2020 @@ -1857,24 +1863,10 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) os << p; return os.str(); }; - REQUIRE( - to_string(p1) - == "HistogramPolicy { .gmem = { .threads_per_block = 128, .items_per_thread = 7, .vec_size = 4" - ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" - ", .work_stealing = 0 }" - ", .static_smem = { .kernel = { .threads_per_block = 96, .items_per_thread = 3, .vec_size = 4" - ", .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG, .rle_compress = 0" - ", .work_stealing = 0 }" - ", .max_privatized_smem_bytes = 2052, .min_blocks_per_sm = 2 }, .dynamic_smem = { .kernel = { " - ".threads_per_block " - "= 128" - ", .items_per_thread = 7, .vec_size = 4, .load_algorithm = BLOCK_LOAD_DIRECT, .load_modifier = LOAD_LDG" - ", .rle_compress = 0, .work_stealing = 0 }, .max_privatized_smem_bytes = 12345" - ", .range_max_bins = 1024, .even_2ch_max_bins = 4096, .even_3ch_max_bins = 8192" - ", .even_4ch_max_bins = 16384 }, .init_kernel_pdl_trigger_max_bins = 2048 }"); + REQUIRE(to_string(p1) == to_string(p2)); } -C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget", "[histogram][device]") +CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget", "[histogram][device]", CUB_SMALL) { using selector_t = cub::detail::histogram::policy_selector_from_types; @@ -1884,39 +1876,44 @@ C2H_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); - STATIC_REQUIRE(cub::detail::histogram::max_privatized_static_smem_bins(sm90_policy, 4, 1) == 256); - STATIC_REQUIRE(cub::detail::histogram::max_privatized_static_smem_bins(sm100_policy, 4, 1) == 512); - STATIC_REQUIRE(cub::detail::histogram::max_privatized_static_smem_bins(sm100_policy, 4, 4) == 128); - STATIC_REQUIRE(sm90_policy.dynamic_smem.max_privatized_smem_bytes == 0); - STATIC_REQUIRE(sm100_policy.dynamic_smem.max_privatized_smem_bytes == 228352); - STATIC_REQUIRE(sm100_wide_counter_policy.dynamic_smem.max_privatized_smem_bytes == 0); - STATIC_REQUIRE(cub::detail::histogram::max_privatized_dynamic_smem_bins(sm100_policy, 4, 1) == 57088); - STATIC_REQUIRE(cub::detail::histogram::max_privatized_dynamic_smem_bins(sm100_policy, 4, 4) == 14272); - STATIC_REQUIRE(sm100_policy.dynamic_smem.range_max_bins == 2048); - STATIC_REQUIRE(sm100_policy.dynamic_smem.even_2ch_max_bins == 28544); - STATIC_REQUIRE(sm100_policy.dynamic_smem.even_3ch_max_bins == 19029); - STATIC_REQUIRE(sm100_policy.dynamic_smem.even_4ch_max_bins == 8192); + STATIC_REQUIRE( + cub::detail::histogram::max_privatized_smem_bins(sm90_policy.max_privatized_static_smem_bytes, 4, 1) == 256); + STATIC_REQUIRE( + cub::detail::histogram::max_privatized_smem_bins(sm100_policy.max_privatized_static_smem_bytes, 4, 1) == 512); + STATIC_REQUIRE( + cub::detail::histogram::max_privatized_smem_bins(sm100_policy.max_privatized_static_smem_bytes, 4, 4) == 128); + STATIC_REQUIRE(sm90_policy.max_privatized_dynamic_smem_bytes == 0); + STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_bytes == 228352); + STATIC_REQUIRE(sm100_wide_counter_policy.max_privatized_dynamic_smem_bytes == 0); + STATIC_REQUIRE( + cub::detail::histogram::max_privatized_smem_bins(sm100_policy.max_privatized_dynamic_smem_bytes, 4, 1) == 57088); + STATIC_REQUIRE( + cub::detail::histogram::max_privatized_smem_bins(sm100_policy.max_privatized_dynamic_smem_bytes, 4, 4) == 14272); + STATIC_REQUIRE(sm100_policy.dynamic_smem_range_max_bins == 2048); + STATIC_REQUIRE(sm100_policy.dynamic_smem_even_2ch_max_bins == 28544); + STATIC_REQUIRE(sm100_policy.dynamic_smem_even_3ch_max_bins == 19029); + STATIC_REQUIRE(sm100_policy.dynamic_smem_even_4ch_max_bins == 8192); STATIC_REQUIRE(sm100_policy.gmem.threads_per_block == 768); STATIC_REQUIRE(sm100_policy.gmem.items_per_thread == 12); - STATIC_REQUIRE(sm100_policy.static_smem.kernel == sm100_policy.gmem); + STATIC_REQUIRE(sm100_policy.static_smem == sm100_policy.gmem); constexpr auto sm100_range_u64_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); STATIC_REQUIRE(sm100_range_u64_policy.gmem.threads_per_block == 768); STATIC_REQUIRE(sm100_range_u64_policy.gmem.items_per_thread == 6); - STATIC_REQUIRE(sm100_range_u64_policy.static_smem.kernel.threads_per_block == 384); - STATIC_REQUIRE(sm100_range_u64_policy.static_smem.kernel.items_per_thread == 8); - STATIC_REQUIRE(sm100_range_u64_policy.static_smem.min_blocks_per_sm == 3); + STATIC_REQUIRE(sm100_range_u64_policy.static_smem.threads_per_block == 384); + STATIC_REQUIRE(sm100_range_u64_policy.static_smem.items_per_thread == 8); + STATIC_REQUIRE(sm100_range_u64_policy.static_smem_min_blocks_per_sm == 3); constexpr auto sm100_multi_range_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); STATIC_REQUIRE(sm100_multi_range_policy.gmem.threads_per_block == 1024); STATIC_REQUIRE(sm100_multi_range_policy.gmem.items_per_thread == 5); - STATIC_REQUIRE(sm100_multi_range_policy.static_smem.kernel.threads_per_block == 384); - STATIC_REQUIRE(sm100_multi_range_policy.static_smem.kernel.items_per_thread == 5); - STATIC_REQUIRE(sm100_multi_range_policy.static_smem.min_blocks_per_sm == 3); + STATIC_REQUIRE(sm100_multi_range_policy.static_smem.threads_per_block == 384); + STATIC_REQUIRE(sm100_multi_range_policy.static_smem.items_per_thread == 5); + STATIC_REQUIRE(sm100_multi_range_policy.static_smem_min_blocks_per_sm == 3); STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57088, 4, 1)); STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57089, 4, 1)); diff --git a/cub/test/catch2_test_device_histogram_env_api.cu b/cub/test/catch2_test_device_histogram_env_api.cu index 8bdbfd7980cf..0d574db4ad6c 100644 --- a/cub/test/catch2_test_device_histogram_env_api.cu +++ b/cub/test/catch2_test_device_histogram_env_api.cu @@ -418,18 +418,20 @@ struct HistogramPolicySelector { __host__ __device__ constexpr auto operator()(cuda::compute_capability cc) const -> cub::HistogramPolicy { - const auto sweep = cub::HistogramPolicy::Kernel{ + const auto sweep = cub::HistogramSweepPolicy{ 128, cc > cuda::compute_capability{9, 0} ? 16 : 7, 4, cub::BLOCK_LOAD_DIRECT, cub::LOAD_LDG, false, false}; return { - .gmem = sweep, - .static_smem = {.kernel = sweep, .max_privatized_smem_bytes = 256 * sizeof(unsigned int), .min_blocks_per_sm = 0}, - .dynamic_smem = {.kernel = sweep, - .max_privatized_smem_bytes = 0, - .range_max_bins = 0, - .even_2ch_max_bins = 0, - .even_3ch_max_bins = 0, - .even_4ch_max_bins = 0}, - .init_kernel_pdl_trigger_max_bins = 2048}; + .gmem = sweep, + .static_smem = sweep, + .dynamic_smem = sweep, + .max_privatized_static_smem_bytes = 256 * sizeof(unsigned int), + .static_smem_min_blocks_per_sm = 0, + .max_privatized_dynamic_smem_bytes = 0, + .dynamic_smem_range_max_bins = 0, + .dynamic_smem_even_2ch_max_bins = 0, + .dynamic_smem_even_3ch_max_bins = 0, + .dynamic_smem_even_4ch_max_bins = 0, + .init_kernel_pdl_trigger_max_bins = 2048}; } }; // example-end histogram-even-policy-selector From 4aa7bad953a8fdb403e4106ceaa6ae091a1a5edf Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sat, 8 Aug 2026 01:34:05 +0000 Subject: [PATCH 34/45] [cub] Address histogram policy review --- c/parallel/src/histogram.cu | 24 +- cub/benchmarks/bench/histogram/even.cu | 10 +- .../bench/histogram/histogram_common.cuh | 25 +- cub/benchmarks/bench/histogram/multi/even.cu | 10 +- cub/benchmarks/bench/histogram/multi/range.cu | 10 +- cub/benchmarks/bench/histogram/range.cu | 10 +- cub/cub/agent/agent_histogram.cuh | 16 +- .../device/dispatch/dispatch_histogram.cuh | 42 ++- .../dispatch/kernels/kernel_histogram.cuh | 14 +- .../dispatch/tuning/tuning_histogram.cuh | 282 +++++++++++------- ...test_device_histogram_custom_policy_hub.cu | 8 +- cub/test/catch2_test_device_histogram_env.cu | 162 ++++++---- .../catch2_test_device_histogram_env_api.cu | 24 +- 13 files changed, 357 insertions(+), 280 deletions(-) diff --git a/c/parallel/src/histogram.cu b/c/parallel/src/histogram.cu index d01e7cc63119..c4c2fdacf4d2 100644 --- a/c/parallel/src/histogram.cu +++ b/c/parallel/src/histogram.cu @@ -56,7 +56,7 @@ struct histogram_kernel_source } template ", chained_policy_t, - privatized_smem_bins, + privatization_mode_t, num_channels, num_active_channels, samples_iterator_t, @@ -324,17 +324,20 @@ static_assert(device_histogram_policy()(detail::current_tuning_cc()) == {4}, "Ho fflush(stdout); #endif - // TODO: This is tricky because we need to know the input to set this to a - // value greater than 0 (see dispatch_histogram.cuh), but we don't have this - // information here. - const int privatized_smem_bins = - num_output_levels_val - 1 > cub::detail::histogram::max_privatized_smem_bins ? 0 : 256; - const bool is_byte_sample = d_samples.value_type.size == 1; + const auto privatization = + is_byte_sample + ? cub::detail::histogram::privatization_mode::static_smem + : cub::detail::histogram::select_privatization_mode_for_counter_size( + active_policy, num_output_levels_val - 1, static_cast(d_output_histograms.value_type.size)); + const std::string_view privatization_mode_t = + privatization == cub::detail::histogram::privatization_mode::static_smem + ? "cub::detail::histogram::HistogramPrivatizedStaticSmem" + : "cub::detail::histogram::HistogramPrivatizedGmem"; std::string init_kernel_name = histogram::get_init_kernel_name(num_active_channels, counter_cpp, offset_cpp); std::string sweep_kernel_name = histogram::get_sweep_kernel_name( - privatized_smem_bins, + privatization_mode_t, num_channels, num_active_channels, d_samples, @@ -574,6 +577,7 @@ CUresult cccl_device_histogram_even_impl( indirect_arg_t, // LevelT OffsetT, // OffsetT cub::detail::histogram::policy_selector, // PolicySelector + void, // PrivatizedCounterT: C Parallel preserves the counter width after type erasure indirect_arg_t, // SampleT histogram::histogram_kernel_source, // KernelSource cub::detail::CudaDriverLauncherFactory // KernelLauncherFactory diff --git a/cub/benchmarks/bench/histogram/even.cu b/cub/benchmarks/bench/histogram/even.cu index a45c3c1aae47..79abc98cebf4 100644 --- a/cub/benchmarks/bench/histogram/even.cu +++ b/cub/benchmarks/bench/histogram/even.cu @@ -12,14 +12,6 @@ // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 -// %RANGE% TUNE_STATIC_SMEM_MAX_BYTES smem_bytes 0:512:256 -// %RANGE% TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM smem_blocks 0:4:1 -// %RANGE% TUNE_DYNAMIC_SMEM_MAX_BYTES dyn_bytes 0:228352:228352 -// %RANGE% TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS dyn_range_bins 0:2048:2048 -// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS dyn_even_2ch_bins 0:28544:28544 -// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_3CH_MAX_BINS dyn_even_3ch_bins 0:19029:19029 -// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_4CH_MAX_BINS dyn_even_4ch_bins 0:8192:8192 -// %RANGE% TUNE_INIT_KERNEL_PDL_TRIGGER_MAX_BINS pdl_bins 0:2048:2048 template static void even(nvbench::state& state, nvbench::type_list) @@ -59,7 +51,7 @@ static void even(nvbench::state& state, nvbench::type_list{}) + cuda::execution::tune(bench_policy_selector{}) #endif // !TUNE_BASE ); _CCCL_TRY_CUDA_API( diff --git a/cub/benchmarks/bench/histogram/histogram_common.cuh b/cub/benchmarks/bench/histogram/histogram_common.cuh index d59b870070c0..f8d735eb672a 100644 --- a/cub/benchmarks/bench/histogram/histogram_common.cuh +++ b/cub/benchmarks/bench/histogram/histogram_common.cuh @@ -27,17 +27,17 @@ # define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_STRIPED # endif // TUNE_LOAD_ALGORITHM_ID -template +template struct bench_policy_selector { - _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability) const -> cub::HistogramPolicy + _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> cub::HistogramPolicy { constexpr cub::BlockLoadAlgorithm load_algorithm = (TUNE_LOAD_ALGORITHM == cub::BLOCK_LOAD_STRIPED) ? (NUM_CHANNELS == 1 ? cub::BLOCK_LOAD_STRIPED : cub::BLOCK_LOAD_DIRECT) : TUNE_LOAD_ALGORITHM; - constexpr auto sweep = cub::HistogramSweepPolicy{ + constexpr auto sweep = cub::HistogramPrivatizationPolicy{ TUNE_THREADS, TUNE_ITEMS, TUNE_VEC_SIZE, @@ -45,18 +45,13 @@ struct bench_policy_selector TUNE_LOAD_MODIFIER, TUNE_RLE_COMPRESS, TUNE_WORK_STEALING}; - return { - sweep, - sweep, - sweep, - TUNE_STATIC_SMEM_MAX_BYTES, - TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM, - TUNE_DYNAMIC_SMEM_MAX_BYTES, - TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS, - TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS, - TUNE_DYNAMIC_SMEM_EVEN_3CH_MAX_BINS, - TUNE_DYNAMIC_SMEM_EVEN_4CH_MAX_BINS, - TUNE_INIT_KERNEL_PDL_TRIGGER_MAX_BINS}; + auto policy = + cub::detail::histogram::policy_selector_from_types{}( + cc); + policy.gmem = sweep; + policy.static_smem = sweep; + policy.dynamic_smem = sweep; + return policy; } }; #endif // !TUNE_BASE diff --git a/cub/benchmarks/bench/histogram/multi/even.cu b/cub/benchmarks/bench/histogram/multi/even.cu index 22df064790f3..2d5df84932ac 100644 --- a/cub/benchmarks/bench/histogram/multi/even.cu +++ b/cub/benchmarks/bench/histogram/multi/even.cu @@ -12,14 +12,6 @@ // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 -// %RANGE% TUNE_STATIC_SMEM_MAX_BYTES smem_bytes 0:512:256 -// %RANGE% TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM smem_blocks 0:4:1 -// %RANGE% TUNE_DYNAMIC_SMEM_MAX_BYTES dyn_bytes 0:228352:228352 -// %RANGE% TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS dyn_range_bins 0:2048:2048 -// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS dyn_even_2ch_bins 0:28544:28544 -// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_3CH_MAX_BINS dyn_even_3ch_bins 0:19029:19029 -// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_4CH_MAX_BINS dyn_even_4ch_bins 0:8192:8192 -// %RANGE% TUNE_INIT_KERNEL_PDL_TRIGGER_MAX_BINS pdl_bins 0:2048:2048 template static void even(nvbench::state& state, nvbench::type_list) @@ -72,7 +64,7 @@ static void even(nvbench::state& state, nvbench::type_list{}) + cuda::execution::tune(bench_policy_selector{}) #endif // !TUNE_BASE ); _CCCL_TRY_CUDA_API( diff --git a/cub/benchmarks/bench/histogram/multi/range.cu b/cub/benchmarks/bench/histogram/multi/range.cu index f38f44cb6c94..9c16e2d676ee 100644 --- a/cub/benchmarks/bench/histogram/multi/range.cu +++ b/cub/benchmarks/bench/histogram/multi/range.cu @@ -14,14 +14,6 @@ // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 -// %RANGE% TUNE_STATIC_SMEM_MAX_BYTES smem_bytes 0:512:256 -// %RANGE% TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM smem_blocks 0:4:1 -// %RANGE% TUNE_DYNAMIC_SMEM_MAX_BYTES dyn_bytes 0:228352:228352 -// %RANGE% TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS dyn_range_bins 0:2048:2048 -// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS dyn_even_2ch_bins 0:28544:28544 -// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_3CH_MAX_BINS dyn_even_3ch_bins 0:19029:19029 -// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_4CH_MAX_BINS dyn_even_4ch_bins 0:8192:8192 -// %RANGE% TUNE_INIT_KERNEL_PDL_TRIGGER_MAX_BINS pdl_bins 0:2048:2048 template static void range(nvbench::state& state, nvbench::type_list) @@ -72,7 +64,7 @@ static void range(nvbench::state& state, nvbench::type_list{}) + cuda::execution::tune(bench_policy_selector{}) #endif // !TUNE_BASE ); _CCCL_TRY_CUDA_API( diff --git a/cub/benchmarks/bench/histogram/range.cu b/cub/benchmarks/bench/histogram/range.cu index 674a7871862c..cd9cbb4f8e3d 100644 --- a/cub/benchmarks/bench/histogram/range.cu +++ b/cub/benchmarks/bench/histogram/range.cu @@ -14,14 +14,6 @@ // %RANGE% TUNE_LOAD ld 0:2:1 // %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1 // %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1 -// %RANGE% TUNE_STATIC_SMEM_MAX_BYTES smem_bytes 0:512:256 -// %RANGE% TUNE_STATIC_SMEM_MIN_BLOCKS_PER_SM smem_blocks 0:4:1 -// %RANGE% TUNE_DYNAMIC_SMEM_MAX_BYTES dyn_bytes 0:228352:228352 -// %RANGE% TUNE_DYNAMIC_SMEM_RANGE_MAX_BINS dyn_range_bins 0:2048:2048 -// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_2CH_MAX_BINS dyn_even_2ch_bins 0:28544:28544 -// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_3CH_MAX_BINS dyn_even_3ch_bins 0:19029:19029 -// %RANGE% TUNE_DYNAMIC_SMEM_EVEN_4CH_MAX_BINS dyn_even_4ch_bins 0:8192:8192 -// %RANGE% TUNE_INIT_KERNEL_PDL_TRIGGER_MAX_BINS pdl_bins 0:2048:2048 template static void range(nvbench::state& state, nvbench::type_list) @@ -58,7 +50,7 @@ static void range(nvbench::state& state, nvbench::type_list{}) + cuda::execution::tune(bench_policy_selector{}) #endif // !TUNE_BASE ); _CCCL_TRY_CUDA_API( diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index 8a5726789abe..7f5d2dbf463b 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -163,7 +163,7 @@ struct AgentHistogram ? policy.dynamic_smem : policy.gmem; static constexpr int privatized_static_smem_bins = - uses_static_smem ? policy.max_privatized_static_smem_bytes / int{sizeof(CounterT)} / NumActiveChannels : 0; + uses_static_smem ? policy.max_privatized_static_smem_single_channel_bytes / int{sizeof(CounterT)} : 0; static_assert(!uses_static_smem || privatized_static_smem_bins > 0, "Static-SMEM privatization requires room for at least one bin"); static constexpr int vec_size = sweep.vec_size; @@ -219,7 +219,7 @@ struct AgentHistogram const int* num_output_bins; // one for each channel const int* num_privatized_bins; // one for each channel CounterT* gmem_privatized_histograms[NumActiveChannels]; // one for each channel - CounterT* dyn_smem_privatized_histograms[NumActiveChannels]; // dynamic shared-memory channel bases, when enabled + CounterT* dynamic_smem_privatized_histograms[NumActiveChannels]; // dynamic shared-memory channel bases, when enabled OutputCounterT** output_histogram; // final output, in global memory const OutputDecodeOpT* output_decode_op; // determines output bin-id from privatized counter index, one for each // channel @@ -228,7 +228,7 @@ struct AgentHistogram { if constexpr (uses_dynamic_smem) { - return dyn_smem_privatized_histograms[channel]; + return dynamic_smem_privatized_histograms[channel]; } else if constexpr (uses_static_smem) { @@ -548,7 +548,7 @@ struct AgentHistogram //! @param privatized_decode_op //! The transform operator for determining privatized counter indices from samples, one for each channel //! - //! @param dyn_smem_privatized_histograms + //! @param dynamic_smem_privatized_histograms //! Base of the runtime-sized shared-memory histogram, or `nullptr` for a static-SMEM or global-memory mode _CCCL_DEVICE _CCCL_FORCEINLINE AgentHistogram( TempStorage& static_smem_storage, @@ -559,7 +559,7 @@ struct AgentHistogram CounterT** gmem_privatized_histograms, const OutputDecodeOpT* output_decode_op, PrivatizedDecodeOpT* privatized_decode_op, - CounterT* dyn_smem_privatized_histograms) + CounterT* dynamic_smem_privatized_histograms) : static_smem_storage(static_smem_storage.Alias()) , d_wrapped_samples(d_samples) , d_native_samples(NativePointer(d_wrapped_samples)) @@ -571,12 +571,12 @@ struct AgentHistogram { if constexpr (uses_dynamic_smem) { - _CCCL_ASSERT(dyn_smem_privatized_histograms != nullptr, "Dynamic-SMEM mode requires a shared-memory base"); - CounterT* channel_histogram = dyn_smem_privatized_histograms; + _CCCL_ASSERT(dynamic_smem_privatized_histograms != nullptr, "Dynamic-SMEM mode requires a shared-memory base"); + CounterT* channel_histogram = dynamic_smem_privatized_histograms; _CCCL_PRAGMA_UNROLL_FULL() for (int ch = 0; ch < NumActiveChannels; ++ch) { - this->dyn_smem_privatized_histograms[ch] = channel_histogram; + this->dynamic_smem_privatized_histograms[ch] = channel_histogram; channel_histogram += num_privatized_bins[ch]; } } diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 646eac045166..4e68cc411064 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -279,7 +279,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( } }(); - const HistogramSweepPolicy sweep = + const HistogramPrivatizationPolicy sweep = is_privatized_static_smem_v ? active_policy.static_smem : is_privatized_dynamic_smem_v ? active_policy.dynamic_smem @@ -296,7 +296,7 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( } NV_IF_TARGET(NV_IS_HOST, ({ if (const auto error = CubDebug(launcher_factory.set_max_dynamic_smem_size_for( - sweep_kernel, active_policy.max_privatized_dynamic_smem_bytes))) + sweep_kernel, dynamic_smem_limit_bytes(active_policy)))) { return error; } @@ -540,7 +540,8 @@ template < typename LevelT, typename OffsetT, typename PolicySelector, - typename SampleT = it_value_t, /// The sample value type of the input iterator + typename PrivatizedCounterT = CounterT, + typename SampleT = it_value_t, /// The sample value type of the input iterator typename KernelSource = DeviceHistogramKernelSource, typename KernelLauncherFactory = CUB_DETAIL_DEFAULT_KERNEL_LAUNCHER_FACTORY, @@ -593,8 +594,20 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device return error; } const HistogramPolicy active_policy = policy_selector(cc); + const auto privatization = [&] { + if constexpr (::cuda::std::is_void_v) + { + return select_privatization_mode_for_counter_size( + active_policy, max_num_output_bins, static_cast(kernel_source.CounterSize())); + } + else + { + return select_privatization_mode( + active_policy, max_num_output_bins); + } + }(); - if (!should_use_static_smem(active_policy, max_num_output_bins, int{kernel_source.CounterSize()}, NUM_ACTIVE_CHANNELS)) + if (privatization != privatization_mode::static_smem) { // Dispatch global-memory-privatized approach if (const auto error = CubDebug( @@ -715,7 +728,8 @@ template < typename LevelT, typename OffsetT, typename PolicySelector, - typename SampleT = it_value_t, /// The sample value type of the input iterator + typename PrivatizedCounterT = CounterT, + typename SampleT = it_value_t, /// The sample value type of the input iterator typename KernelSource = DeviceHistogramKernelSource, typename KernelLauncherFactory = CUB_DETAIL_DEFAULT_KERNEL_LAUNCHER_FACTORY, @@ -815,7 +829,7 @@ template _CCCL_HOST_DEVICE_API constexpr auto convert_legacy_policy() -> HistogramPolicy { using sweep = typename ActivePolicy::AgentHistogramPolicyT; - const auto kernel_config = HistogramSweepPolicy{ + const auto kernel_config = HistogramPrivatizationPolicy{ sweep::BLOCK_THREADS, sweep::PIXELS_PER_THREAD, sweep::VEC_SIZE, @@ -1001,8 +1015,8 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( ::cuda::std::is_integral_v || ::cuda::std::is_floating_point_v; if constexpr (supports_cached_search) { - if (should_use_dynamic_smem( - active_policy, max_num_output_bins, int{kernel_source.CounterSize()}, NUM_ACTIVE_CHANNELS)) + if (select_privatization_mode(active_policy, max_num_output_bins) + == privatization_mode::dynamic_smem) { using PrivatizedDecodeOpT = typename TransformsT::template CachedSearchTransform; ::cuda::std::array privatized_decode_op{}; @@ -1044,8 +1058,8 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( } // Dispatch - if (!should_use_static_smem( - active_policy, max_num_output_bins, int{kernel_source.CounterSize()}, NUM_ACTIVE_CHANNELS)) + if (select_privatization_mode(active_policy, max_num_output_bins) + != privatization_mode::static_smem) { // Too many bins to keep in shared memory. if (const auto error = CubDebug( @@ -1259,8 +1273,9 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t dispatch_even( } const HistogramPolicy active_policy = policy_selector(cc); - if (should_use_dynamic_smem( - active_policy, max_num_output_bins, int{kernel_source.CounterSize()}, NUM_ACTIVE_CHANNELS)) + const auto privatization = + select_privatization_mode(active_policy, max_num_output_bins); + if (privatization == privatization_mode::dynamic_smem) { return CubDebug((detail::histogram::dispatch; // Shared memory for AgentHistogram - __shared__ typename AgentHistogramT::TempStorage static_smem_storage; + __shared__ typename AgentHistogramT::TempStorage static_smem; AgentHistogramT agent( - static_smem_storage, + static_smem, d_samples, num_output_bins_wrapper.data(), num_privatized_bins_wrapper.data(), @@ -808,7 +808,7 @@ __launch_bounds__(int(current_policy().dynamic_smem.threads_per_ OffsetT, OutputCounterT>; - __shared__ typename AgentHistogramT::TempStorage static_smem_storage; + __shared__ typename AgentHistogramT::TempStorage static_smem; extern __shared__ __align__(16) unsigned char dynamic_smem[]; OutputDecodeOpT output_decode_op[NumActiveChannels]; @@ -823,7 +823,7 @@ __launch_bounds__(int(current_policy().dynamic_smem.threads_per_ } AgentHistogramT agent( - static_smem_storage, + static_smem, d_samples, num_output_bins_wrapper.data(), num_privatized_bins_wrapper.data(), @@ -996,10 +996,10 @@ __launch_bounds__(int( OutputCounterT>; // Shared memory for AgentHistogram - __shared__ typename AgentHistogramT::TempStorage static_smem_storage; + __shared__ typename AgentHistogramT::TempStorage static_smem; AgentHistogramT agent( - static_smem_storage, + static_smem, d_samples, num_output_bins_wrapper.data(), num_privatized_bins_wrapper.data(), diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index f5bc98bd1286..fe4cba855e35 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -22,11 +22,12 @@ #include #include #include +#include CUB_NAMESPACE_BEGIN -//! The tuning policy for one DeviceHistogram sweep pass. -struct HistogramSweepPolicy +//! The tuning policy for one DeviceHistogram privatization technique. +struct HistogramPrivatizationPolicy { int threads_per_block; //!< Number of threads in a CUDA block int items_per_thread; //!< Number of items processed per thread @@ -37,7 +38,7 @@ struct HistogramSweepPolicy bool work_stealing; //!< Whether blocks dequeue tiles from a global work queue [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool - operator==(const HistogramSweepPolicy& lhs, const HistogramSweepPolicy& rhs) noexcept + operator==(const HistogramPrivatizationPolicy& lhs, const HistogramPrivatizationPolicy& rhs) noexcept { return lhs.threads_per_block == rhs.threads_per_block && lhs.items_per_thread == rhs.items_per_thread && lhs.vec_size == rhs.vec_size && lhs.load_algorithm == rhs.load_algorithm @@ -46,10 +47,10 @@ struct HistogramSweepPolicy } #if _CCCL_HOSTED() - friend ::std::ostream& operator<<(::std::ostream& os, const HistogramSweepPolicy& p) + friend ::std::ostream& operator<<(::std::ostream& os, const HistogramPrivatizationPolicy& p) { return os - << "HistogramSweepPolicy { .threads_per_block = " << p.threads_per_block + << "HistogramPrivatizationPolicy { .threads_per_block = " << p.threads_per_block << ", .items_per_thread = " << p.items_per_thread << ", .vec_size = " << p.vec_size << ", .load_algorithm = " << p.load_algorithm << ", .load_modifier = " << p.load_modifier << ", .rle_compress = " << p.rle_compress << ", .work_stealing = " << p.work_stealing << " }"; @@ -60,30 +61,31 @@ struct HistogramSweepPolicy //! The tuning policy for all DeviceHistogram sweep passes. struct HistogramPolicy { - HistogramSweepPolicy gmem; //!< Policy for global-memory privatization - HistogramSweepPolicy static_smem; //!< Policy for compile-time-sized shared-memory privatization - HistogramSweepPolicy dynamic_smem; //!< Policy for runtime-sized shared-memory privatization - int max_privatized_static_smem_bytes; //!< Maximum compile-time-sized shared-memory allocation + HistogramPrivatizationPolicy gmem; //!< Policy for global-memory privatization + HistogramPrivatizationPolicy static_smem; //!< Policy for compile-time-sized shared-memory privatization + HistogramPrivatizationPolicy dynamic_smem; //!< Policy for runtime-sized shared-memory privatization + int max_privatized_static_smem_single_channel_bytes; //!< Single-channel compile-time-sized SMEM limit + int max_privatized_dynamic_smem_single_channel_bytes; //!< Single-channel runtime-sized SMEM limit int static_smem_min_blocks_per_sm; //!< Minimum blocks per SM requested by the static-SMEM launch bounds - int max_privatized_dynamic_smem_bytes; //!< Maximum runtime-sized shared-memory allocation - int dynamic_smem_range_max_bins; //!< Multi-channel HistogramRange limit, in bins per channel - int dynamic_smem_even_2ch_max_bins; //!< Two-channel HistogramEven limit, in bins per channel - int dynamic_smem_even_3ch_max_bins; //!< Three-channel HistogramEven limit, in bins per channel - int dynamic_smem_even_4ch_max_bins; //!< Four-channel HistogramEven limit, in bins per channel - int init_kernel_pdl_trigger_max_bins; //!< Common init-kernel PDL threshold, independent of accumulation tier + int max_privatized_dynamic_smem_multi_channel_range_bytes; //!< Multi-channel HistogramRange SMEM limit + int max_privatized_dynamic_smem_2_channel_even_bytes; //!< Two-channel HistogramEven SMEM limit + int max_privatized_dynamic_smem_3_channel_even_bytes; //!< Three-channel HistogramEven SMEM limit + int max_privatized_dynamic_smem_4_channel_even_bytes; //!< Four-channel HistogramEven SMEM limit + int max_num_bins_for_init_kernel_pdl_trigger; //!< Largest output histogram for which the init kernel triggers PDL [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(const HistogramPolicy& lhs, const HistogramPolicy& rhs) noexcept { return lhs.gmem == rhs.gmem && lhs.static_smem == rhs.static_smem && lhs.dynamic_smem == rhs.dynamic_smem - && lhs.max_privatized_static_smem_bytes == rhs.max_privatized_static_smem_bytes + && lhs.max_privatized_static_smem_single_channel_bytes == rhs.max_privatized_static_smem_single_channel_bytes + && lhs.max_privatized_dynamic_smem_single_channel_bytes == rhs.max_privatized_dynamic_smem_single_channel_bytes && lhs.static_smem_min_blocks_per_sm == rhs.static_smem_min_blocks_per_sm - && lhs.max_privatized_dynamic_smem_bytes == rhs.max_privatized_dynamic_smem_bytes - && lhs.dynamic_smem_range_max_bins == rhs.dynamic_smem_range_max_bins - && lhs.dynamic_smem_even_2ch_max_bins == rhs.dynamic_smem_even_2ch_max_bins - && lhs.dynamic_smem_even_3ch_max_bins == rhs.dynamic_smem_even_3ch_max_bins - && lhs.dynamic_smem_even_4ch_max_bins == rhs.dynamic_smem_even_4ch_max_bins - && lhs.init_kernel_pdl_trigger_max_bins == rhs.init_kernel_pdl_trigger_max_bins; + && lhs.max_privatized_dynamic_smem_multi_channel_range_bytes + == rhs.max_privatized_dynamic_smem_multi_channel_range_bytes + && lhs.max_privatized_dynamic_smem_2_channel_even_bytes == rhs.max_privatized_dynamic_smem_2_channel_even_bytes + && lhs.max_privatized_dynamic_smem_3_channel_even_bytes == rhs.max_privatized_dynamic_smem_3_channel_even_bytes + && lhs.max_privatized_dynamic_smem_4_channel_even_bytes == rhs.max_privatized_dynamic_smem_4_channel_even_bytes + && lhs.max_num_bins_for_init_kernel_pdl_trigger == rhs.max_num_bins_for_init_kernel_pdl_trigger; } [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool @@ -96,74 +98,110 @@ struct HistogramPolicy friend ::std::ostream& operator<<(::std::ostream& os, const HistogramPolicy& p) { return os - << "HistogramPolicy { .gmem = " << p.gmem << ", .static_smem = " << p.static_smem << ", .dynamic_smem = " - << p.dynamic_smem << ", .max_privatized_static_smem_bytes = " << p.max_privatized_static_smem_bytes - << ", .static_smem_min_blocks_per_sm = " << p.static_smem_min_blocks_per_sm - << ", .max_privatized_dynamic_smem_bytes = " << p.max_privatized_dynamic_smem_bytes - << ", .dynamic_smem_range_max_bins = " << p.dynamic_smem_range_max_bins - << ", .dynamic_smem_even_2ch_max_bins = " << p.dynamic_smem_even_2ch_max_bins - << ", .dynamic_smem_even_3ch_max_bins = " << p.dynamic_smem_even_3ch_max_bins - << ", .dynamic_smem_even_4ch_max_bins = " << p.dynamic_smem_even_4ch_max_bins - << ", .init_kernel_pdl_trigger_max_bins = " << p.init_kernel_pdl_trigger_max_bins << " }"; + << "HistogramPolicy { .gmem = " << p.gmem << ", .static_smem = " << p.static_smem + << ", .dynamic_smem = " << p.dynamic_smem << ", .max_privatized_static_smem_single_channel_bytes = " + << p.max_privatized_static_smem_single_channel_bytes << ", .max_privatized_dynamic_smem_single_channel_bytes = " + << p.max_privatized_dynamic_smem_single_channel_bytes << ", .static_smem_min_blocks_per_sm = " + << p.static_smem_min_blocks_per_sm << ", .max_privatized_dynamic_smem_multi_channel_range_bytes = " + << p.max_privatized_dynamic_smem_multi_channel_range_bytes + << ", .max_privatized_dynamic_smem_2_channel_even_bytes = " + << p.max_privatized_dynamic_smem_2_channel_even_bytes + << ", .max_privatized_dynamic_smem_3_channel_even_bytes = " + << p.max_privatized_dynamic_smem_3_channel_even_bytes + << ", .max_privatized_dynamic_smem_4_channel_even_bytes = " + << p.max_privatized_dynamic_smem_4_channel_even_bytes + << ", .max_num_bins_for_init_kernel_pdl_trigger = " << p.max_num_bins_for_init_kernel_pdl_trigger << " }"; } #endif }; namespace detail::histogram { -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int -max_privatized_smem_bins(int max_privatized_smem_bytes, int counter_size, int num_active_channels) +enum class privatization_mode { - if (max_privatized_smem_bytes <= 0 || counter_size <= 0 || num_active_channels <= 0) + gmem, + static_smem, + dynamic_smem +}; + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int max_privatized_smem_bins(int max_privatized_smem_bytes) +{ + static_assert(NumActiveChannels > 0); + if (max_privatized_smem_bytes <= 0) { return 0; } - return max_privatized_smem_bytes / counter_size / num_active_channels; + return max_privatized_smem_bytes / int{sizeof(CounterT)} / NumActiveChannels; } -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool -should_use_static_smem(const HistogramPolicy& policy, int num_bins, int counter_size, int num_active_channels) +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int dynamic_smem_limit_bytes(const HistogramPolicy& policy) { - return num_bins > 0 - && num_bins - <= max_privatized_smem_bins(policy.max_privatized_static_smem_bytes, counter_size, num_active_channels); + int dynamic_smem_max_bytes = policy.max_privatized_dynamic_smem_single_channel_bytes; + if constexpr (NumActiveChannels > 1) + { + if constexpr (IsEven) + { + dynamic_smem_max_bytes = + NumActiveChannels == 2 ? policy.max_privatized_dynamic_smem_2_channel_even_bytes + : NumActiveChannels == 3 ? policy.max_privatized_dynamic_smem_3_channel_even_bytes + : NumActiveChannels == 4 + ? policy.max_privatized_dynamic_smem_4_channel_even_bytes + : 0; + } + else + { + dynamic_smem_max_bytes = policy.max_privatized_dynamic_smem_multi_channel_range_bytes; + } + } + return dynamic_smem_max_bytes; } -template -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool -should_use_dynamic_smem(const HistogramPolicy& policy, int num_bins, int counter_size, int num_active_channels) +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto +select_privatization_mode(const HistogramPolicy& policy, int num_bins) -> privatization_mode { - // Single-channel limits are intentionally byte-derived: the B200 tuning - // characterized that path through the full opt-in shared-memory budget. - // Multi-channel paths use explicit per-channel caps in addition to the byte - // budget because their channel-interleaved launch shapes have distinct - // measured crossover points. if (num_bins <= 0) { - return false; + return privatization_mode::gmem; } - const bool prefer_dynamic_smem = counter_size > int{sizeof(unsigned int)} - || !should_use_static_smem(policy, num_bins, counter_size, num_active_channels); + const int static_smem_max_bins = + max_privatized_smem_bins(policy.max_privatized_static_smem_single_channel_bytes); + const int dynamic_smem_max_bytes = dynamic_smem_limit_bytes(policy); + const int dynamic_smem_max_bins = max_privatized_smem_bins(dynamic_smem_max_bytes); + const bool static_smem_fits = num_bins <= static_smem_max_bins; + const bool prefer_dynamic_smem = sizeof(CounterT) > sizeof(::cuda::std::uint32_t) || !static_smem_fits; + if (prefer_dynamic_smem && num_bins <= dynamic_smem_max_bins) + { + return privatization_mode::dynamic_smem; + } + return static_smem_fits ? privatization_mode::static_smem : privatization_mode::gmem; +} - int max_bins = max_privatized_smem_bins(policy.max_privatized_dynamic_smem_bytes, counter_size, num_active_channels); - if (num_active_channels > 1) +// The C Parallel API erases CounterT before host dispatch, so its bridge must select from the +// preserved runtime counter width. Typed CUB dispatch uses the overload above. +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto +select_privatization_mode_for_counter_size(const HistogramPolicy& policy, int num_bins, int counter_size_bytes) + -> privatization_mode +{ + if (num_bins <= 0 || counter_size_bytes <= 0) { - if constexpr (IsEven) - { - max_bins = num_active_channels == 2 ? policy.dynamic_smem_even_2ch_max_bins - : num_active_channels == 3 ? policy.dynamic_smem_even_3ch_max_bins - : num_active_channels == 4 - ? policy.dynamic_smem_even_4ch_max_bins - : 0; - } - else - { - max_bins = policy.dynamic_smem_range_max_bins; - } + return privatization_mode::gmem; } - return prefer_dynamic_smem && max_bins > 0 && num_bins <= max_bins; + const int static_smem_max_bins = policy.max_privatized_static_smem_single_channel_bytes / counter_size_bytes; + const int dynamic_smem_max_bins = + dynamic_smem_limit_bytes(policy) / counter_size_bytes / NumActiveChannels; + const bool static_smem_fits = num_bins <= static_smem_max_bins; + const bool prefer_dynamic_smem = counter_size_bytes > int{sizeof(::cuda::std::uint32_t)} || !static_smem_fits; + if (prefer_dynamic_smem && num_bins <= dynamic_smem_max_bins) + { + return privatization_mode::dynamic_smem; + } + return static_smem_fits ? privatization_mode::static_smem : privatization_mode::gmem; } #if _CCCL_HAS_CONCEPTS() @@ -173,9 +211,11 @@ concept histogram_policy_selector = policy_selector; struct policy_selector { - bool sample_is_primitive; - int sample_size_bytes; + bool sample_is_primitive; //!< Whether the sample opts into CUB's primitive-type tuning category + // Kept separately from sample_size_bytes to preserve the serialized C Parallel selector layout. + int sample_size; int counter_size_bytes; + int sample_size_bytes; int num_channels; int num_active_channels; bool is_even; @@ -190,66 +230,79 @@ private: public: [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy { + // SM100 and newer use the autoresearch launch shapes and dynamic-SMEM byte budgets. if (cc >= ::cuda::compute_capability{10, 0}) { const bool single_channel = num_channels == 1 && num_active_channels == 1; - auto gmem = HistogramSweepPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + auto gmem = HistogramPrivatizationPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; - if (num_channels > 1 && counter_size_bytes == int{sizeof(unsigned int)} && sample_is_primitive) + // Multi-channel 32-bit-counter histograms use the wider SM100 sweep tuned by autoresearch. + if (num_channels > 1 && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive) { - gmem = HistogramSweepPolicy{1024, t_scale(is_even ? 8 : 16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + gmem = + HistogramPrivatizationPolicy{1024, t_scale(is_even ? 8 : 16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; } - else if (single_channel && counter_size_bytes == int{sizeof(unsigned int)} && sample_is_primitive) + // Single-channel primitive samples with 32-bit counters use their per-sample-width tuning. + else if (single_channel && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive) { + // Eight-bit EVEN and RANGE histograms retain the dedicated SM100 tunings already in main. if (sample_size_bytes == 1) { - gmem = is_even ? HistogramSweepPolicy{928, 12, 4, BLOCK_LOAD_DIRECT, LOAD_CA, false, false} - : HistogramSweepPolicy{448, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, false, false}; + gmem = is_even ? HistogramPrivatizationPolicy{928, 12, 4, BLOCK_LOAD_DIRECT, LOAD_CA, false, false} + : HistogramPrivatizationPolicy{448, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, false, false}; } + // Sixteen-bit samples retain the SM90 tuning because autoresearch did not improve it. else if (sample_size_bytes == 2) { - // Retain the SM90 U16 sweep shape on SM100. - gmem = HistogramSweepPolicy{960, 10, 4, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, false}; + gmem = HistogramPrivatizationPolicy{960, 10, 4, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, false}; } + // Thirty-two-bit samples use the best sweep shape measured by autoresearch. else if (sample_size_bytes == 4) { - gmem = HistogramSweepPolicy{768, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + gmem = HistogramPrivatizationPolicy{768, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; } + // Sixty-four-bit samples use the best sweep shape measured by autoresearch. else if (sample_size_bytes == 8) { - gmem = HistogramSweepPolicy{768, 6, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + gmem = HistogramPrivatizationPolicy{768, 6, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; } } auto static_smem = gmem; const bool range_multi_static = - !is_even && num_channels > 1 && counter_size_bytes == int{sizeof(unsigned int)} && sample_is_primitive; - const bool range_u32_static = !is_even && single_channel && counter_size_bytes == int{sizeof(unsigned int)} - && sample_is_primitive && sample_size_bytes == 4; - const bool range_u64_static = !is_even && single_channel && counter_size_bytes == int{sizeof(unsigned int)} - && sample_is_primitive && sample_size_bytes == 8; + !is_even && num_channels > 1 && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive; + const bool range_u32_static = + !is_even && single_channel && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive + && sample_size_bytes == 4; + const bool range_u64_static = + !is_even && single_channel && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive + && sample_size_bytes == 8; + // Multi-channel and 64-bit-sample RANGE favor narrower blocks in the static-SMEM tier. if (range_multi_static || range_u64_static) { static_smem.threads_per_block = 384; } + // Thirty-two-bit-sample RANGE retains the wider block that won in the static-SMEM tier. else if (range_u32_static) { static_smem.threads_per_block = 768; } + // Sixty-four-bit-sample RANGE recovers the higher static-tier items-per-thread count. if (range_u64_static) { static_smem.items_per_thread = t_scale(16); } + // Dynamic-SMEM tuning exists only for the type and channel combinations measured by autoresearch. const bool has_dynamic_smem_tuning = - counter_size_bytes == int{sizeof(unsigned int)} && sample_is_primitive + counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive && ((single_channel && (sample_size_bytes == 1 || sample_size_bytes == 4 || sample_size_bytes == 8)) || num_channels > 1); // B200 provides 232448 bytes of opt-in shared memory. Reserve 4096 bytes // for the kernel's statically allocated shared-memory state. - const int max_privatized_dynamic_smem_bytes = has_dynamic_smem_tuning ? 232448 - 4096 : 0; - const int init_kernel_pdl_trigger_max_bins = - single_channel && counter_size_bytes == int{sizeof(unsigned int)} && sample_is_primitive + const int max_privatized_dynamic_smem_single_channel_bytes = has_dynamic_smem_tuning ? 232448 - 4096 : 0; + const int max_num_bins_for_init_kernel_pdl_trigger = + single_channel && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive && (sample_size_bytes == 1 || sample_size_bytes == 2 || sample_size_bytes == 4 || sample_size_bytes == 8) ? 2048 : 0; @@ -258,52 +311,47 @@ public: gmem, static_smem, gmem, - 512 * counter_size_bytes * num_active_channels, + 512 * counter_size_bytes, + max_privatized_dynamic_smem_single_channel_bytes, range_multi_static || range_u64_static ? 3 : 0, - max_privatized_dynamic_smem_bytes, - has_dynamic_smem_tuning ? 2048 : 0, - has_dynamic_smem_tuning ? 28544 : 0, - has_dynamic_smem_tuning ? 19029 : 0, - has_dynamic_smem_tuning ? 8192 : 0, - init_kernel_pdl_trigger_max_bins}; + has_dynamic_smem_tuning ? 2048 * counter_size_bytes * num_active_channels : 0, + has_dynamic_smem_tuning ? 28544 * counter_size_bytes * 2 : 0, + has_dynamic_smem_tuning ? 19029 * counter_size_bytes * 3 : 0, + has_dynamic_smem_tuning ? 8192 * counter_size_bytes * 4 : 0, + max_num_bins_for_init_kernel_pdl_trigger}; } + // SM90 uses its established single-channel 8-bit and 16-bit specializations. if (cc >= ::cuda::compute_capability{9, 0}) { - auto sweep = HistogramSweepPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; - if (num_channels == 1 && num_active_channels == 1 && counter_size_bytes == int{sizeof(unsigned int)} + auto sweep = HistogramPrivatizationPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + // Single-channel primitive samples with 32-bit counters use the established SM90 specializations. + if (num_channels == 1 && num_active_channels == 1 && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive) { + // Eight-bit samples use the tuned SM90 sweep. if (sample_size_bytes == 1) { - sweep = HistogramSweepPolicy{768, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, false, false}; + sweep = HistogramPrivatizationPolicy{768, 12, 4, BLOCK_LOAD_DIRECT, LOAD_LDG, false, false}; } + // Sixteen-bit samples use the tuned SM90 sweep. else if (sample_size_bytes == 2) { - sweep = HistogramSweepPolicy{960, 10, 4, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, false}; + sweep = HistogramPrivatizationPolicy{960, 10, 4, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, false}; } } - const int init_kernel_pdl_trigger_max_bins = - num_channels == 1 && num_active_channels == 1 && counter_size_bytes == int{sizeof(unsigned int)} + const int max_num_bins_for_init_kernel_pdl_trigger = + num_channels == 1 && num_active_channels == 1 && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive && (sample_size_bytes == 1 || sample_size_bytes == 2) ? 2048 : 0; return HistogramPolicy{ - sweep, - sweep, - sweep, - 256 * counter_size_bytes * num_active_channels, - 0, - 0, - 0, - 0, - 0, - 0, - init_kernel_pdl_trigger_max_bins}; + sweep, sweep, sweep, 256 * counter_size_bytes, 0, 0, 0, 0, 0, 0, max_num_bins_for_init_kernel_pdl_trigger}; } - const auto sweep = HistogramSweepPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; - return HistogramPolicy{sweep, sweep, sweep, 256 * counter_size_bytes * num_active_channels, 0, 0, 0, 0, 0, 0, 0}; + // Architectures before SM90 use the longstanding generic histogram tuning. + const auto sweep = HistogramPrivatizationPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; + return HistogramPolicy{sweep, sweep, sweep, 256 * counter_size_bytes, 0, 0, 0, 0, 0, 0, 0}; } }; @@ -317,7 +365,13 @@ struct policy_selector_from_types [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy { return policy_selector{ - is_primitive_v, int{sizeof(SampleT)}, int{sizeof(CounterT)}, NumChannels, NumActiveChannels, IsEven}(cc); + is_primitive_v, + int{sizeof(SampleT)}, + int{sizeof(CounterT)}, + int{sizeof(SampleT)}, + NumChannels, + NumActiveChannels, + IsEven}(cc); } }; } // namespace detail::histogram diff --git a/cub/test/catch2_test_device_histogram_custom_policy_hub.cu b/cub/test/catch2_test_device_histogram_custom_policy_hub.cu index 6773673b20b5..f1852f75b997 100644 --- a/cub/test/catch2_test_device_histogram_custom_policy_hub.cu +++ b/cub/test/catch2_test_device_histogram_custom_policy_hub.cu @@ -46,10 +46,10 @@ CUB_TEST("DispatchHistogram::DispatchEven: custom policy hub", "[histogram][devi REQUIRE(custom_sm75_policy.gmem.items_per_thread == 16); REQUIRE(custom_sm90_policy.gmem.threads_per_block == 256); REQUIRE(custom_sm90_policy.gmem.items_per_thread == 8); - REQUIRE(custom_sm75_policy.max_privatized_static_smem_bytes == 256 * sizeof(unsigned int)); - REQUIRE(custom_sm90_policy.max_privatized_dynamic_smem_bytes == 0); - REQUIRE(custom_sm75_policy.init_kernel_pdl_trigger_max_bins == 0); - REQUIRE(custom_sm90_policy.init_kernel_pdl_trigger_max_bins == 2048); + REQUIRE(custom_sm75_policy.max_privatized_static_smem_single_channel_bytes == 256 * sizeof(unsigned int)); + REQUIRE(custom_sm90_policy.max_privatized_dynamic_smem_single_channel_bytes == 0); + REQUIRE(custom_sm75_policy.max_num_bins_for_init_kernel_pdl_trigger == 0); + REQUIRE(custom_sm90_policy.max_num_bins_for_init_kernel_pdl_trigger == 2048); using sample_t = cuda::std::uint8_t; using counter_t = int; diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index 0e07f2f3d763..ced5ac8a6e86 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1627,7 +1627,7 @@ struct histogram_tuning _CCCL_HOST_DEVICE_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { constexpr auto sweep = - cub::HistogramSweepPolicy{BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; + cub::HistogramPrivatizationPolicy{BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; return {sweep, sweep, sweep, 256 * sizeof(unsigned int), 0, 0, 0, 0, 0, 0, 0}; } }; @@ -1645,8 +1645,19 @@ struct mixed_counter_histogram_tuning _CCCL_API constexpr auto operator()(cuda::compute_capability) const -> cub::HistogramPolicy { constexpr auto sweep = - cub::HistogramSweepPolicy{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; - return {sweep, sweep, sweep, 512 * sizeof(unsigned int), 0, 228352, 2048, 28544, 19029, 8192, 0}; + cub::HistogramPrivatizationPolicy{128, 4, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; + return { + sweep, + sweep, + sweep, + 512 * sizeof(unsigned int), + 228352, + 0, + 2048 * sizeof(unsigned int), + 28544 * sizeof(unsigned int) * 2, + 19029 * sizeof(unsigned int) * 3, + 8192 * sizeof(unsigned int) * 4, + 0}; } }; @@ -1810,8 +1821,8 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) {96, 3, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, {128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, 2052, - 2, 12345, + 2, 1024, 4096, 8192, @@ -1821,35 +1832,35 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) # if _CCCL_STD_VER >= 2020 // designated init constexpr auto p2 = cub::HistogramPolicy{ - .gmem = {.threads_per_block = 128, - .items_per_thread = 7, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .static_smem = {.threads_per_block = 96, - .items_per_thread = 3, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .dynamic_smem = {.threads_per_block = 128, - .items_per_thread = 7, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .max_privatized_static_smem_bytes = 2052, - .static_smem_min_blocks_per_sm = 2, - .max_privatized_dynamic_smem_bytes = 12345, - .dynamic_smem_range_max_bins = 1024, - .dynamic_smem_even_2ch_max_bins = 4096, - .dynamic_smem_even_3ch_max_bins = 8192, - .dynamic_smem_even_4ch_max_bins = 16384, - .init_kernel_pdl_trigger_max_bins = 2048}; + .gmem = {.threads_per_block = 128, + .items_per_thread = 7, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .static_smem = {.threads_per_block = 96, + .items_per_thread = 3, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .dynamic_smem = {.threads_per_block = 128, + .items_per_thread = 7, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .max_privatized_static_smem_single_channel_bytes = 2052, + .max_privatized_dynamic_smem_single_channel_bytes = 12345, + .static_smem_min_blocks_per_sm = 2, + .max_privatized_dynamic_smem_multi_channel_range_bytes = 1024, + .max_privatized_dynamic_smem_2_channel_even_bytes = 4096, + .max_privatized_dynamic_smem_3_channel_even_bytes = 8192, + .max_privatized_dynamic_smem_4_channel_even_bytes = 16384, + .max_num_bins_for_init_kernel_pdl_trigger = 2048}; # else // _CCCL_STD_VER >= 2020 constexpr auto p2 = p1; # endif // _CCCL_STD_VER >= 2020 @@ -1876,23 +1887,22 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); - STATIC_REQUIRE( - cub::detail::histogram::max_privatized_smem_bins(sm90_policy.max_privatized_static_smem_bytes, 4, 1) == 256); - STATIC_REQUIRE( - cub::detail::histogram::max_privatized_smem_bins(sm100_policy.max_privatized_static_smem_bytes, 4, 1) == 512); - STATIC_REQUIRE( - cub::detail::histogram::max_privatized_smem_bins(sm100_policy.max_privatized_static_smem_bytes, 4, 4) == 128); - STATIC_REQUIRE(sm90_policy.max_privatized_dynamic_smem_bytes == 0); - STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_bytes == 228352); - STATIC_REQUIRE(sm100_wide_counter_policy.max_privatized_dynamic_smem_bytes == 0); - STATIC_REQUIRE( - cub::detail::histogram::max_privatized_smem_bins(sm100_policy.max_privatized_dynamic_smem_bytes, 4, 1) == 57088); - STATIC_REQUIRE( - cub::detail::histogram::max_privatized_smem_bins(sm100_policy.max_privatized_dynamic_smem_bytes, 4, 4) == 14272); - STATIC_REQUIRE(sm100_policy.dynamic_smem_range_max_bins == 2048); - STATIC_REQUIRE(sm100_policy.dynamic_smem_even_2ch_max_bins == 28544); - STATIC_REQUIRE(sm100_policy.dynamic_smem_even_3ch_max_bins == 19029); - STATIC_REQUIRE(sm100_policy.dynamic_smem_even_4ch_max_bins == 8192); + STATIC_REQUIRE(cub::detail::histogram::max_privatized_smem_bins( + sm90_policy.max_privatized_static_smem_single_channel_bytes) + == 256); + STATIC_REQUIRE(cub::detail::histogram::max_privatized_smem_bins( + sm100_policy.max_privatized_static_smem_single_channel_bytes) + == 512); + STATIC_REQUIRE(sm90_policy.max_privatized_dynamic_smem_single_channel_bytes == 0); + STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_single_channel_bytes == 228352); + STATIC_REQUIRE(sm100_wide_counter_policy.max_privatized_dynamic_smem_single_channel_bytes == 0); + STATIC_REQUIRE(cub::detail::histogram::max_privatized_smem_bins( + sm100_policy.max_privatized_dynamic_smem_single_channel_bytes) + == 57088); + STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_multi_channel_range_bytes == 2048 * 4 * 1); + STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_2_channel_even_bytes == 28544 * 4 * 2); + STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_3_channel_even_bytes == 19029 * 4 * 3); + STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_4_channel_even_bytes == 8192 * 4 * 4); STATIC_REQUIRE(sm100_policy.gmem.threads_per_block == 768); STATIC_REQUIRE(sm100_policy.gmem.items_per_thread == 12); STATIC_REQUIRE(sm100_policy.static_smem == sm100_policy.gmem); @@ -1909,21 +1919,53 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" constexpr auto sm100_multi_range_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); + constexpr auto sm100_even_2ch_policy = + cub::detail::histogram::policy_selector_from_types{}( + cuda::compute_capability{10, 0}); + constexpr auto sm100_even_3ch_policy = + cub::detail::histogram::policy_selector_from_types{}( + cuda::compute_capability{10, 0}); + constexpr auto sm100_even_4ch_policy = + cub::detail::histogram::policy_selector_from_types{}( + cuda::compute_capability{10, 0}); STATIC_REQUIRE(sm100_multi_range_policy.gmem.threads_per_block == 1024); STATIC_REQUIRE(sm100_multi_range_policy.gmem.items_per_thread == 5); STATIC_REQUIRE(sm100_multi_range_policy.static_smem.threads_per_block == 384); STATIC_REQUIRE(sm100_multi_range_policy.static_smem.items_per_thread == 5); STATIC_REQUIRE(sm100_multi_range_policy.static_smem_min_blocks_per_sm == 3); - STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57088, 4, 1)); - STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 57089, 4, 1)); - STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 2048, 4, 3)); - STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 2049, 4, 3)); - STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 8192, 4, 4)); - STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 8193, 4, 4)); - STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 28544, 4, 2)); - STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 28545, 4, 2)); - STATIC_REQUIRE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 19029, 4, 3)); - STATIC_REQUIRE_FALSE(cub::detail::histogram::should_use_dynamic_smem(sm100_policy, 19030, 4, 3)); + using cub::detail::histogram::privatization_mode; + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_policy, 512) + == privatization_mode::static_smem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_policy, 513) + == privatization_mode::dynamic_smem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_policy, 57088) + == privatization_mode::dynamic_smem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_policy, 57089) + == privatization_mode::gmem); + STATIC_REQUIRE( + cub::detail::histogram::select_privatization_mode(sm100_multi_range_policy, 2048) + == privatization_mode::dynamic_smem); + STATIC_REQUIRE( + cub::detail::histogram::select_privatization_mode(sm100_multi_range_policy, 2049) + == privatization_mode::gmem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_4ch_policy, 8192) + == privatization_mode::dynamic_smem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_4ch_policy, 8193) + == privatization_mode::gmem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_2ch_policy, 28544) + == privatization_mode::dynamic_smem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_2ch_policy, 28545) + == privatization_mode::gmem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_3ch_policy, 19029) + == privatization_mode::dynamic_smem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_3ch_policy, 19030) + == privatization_mode::gmem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode_for_counter_size( + sm100_policy, 256, sizeof(cuda::std::uint64_t)) + == privatization_mode::dynamic_smem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode_for_counter_size( + sm100_policy, 28545, sizeof(cuda::std::uint64_t)) + == privatization_mode::gmem); } #endif // _CCCL_COMPILER(GCC, >=, 8) diff --git a/cub/test/catch2_test_device_histogram_env_api.cu b/cub/test/catch2_test_device_histogram_env_api.cu index 0d574db4ad6c..becd885a5720 100644 --- a/cub/test/catch2_test_device_histogram_env_api.cu +++ b/cub/test/catch2_test_device_histogram_env_api.cu @@ -418,20 +418,20 @@ struct HistogramPolicySelector { __host__ __device__ constexpr auto operator()(cuda::compute_capability cc) const -> cub::HistogramPolicy { - const auto sweep = cub::HistogramSweepPolicy{ + const auto sweep = cub::HistogramPrivatizationPolicy{ 128, cc > cuda::compute_capability{9, 0} ? 16 : 7, 4, cub::BLOCK_LOAD_DIRECT, cub::LOAD_LDG, false, false}; return { - .gmem = sweep, - .static_smem = sweep, - .dynamic_smem = sweep, - .max_privatized_static_smem_bytes = 256 * sizeof(unsigned int), - .static_smem_min_blocks_per_sm = 0, - .max_privatized_dynamic_smem_bytes = 0, - .dynamic_smem_range_max_bins = 0, - .dynamic_smem_even_2ch_max_bins = 0, - .dynamic_smem_even_3ch_max_bins = 0, - .dynamic_smem_even_4ch_max_bins = 0, - .init_kernel_pdl_trigger_max_bins = 2048}; + .gmem = sweep, + .static_smem = sweep, + .dynamic_smem = sweep, + .max_privatized_static_smem_single_channel_bytes = 256 * sizeof(unsigned int), + .max_privatized_dynamic_smem_single_channel_bytes = 0, + .static_smem_min_blocks_per_sm = 0, + .max_privatized_dynamic_smem_multi_channel_range_bytes = 0, + .max_privatized_dynamic_smem_2_channel_even_bytes = 0, + .max_privatized_dynamic_smem_3_channel_even_bytes = 0, + .max_privatized_dynamic_smem_4_channel_even_bytes = 0, + .max_num_bins_for_init_kernel_pdl_trigger = 2048}; } }; // example-end histogram-even-policy-selector From 896b53b404a973a8c80d90adef0d43d57abf3c17 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Tue, 11 Aug 2026 14:56:14 +0000 Subject: [PATCH 35/45] [cub] Align histogram changes with current trunk --- cub/cub/device/dispatch/dispatch_histogram.cuh | 12 ++++++------ cub/test/catch2_test_device_histogram.cu | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 4e68cc411064..21be1771def4 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -1489,9 +1489,9 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc { static constexpr bool uses_default_policy = ::cuda::std::is_void_v; using policy_selector_t = ::cuda::std::_If< - uses_default_policy, - detail::histogram::policy_selector_from_types, - detail::histogram::policy_selector_from_hub>; + uses_default_policy, + detail::histogram::policy_selector_from_types, + detail::histogram::policy_selector_from_hub>; return detail::histogram::dispatch_range( d_temp_storage, temp_storage_bytes, @@ -1577,9 +1577,9 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc { static constexpr bool uses_default_policy = ::cuda::std::is_void_v; using policy_selector_t = ::cuda::std::_If< - uses_default_policy, - detail::histogram::policy_selector_from_types, - detail::histogram::policy_selector_from_hub>; + uses_default_policy, + detail::histogram::policy_selector_from_types, + detail::histogram::policy_selector_from_hub>; return detail::histogram::dispatch_even( d_temp_storage, temp_storage_bytes, diff --git a/cub/test/catch2_test_device_histogram.cu b/cub/test/catch2_test_device_histogram.cu index 0fcf83c8f16c..cdee59980c4d 100644 --- a/cub/test/catch2_test_device_histogram.cu +++ b/cub/test/catch2_test_device_histogram.cu @@ -574,7 +574,7 @@ CUB_TEST_LIST("DeviceHistogram::Histogram* channel configs", test_even_and_range(256, 256 + 1, 128, 32); } -C2H_TEST("DeviceHistogram::Histogram* dynamic shared-memory privatization", "[histogram][device]") +CUB_TEST("DeviceHistogram::Histogram* dynamic shared-memory privatization", "[histogram][device]", CUB_SMALL) { int current_device{}; REQUIRE(cudaSuccess == cudaGetDevice(¤t_device)); @@ -675,7 +675,7 @@ CUB_TEST("DeviceHistogram::HistogramRange levels/samples aliasing", "[histogram_ } } -C2H_TEST("DeviceHistogram::HistogramRange interpolation avoids signed overflow", "[histogram_range][device]") +CUB_TEST("DeviceHistogram::HistogramRange interpolation avoids signed overflow", "[histogram_range][device]", CUB_SMALL) { int current_device{}; REQUIRE(cudaSuccess == cudaGetDevice(¤t_device)); From da06decdd4f0dc4e91758f82067812ef68c88e52 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Tue, 11 Aug 2026 20:50:32 +0000 Subject: [PATCH 36/45] [cub] Restore tuned static histogram limits --- .../device/dispatch/tuning/tuning_histogram.cuh | 6 +++++- cub/test/catch2_test_device_histogram_env.cu | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index fe4cba855e35..36e37dec9421 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -306,12 +306,16 @@ public: && (sample_size_bytes == 1 || sample_size_bytes == 2 || sample_size_bytes == 4 || sample_size_bytes == 8) ? 2048 : 0; + // The larger compile-time-sized histogram regresses the tuned RANGE kernels even when the runtime histogram + // has at most 256 bins. Keep their static allocation at 256 bins and use the runtime-sized tier above it. + const int max_privatized_static_smem_single_channel_bytes = + (range_multi_static || range_u64_static ? 256 : 512) * counter_size_bytes; return HistogramPolicy{ gmem, static_smem, gmem, - 512 * counter_size_bytes, + max_privatized_static_smem_single_channel_bytes, max_privatized_dynamic_smem_single_channel_bytes, range_multi_static || range_u64_static ? 3 : 0, has_dynamic_smem_tuning ? 2048 * counter_size_bytes * num_active_channels : 0, diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index ced5ac8a6e86..14fc86138b3b 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1915,6 +1915,9 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE(sm100_range_u64_policy.static_smem.threads_per_block == 384); STATIC_REQUIRE(sm100_range_u64_policy.static_smem.items_per_thread == 8); STATIC_REQUIRE(sm100_range_u64_policy.static_smem_min_blocks_per_sm == 3); + STATIC_REQUIRE(cub::detail::histogram::max_privatized_smem_bins( + sm100_range_u64_policy.max_privatized_static_smem_single_channel_bytes) + == 256); constexpr auto sm100_multi_range_policy = cub::detail::histogram::policy_selector_from_types{}( @@ -1933,6 +1936,9 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE(sm100_multi_range_policy.static_smem.threads_per_block == 384); STATIC_REQUIRE(sm100_multi_range_policy.static_smem.items_per_thread == 5); STATIC_REQUIRE(sm100_multi_range_policy.static_smem_min_blocks_per_sm == 3); + STATIC_REQUIRE(cub::detail::histogram::max_privatized_smem_bins( + sm100_multi_range_policy.max_privatized_static_smem_single_channel_bytes) + == 256); using cub::detail::histogram::privatization_mode; STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_policy, 512) @@ -1943,6 +1949,16 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" == privatization_mode::dynamic_smem); STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_policy, 57089) == privatization_mode::gmem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_range_u64_policy, 256) + == privatization_mode::static_smem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_range_u64_policy, 257) + == privatization_mode::dynamic_smem); + STATIC_REQUIRE( + cub::detail::histogram::select_privatization_mode(sm100_multi_range_policy, 256) + == privatization_mode::static_smem); + STATIC_REQUIRE( + cub::detail::histogram::select_privatization_mode(sm100_multi_range_policy, 257) + == privatization_mode::dynamic_smem); STATIC_REQUIRE( cub::detail::histogram::select_privatization_mode(sm100_multi_range_policy, 2048) == privatization_mode::dynamic_smem); From a20fb201f65bf03d984719f4f0840f938af102d5 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sat, 15 Aug 2026 17:25:38 +0000 Subject: [PATCH 37/45] [cub] Address histogram review feedback --- c/parallel/src/histogram.cu | 14 +- cub/cub/agent/agent_histogram.cuh | 27 +- .../device/dispatch/dispatch_histogram.cuh | 279 +++++++++++------- .../dispatch/kernels/kernel_histogram.cuh | 180 +++++------ .../dispatch/tuning/tuning_histogram.cuh | 73 +++-- ...test_device_histogram_custom_policy_hub.cu | 4 +- cub/test/catch2_test_device_histogram_env.cu | 82 ++--- .../catch2_test_device_histogram_env_api.cu | 22 +- 8 files changed, 372 insertions(+), 309 deletions(-) diff --git a/c/parallel/src/histogram.cu b/c/parallel/src/histogram.cu index c4c2fdacf4d2..c80face8d5aa 100644 --- a/c/parallel/src/histogram.cu +++ b/c/parallel/src/histogram.cu @@ -325,14 +325,20 @@ static_assert(device_histogram_policy()(detail::current_tuning_cc()) == {4}, "Ho #endif const bool is_byte_sample = d_samples.value_type.size == 1; + const int num_privatized_bins = + is_byte_sample ? cub::detail::histogram::byte_sample_privatized_levels - 1 : num_output_levels_val - 1; + const int counter_size_bytes = static_cast(d_output_histograms.value_type.size); const auto privatization = - is_byte_sample - ? cub::detail::histogram::privatization_mode::static_smem - : cub::detail::histogram::select_privatization_mode_for_counter_size( - active_policy, num_output_levels_val - 1, static_cast(d_output_histograms.value_type.size)); + is_evenly_segmented + ? cub::detail::histogram::select_privatization_mode_for_counter_size( + active_policy, num_privatized_bins, counter_size_bytes) + : cub::detail::histogram::select_privatization_mode_for_counter_size( + active_policy, num_privatized_bins, counter_size_bytes); const std::string_view privatization_mode_t = privatization == cub::detail::histogram::privatization_mode::static_smem ? "cub::detail::histogram::HistogramPrivatizedStaticSmem" + : privatization == cub::detail::histogram::privatization_mode::dynamic_smem + ? "cub::detail::histogram::HistogramPrivatizedDynamicSmem" : "cub::detail::histogram::HistogramPrivatizedGmem"; std::string init_kernel_name = histogram::get_init_kernel_name(num_active_channels, counter_cpp, offset_cpp); diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index 7f5d2dbf463b..5ff9ac75d256 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -157,7 +157,7 @@ struct AgentHistogram static constexpr bool uses_dynamic_smem = is_privatized_dynamic_smem_v; static constexpr bool uses_gmem = is_privatized_gmem_v; static constexpr auto policy = current_policy(); - static constexpr auto sweep = + static constexpr auto privatization_policy = uses_static_smem ? policy.static_smem : uses_dynamic_smem ? policy.dynamic_smem @@ -166,16 +166,16 @@ struct AgentHistogram uses_static_smem ? policy.max_privatized_static_smem_single_channel_bytes / int{sizeof(CounterT)} : 0; static_assert(!uses_static_smem || privatized_static_smem_bins > 0, "Static-SMEM privatization requires room for at least one bin"); - static constexpr int vec_size = sweep.vec_size; - static constexpr int threads_per_block = sweep.threads_per_block; - static constexpr int pixels_per_thread = sweep.items_per_thread; + static constexpr int vec_size = privatization_policy.vec_size; + static constexpr int threads_per_block = privatization_policy.threads_per_block; + static constexpr int pixels_per_thread = privatization_policy.items_per_thread; static constexpr int samples_per_thread = pixels_per_thread * NumChannels; static constexpr int vecs_per_thread = samples_per_thread / vec_size; static constexpr int tile_pixels = pixels_per_thread * threads_per_block; static constexpr int tile_samples = samples_per_thread * threads_per_block; - static constexpr bool is_rle_compress = sweep.rle_compress; - static constexpr bool is_work_stealing = sweep.work_stealing; - static constexpr CacheLoadModifier load_modifier = sweep.load_modifier; + static constexpr bool is_rle_compress = privatization_policy.rle_compress; + static constexpr bool is_work_stealing = privatization_policy.work_stealing; + static constexpr CacheLoadModifier load_modifier = privatization_policy.load_modifier; using SampleT = it_value_t; using PixelT = typename CubVector::Type; @@ -190,9 +190,10 @@ struct AgentHistogram SampleIteratorT>; using WrappedPixelIteratorT = CacheModifiedInputIterator; using WrappedVecsIteratorT = CacheModifiedInputIterator; - using BlockLoadSampleT = BlockLoad; - using BlockLoadPixelT = BlockLoad; - using BlockLoadVecT = BlockLoad; + using BlockLoadSampleT = + BlockLoad; + using BlockLoadPixelT = BlockLoad; + using BlockLoadVecT = BlockLoad; struct _TempStorage { @@ -216,8 +217,8 @@ struct AgentHistogram _TempStorage& static_smem_storage; WrappedSampleIteratorT d_wrapped_samples; // with cache modifier applied, if possible SampleT* d_native_samples; // possibly nullptr if unavailable - const int* num_output_bins; // one for each channel - const int* num_privatized_bins; // one for each channel + const int* num_output_bins; // array of ints, one for each channel + const int* num_privatized_bins; // array of ints, one for each channel CounterT* gmem_privatized_histograms[NumActiveChannels]; // one for each channel CounterT* dynamic_smem_privatized_histograms[NumActiveChannels]; // dynamic shared-memory channel bases, when enabled OutputCounterT** output_histogram; // final output, in global memory @@ -415,7 +416,7 @@ struct AgentHistogram bool is_valid[pixels_per_thread]; LoadTile(block_offset, valid_samples, samples); - MarkValid(is_valid, valid_samples); + MarkValid(is_valid, valid_samples); AccumulatePixels(samples, is_valid, ::cuda::std::bool_constant{}); } diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 21be1771def4..505c63c496c0 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -110,21 +110,6 @@ struct DeviceHistogramKernelSource OutputCounterT>; } - template - _CCCL_HIDE_FROM_ABI CUB_RUNTIME_FUNCTION static constexpr auto HistogramSweepDynamicSmemKernel() - { - return &DeviceHistogramSweepDynamicSmemKernel< - PolicyT, - NUM_CHANNELS, - NUM_ACTIVE_CHANNELS, - SampleIteratorT, - CounterT, - PrivatizedDecodeOpT, - OutputDecodeOpT, - OffsetT, - OutputCounterT>; - } - /// Returns the device-init histogram sweep kernel that initializes decode operators from level arrays in the kernel. template (); auto sweep_kernel = [&] { - if constexpr (is_privatized_dynamic_smem_v) - { - static_assert(!IsDeviceInit, "Dynamic shared-memory histograms require host-initialized transforms"); - using output_decode_op_t = typename FirstLevelArrayT::value_type; - using privatized_decode_op_t = typename SecondLevelArrayT::value_type; - return kernel_source - .template HistogramSweepDynamicSmemKernel(); - } - else if constexpr (IsDeviceInit) + if constexpr (IsDeviceInit) { return kernel_source.template HistogramSweepKernelDeviceInit< PolicySelector, @@ -607,7 +584,36 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device } }(); - if (privatization != privatization_mode::static_smem) + if (privatization == privatization_mode::dynamic_smem) + { + if (const auto error = CubDebug( + (detail::histogram::dispatch( + d_temp_storage, + temp_storage_bytes, + d_samples, + d_output_histograms, + num_output_levels, + num_output_levels, + upper_level, + lower_level, + max_num_output_bins, + num_row_pixels, + num_rows, + row_stride_samples, + stream, + policy_selector, + kernel_source, + launcher_factory)))) + { + return error; + } + } + else if (privatization == privatization_mode::gmem) { // Dispatch global-memory-privatized approach if (const auto error = CubDebug( @@ -779,29 +785,58 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device } int max_num_output_bins = max_levels - 1; - if (const auto error = CubDebug( - (detail::histogram::dispatch( - d_temp_storage, - temp_storage_bytes, - d_samples, - d_output_histograms, - num_privatized_levels, - num_output_levels, - upper_level, - lower_level, - max_num_output_bins, - num_row_pixels, - num_rows, - row_stride_samples, - stream, - policy_selector, - kernel_source, - launcher_factory)))) + ::cuda::compute_capability cc{}; + if (const auto error = CubDebug(launcher_factory.PtxComputeCap(cc))) + { + return error; + } + const HistogramPolicy active_policy = policy_selector(cc); + constexpr int num_privatized_bins = byte_sample_privatized_levels - 1; + const auto privatization = [&] { + if constexpr (::cuda::std::is_void_v) + { + return select_privatization_mode_for_counter_size( + active_policy, num_privatized_bins, static_cast(kernel_source.CounterSize())); + } + else + { + return select_privatization_mode( + active_policy, num_privatized_bins); + } + }(); + + const auto dispatch_with = [&](auto mode) { + using privatization_mode_t = decltype(mode); + return detail::histogram::dispatch( + d_temp_storage, + temp_storage_bytes, + d_samples, + d_output_histograms, + num_privatized_levels, + num_output_levels, + upper_level, + lower_level, + max_num_output_bins, + num_row_pixels, + num_rows, + row_stride_samples, + stream, + policy_selector, + kernel_source, + launcher_factory); + }; + + const auto error = + privatization == privatization_mode::static_smem ? dispatch_with(HistogramPrivatizedStaticSmem{}) + : privatization == privatization_mode::dynamic_smem + ? dispatch_with(HistogramPrivatizedDynamicSmem{}) + : dispatch_with(HistogramPrivatizedGmem{}); + if (CubDebug(error)) { return error; } @@ -811,15 +846,15 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device // TODO(bgruber): drop in CCCL 4.0 template -_CCCL_HOST_DEVICE_API constexpr auto convert_pdl_trigger(int) +_CCCL_HOST_DEVICE_API constexpr auto convert_pdl_trigger_bytes(int) -> decltype(ActivePolicy::init_kernel_pdl_trigger_max_bins) { - return ActivePolicy::init_kernel_pdl_trigger_max_bins; + return ActivePolicy::init_kernel_pdl_trigger_max_bins * int{sizeof(unsigned int)}; } // TODO(bgruber): drop in CCCL 4.0 template -_CCCL_HOST_DEVICE_API constexpr auto convert_pdl_trigger(long) +_CCCL_HOST_DEVICE_API constexpr auto convert_pdl_trigger_bytes(long) { return 0; } @@ -838,17 +873,7 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_legacy_policy() -> HistogramPolicy sweep::IS_RLE_COMPRESS, sweep::IS_WORK_STEALING}; return { - kernel_config, - kernel_config, - kernel_config, - 256 * int{sizeof(unsigned int)}, - 0, - 0, - 0, - 0, - 0, - 0, - convert_pdl_trigger(0)}; + kernel_config, kernel_config, kernel_config, 1024, 0, 0, 0, 0, 0, 0, convert_pdl_trigger_bytes(0)}; } // TODO(bgruber): drop in CCCL 4.0 @@ -958,29 +983,48 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( } int max_num_output_bins = max_levels - 1; - if (const auto error = CubDebug( - (detail::histogram::dispatch( - d_temp_storage, - temp_storage_bytes, - d_samples, - d_output_histograms, - num_privatized_levels, - num_output_levels, - output_decode_op, - privatized_decode_op, - max_num_output_bins, - num_row_pixels, - num_rows, - row_stride_samples, - stream, - policy_selector, - kernel_source, - launcher_factory)))) + ::cuda::compute_capability cc{}; + if (const auto error = CubDebug(launcher_factory.PtxComputeCap(cc))) + { + return error; + } + const HistogramPolicy active_policy = policy_selector(cc); + constexpr int num_privatized_bins = byte_sample_privatized_levels - 1; + const auto privatization = + select_privatization_mode(active_policy, num_privatized_bins); + + const auto dispatch_with = [&](auto mode) { + using privatization_mode_t = decltype(mode); + return detail::histogram::dispatch( + d_temp_storage, + temp_storage_bytes, + d_samples, + d_output_histograms, + num_privatized_levels, + num_output_levels, + output_decode_op, + privatized_decode_op, + max_num_output_bins, + num_row_pixels, + num_rows, + row_stride_samples, + stream, + policy_selector, + kernel_source, + launcher_factory); + }; + + const auto error = + privatization == privatization_mode::static_smem ? dispatch_with(HistogramPrivatizedStaticSmem{}) + : privatization == privatization_mode::dynamic_smem + ? dispatch_with(HistogramPrivatizedDynamicSmem{}) + : dispatch_with(HistogramPrivatizedGmem{}); + if (CubDebug(error)) { return error; } @@ -1202,29 +1246,48 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t dispatch_even( } int max_num_output_bins = max_levels - 1; - if (const auto error = CubDebug( - (detail::histogram::dispatch( - d_temp_storage, - temp_storage_bytes, - d_samples, - d_output_histograms, - num_privatized_levels, - num_output_levels, - output_decode_op, - privatized_decode_op, - max_num_output_bins, - num_row_pixels, - num_rows, - row_stride_samples, - stream, - policy_selector, - kernel_source, - launcher_factory)))) + ::cuda::compute_capability cc{}; + if (const auto error = CubDebug(launcher_factory.PtxComputeCap(cc))) + { + return error; + } + const HistogramPolicy active_policy = policy_selector(cc); + constexpr int num_privatized_bins = byte_sample_privatized_levels - 1; + const auto privatization = + select_privatization_mode(active_policy, num_privatized_bins); + + const auto dispatch_with = [&](auto mode) { + using privatization_mode_t = decltype(mode); + return detail::histogram::dispatch( + d_temp_storage, + temp_storage_bytes, + d_samples, + d_output_histograms, + num_privatized_levels, + num_output_levels, + output_decode_op, + privatized_decode_op, + max_num_output_bins, + num_row_pixels, + num_rows, + row_stride_samples, + stream, + policy_selector, + kernel_source, + launcher_factory); + }; + + const auto error = + privatization == privatization_mode::static_smem ? dispatch_with(HistogramPrivatizedStaticSmem{}) + : privatization == privatization_mode::dynamic_smem + ? dispatch_with(HistogramPrivatizedDynamicSmem{}) + : dispatch_with(HistogramPrivatizedGmem{}); + if (CubDebug(error)) { return error; } diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index 21129d57f5a9..2e904f3eb3d5 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -594,14 +594,18 @@ _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramInitKernel( _CCCL_PDL_GRID_DEPENDENCY_SYNC(); // TODO(bgruber): if we had the guarantee that there would be no pending // writes/reads to the temp storage, we could omit the sync here - // we trigger the sweep kernel only if we have a small number of remaining writes in this kernel - NV_IF_TARGET(NV_PROVIDES_SM_90, ({ - if (::cuda::std::reduce(num_output_bins_wrapper.begin(), num_output_bins_wrapper.end()) - <= policy.max_num_bins_for_init_kernel_pdl_trigger) - { - _CCCL_PDL_TRIGGER_NEXT_LAUNCH(); - } - })); + // Trigger the sweep only when the remaining output writes occupy at most the tuned byte threshold. + NV_IF_TARGET( + NV_PROVIDES_SM_90, ({ + const auto output_histogram_bytes = + ::cuda::std::reduce(num_output_bins_wrapper.begin(), num_output_bins_wrapper.end(), ::cuda::std::uint64_t{0}) + * sizeof(CounterT); + if (output_histogram_bytes + <= static_cast<::cuda::std::uint64_t>(policy.max_output_histogram_bytes_for_init_kernel_pdl_trigger)) + { + _CCCL_PDL_TRIGGER_NEXT_LAUNCH(); + } + })); if ((threadIdx.x == 0) && (blockIdx.x == 0)) { @@ -620,6 +624,36 @@ _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramInitKernel( } } +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto histogram_privatization_policy() -> HistogramPrivatizationPolicy +{ + if constexpr (is_privatized_static_smem_v) + { + return current_policy().static_smem; + } + else if constexpr (is_privatized_dynamic_smem_v) + { + return current_policy().dynamic_smem; + } + else + { + return current_policy().gmem; + } +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr int histogram_min_blocks_per_sm() +{ + if constexpr (is_privatized_static_smem_v) + { + return current_policy().static_smem_min_blocks_per_sm; + } + else + { + return 0; + } +} + //! Histogram privatized sweep kernel entry point (multi-block). //! Computes privatized histograms, one per thread block. //! This kernel receives pre-initialized decode operators from the host. @@ -704,21 +738,15 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__( - int(is_privatized_static_smem_v ? current_policy().static_smem.threads_per_block - : is_privatized_dynamic_smem_v - ? current_policy().dynamic_smem.threads_per_block - : current_policy().gmem.threads_per_block), - int(is_privatized_static_smem_v - ? current_policy().static_smem_min_blocks_per_sm - : 0)) +__launch_bounds__(int(histogram_privatization_policy().threads_per_block), + int(histogram_min_blocks_per_sm())) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepKernel( const SampleIteratorT d_samples, const ::cuda::std::array num_output_bins_wrapper, const ::cuda::std::array num_privatized_bins_wrapper, ::cuda::std::array d_output_histograms_wrapper, ::cuda::std::array d_privatized_histograms_wrapper, - const ::cuda::std::array output_decode_op_wrapper, + ::cuda::std::array output_decode_op_wrapper, ::cuda::std::array privatized_decode_op_wrapper, const OffsetT num_row_pixels, const OffsetT num_rows, @@ -738,8 +766,21 @@ __launch_bounds__( OffsetT, OutputCounterT>; - // Shared memory for AgentHistogram __shared__ typename AgentHistogramT::TempStorage static_smem; + extern __shared__ __align__(16) unsigned char dynamic_smem[]; + + CounterT* dynamic_smem_privatized_histograms = nullptr; + if constexpr (is_privatized_dynamic_smem_v) + { + dynamic_smem_privatized_histograms = reinterpret_cast(dynamic_smem); + } + + _CCCL_PRAGMA_UNROLL_FULL() + for (int channel = 0; channel < NumActiveChannels; ++channel) + { + output_decode_op_wrapper[channel].Precompute(); + privatized_decode_op_wrapper[channel].Precompute(); + } AgentHistogramT agent( static_smem, @@ -750,7 +791,7 @@ __launch_bounds__( d_privatized_histograms_wrapper.data(), output_decode_op_wrapper.data(), privatized_decode_op_wrapper.data(), - nullptr); + dynamic_smem_privatized_histograms); // Initialize counters agent.InitBinCounters(); @@ -762,82 +803,6 @@ __launch_bounds__( agent.StoreOutput(); } -//! Histogram sweep kernel with the privatized histogram in dynamic shared memory. -//! -//! The host supplies `sum(num_privatized_bins[ch]) * sizeof(CounterT)` bytes of -//! dynamic shared memory, which the agent partitions per channel. Keeping the -//! runtime-sized histogram outside `TempStorage` -//! allows one kernel instantiation to cover larger histograms without a ladder -//! of statically sized kernels. -template -#if _CCCL_HAS_CONCEPTS() - requires histogram_policy_selector -#endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int(current_policy().dynamic_smem.threads_per_block)) - _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDynamicSmemKernel( - const SampleIteratorT d_samples, - const ::cuda::std::array num_output_bins_wrapper, - const ::cuda::std::array num_privatized_bins_wrapper, - ::cuda::std::array d_output_histograms_wrapper, - ::cuda::std::array d_privatized_histograms_wrapper, - const ::cuda::std::array output_decode_op_wrapper, - const ::cuda::std::array privatized_decode_op_wrapper, - const OffsetT num_row_pixels, - const OffsetT num_rows, - const OffsetT row_stride_samples, - const int tiles_per_row, - GridQueue tile_queue) -{ - using AgentHistogramT = - AgentHistogram; - - __shared__ typename AgentHistogramT::TempStorage static_smem; - extern __shared__ __align__(16) unsigned char dynamic_smem[]; - - OutputDecodeOpT output_decode_op[NumActiveChannels]; - PrivatizedDecodeOpT privatized_decode_op[NumActiveChannels]; - _CCCL_PRAGMA_UNROLL_FULL() - for (int channel = 0; channel < NumActiveChannels; ++channel) - { - output_decode_op[channel] = output_decode_op_wrapper[channel]; - privatized_decode_op[channel] = privatized_decode_op_wrapper[channel]; - output_decode_op[channel].Precompute(); - privatized_decode_op[channel].Precompute(); - } - - AgentHistogramT agent( - static_smem, - d_samples, - num_output_bins_wrapper.data(), - num_privatized_bins_wrapper.data(), - d_output_histograms_wrapper.data(), - d_privatized_histograms_wrapper.data(), - output_decode_op, - privatized_decode_op, - reinterpret_cast(dynamic_smem)); - - agent.InitBinCounters(); - agent.ConsumeTiles(num_row_pixels, num_rows, row_stride_samples, tiles_per_row, tile_queue); - agent.StoreOutput(); -} - //! Histogram privatized sweep kernel entry point (multi-block) with device-side initialization. //! Computes privatized histograms, one per thread block. //! This kernel initializes decode operators from level arrays inside the kernel. @@ -938,11 +903,8 @@ template #endif // _CCCL_HAS_CONCEPTS() -__launch_bounds__(int( - is_privatized_static_smem_v ? current_policy().static_smem.threads_per_block - : is_privatized_dynamic_smem_v - ? current_policy().dynamic_smem.threads_per_block - : current_policy().gmem.threads_per_block)) +__launch_bounds__(int(histogram_privatization_policy().threads_per_block), + int(histogram_min_blocks_per_sm())) _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramSweepDeviceInitKernel( const SampleIteratorT d_samples, ::cuda::std::array num_output_bins_wrapper, @@ -971,6 +933,7 @@ __launch_bounds__(int( output_decode_op[channel].Init(num_levels, upper_level, lower_level); } } + else { _CCCL_PRAGMA_UNROLL_FULL() @@ -983,6 +946,13 @@ __launch_bounds__(int( } } + _CCCL_PRAGMA_UNROLL_FULL() + for (int channel = 0; channel < NumActiveChannels; ++channel) + { + output_decode_op[channel].Precompute(); + privatized_decode_op[channel].Precompute(); + } + using AgentHistogramT = AgentHistogram; - // Shared memory for AgentHistogram __shared__ typename AgentHistogramT::TempStorage static_smem; + extern __shared__ __align__(16) unsigned char dynamic_smem[]; + + CounterT* dynamic_smem_privatized_histograms = nullptr; + if constexpr (is_privatized_dynamic_smem_v) + { + dynamic_smem_privatized_histograms = reinterpret_cast(dynamic_smem); + } AgentHistogramT agent( static_smem, @@ -1007,7 +983,7 @@ __launch_bounds__(int( d_privatized_histograms_wrapper.data(), output_decode_op, privatized_decode_op, - nullptr); + dynamic_smem_privatized_histograms); // Initialize counters agent.InitBinCounters(); diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 36e37dec9421..97e6ac2ab483 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -71,7 +71,7 @@ struct HistogramPolicy int max_privatized_dynamic_smem_2_channel_even_bytes; //!< Two-channel HistogramEven SMEM limit int max_privatized_dynamic_smem_3_channel_even_bytes; //!< Three-channel HistogramEven SMEM limit int max_privatized_dynamic_smem_4_channel_even_bytes; //!< Four-channel HistogramEven SMEM limit - int max_num_bins_for_init_kernel_pdl_trigger; //!< Largest output histogram for which the init kernel triggers PDL + int max_output_histogram_bytes_for_init_kernel_pdl_trigger; //!< Largest output allocation for init-kernel PDL [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(const HistogramPolicy& lhs, const HistogramPolicy& rhs) noexcept @@ -85,7 +85,8 @@ struct HistogramPolicy && lhs.max_privatized_dynamic_smem_2_channel_even_bytes == rhs.max_privatized_dynamic_smem_2_channel_even_bytes && lhs.max_privatized_dynamic_smem_3_channel_even_bytes == rhs.max_privatized_dynamic_smem_3_channel_even_bytes && lhs.max_privatized_dynamic_smem_4_channel_even_bytes == rhs.max_privatized_dynamic_smem_4_channel_even_bytes - && lhs.max_num_bins_for_init_kernel_pdl_trigger == rhs.max_num_bins_for_init_kernel_pdl_trigger; + && lhs.max_output_histogram_bytes_for_init_kernel_pdl_trigger + == rhs.max_output_histogram_bytes_for_init_kernel_pdl_trigger; } [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool @@ -110,7 +111,8 @@ struct HistogramPolicy << p.max_privatized_dynamic_smem_3_channel_even_bytes << ", .max_privatized_dynamic_smem_4_channel_even_bytes = " << p.max_privatized_dynamic_smem_4_channel_even_bytes - << ", .max_num_bins_for_init_kernel_pdl_trigger = " << p.max_num_bins_for_init_kernel_pdl_trigger << " }"; + << ", .max_output_histogram_bytes_for_init_kernel_pdl_trigger = " + << p.max_output_histogram_bytes_for_init_kernel_pdl_trigger << " }"; } #endif }; @@ -171,13 +173,15 @@ select_privatization_mode(const HistogramPolicy& policy, int num_bins) -> privat max_privatized_smem_bins(policy.max_privatized_static_smem_single_channel_bytes); const int dynamic_smem_max_bytes = dynamic_smem_limit_bytes(policy); const int dynamic_smem_max_bins = max_privatized_smem_bins(dynamic_smem_max_bytes); - const bool static_smem_fits = num_bins <= static_smem_max_bins; - const bool prefer_dynamic_smem = sizeof(CounterT) > sizeof(::cuda::std::uint32_t) || !static_smem_fits; - if (prefer_dynamic_smem && num_bins <= dynamic_smem_max_bins) + if (num_bins <= static_smem_max_bins) + { + return privatization_mode::static_smem; + } + if (num_bins <= dynamic_smem_max_bins) { return privatization_mode::dynamic_smem; } - return static_smem_fits ? privatization_mode::static_smem : privatization_mode::gmem; + return privatization_mode::gmem; } // The C Parallel API erases CounterT before host dispatch, so its bridge must select from the @@ -195,13 +199,15 @@ select_privatization_mode_for_counter_size(const HistogramPolicy& policy, int nu const int static_smem_max_bins = policy.max_privatized_static_smem_single_channel_bytes / counter_size_bytes; const int dynamic_smem_max_bins = dynamic_smem_limit_bytes(policy) / counter_size_bytes / NumActiveChannels; - const bool static_smem_fits = num_bins <= static_smem_max_bins; - const bool prefer_dynamic_smem = counter_size_bytes > int{sizeof(::cuda::std::uint32_t)} || !static_smem_fits; - if (prefer_dynamic_smem && num_bins <= dynamic_smem_max_bins) + if (num_bins <= static_smem_max_bins) + { + return privatization_mode::static_smem; + } + if (num_bins <= dynamic_smem_max_bins) { return privatization_mode::dynamic_smem; } - return static_smem_fits ? privatization_mode::static_smem : privatization_mode::gmem; + return privatization_mode::gmem; } #if _CCCL_HAS_CONCEPTS() @@ -293,36 +299,39 @@ public: static_smem.items_per_thread = t_scale(16); } + // All storage thresholds are byte budgets. Dispatch derives the corresponding + // bin limits from the local counter width and active channel count. + constexpr int max_privatized_static_smem_bytes = 1024; + constexpr int max_privatized_dynamic_smem_single_channel_bytes = 228352; + constexpr int max_privatized_dynamic_smem_range_bytes_per_channel = 8192; + constexpr int max_privatized_dynamic_smem_2_channel_even_bytes = 228352; + constexpr int max_privatized_dynamic_smem_3_channel_even_bytes = 228348; + constexpr int max_privatized_dynamic_smem_4_channel_even_bytes = 131072; + constexpr int max_output_histogram_bytes_for_init_kernel_pdl_trigger = 8192; + // Dynamic-SMEM tuning exists only for the type and channel combinations measured by autoresearch. const bool has_dynamic_smem_tuning = counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive && ((single_channel && (sample_size_bytes == 1 || sample_size_bytes == 4 || sample_size_bytes == 8)) || num_channels > 1); - // B200 provides 232448 bytes of opt-in shared memory. Reserve 4096 bytes - // for the kernel's statically allocated shared-memory state. - const int max_privatized_dynamic_smem_single_channel_bytes = has_dynamic_smem_tuning ? 232448 - 4096 : 0; - const int max_num_bins_for_init_kernel_pdl_trigger = + const int init_kernel_pdl_trigger_bytes = single_channel && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive && (sample_size_bytes == 1 || sample_size_bytes == 2 || sample_size_bytes == 4 || sample_size_bytes == 8) - ? 2048 + ? max_output_histogram_bytes_for_init_kernel_pdl_trigger : 0; - // The larger compile-time-sized histogram regresses the tuned RANGE kernels even when the runtime histogram - // has at most 256 bins. Keep their static allocation at 256 bins and use the runtime-sized tier above it. - const int max_privatized_static_smem_single_channel_bytes = - (range_multi_static || range_u64_static ? 256 : 512) * counter_size_bytes; return HistogramPolicy{ gmem, static_smem, gmem, - max_privatized_static_smem_single_channel_bytes, - max_privatized_dynamic_smem_single_channel_bytes, + max_privatized_static_smem_bytes, + has_dynamic_smem_tuning ? max_privatized_dynamic_smem_single_channel_bytes : 0, range_multi_static || range_u64_static ? 3 : 0, - has_dynamic_smem_tuning ? 2048 * counter_size_bytes * num_active_channels : 0, - has_dynamic_smem_tuning ? 28544 * counter_size_bytes * 2 : 0, - has_dynamic_smem_tuning ? 19029 * counter_size_bytes * 3 : 0, - has_dynamic_smem_tuning ? 8192 * counter_size_bytes * 4 : 0, - max_num_bins_for_init_kernel_pdl_trigger}; + has_dynamic_smem_tuning ? max_privatized_dynamic_smem_range_bytes_per_channel * num_active_channels : 0, + has_dynamic_smem_tuning ? max_privatized_dynamic_smem_2_channel_even_bytes : 0, + has_dynamic_smem_tuning ? max_privatized_dynamic_smem_3_channel_even_bytes : 0, + has_dynamic_smem_tuning ? max_privatized_dynamic_smem_4_channel_even_bytes : 0, + init_kernel_pdl_trigger_bytes}; } // SM90 uses its established single-channel 8-bit and 16-bit specializations. @@ -344,18 +353,20 @@ public: sweep = HistogramPrivatizationPolicy{960, 10, 4, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, false}; } } - const int max_num_bins_for_init_kernel_pdl_trigger = + constexpr int max_privatized_static_smem_bytes = 1024; + constexpr int max_output_histogram_bytes_for_init_kernel_pdl_trigger = 8192; + const int init_kernel_pdl_trigger_bytes = num_channels == 1 && num_active_channels == 1 && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive && (sample_size_bytes == 1 || sample_size_bytes == 2) - ? 2048 + ? max_output_histogram_bytes_for_init_kernel_pdl_trigger : 0; return HistogramPolicy{ - sweep, sweep, sweep, 256 * counter_size_bytes, 0, 0, 0, 0, 0, 0, max_num_bins_for_init_kernel_pdl_trigger}; + sweep, sweep, sweep, max_privatized_static_smem_bytes, 0, 0, 0, 0, 0, 0, init_kernel_pdl_trigger_bytes}; } // Architectures before SM90 use the longstanding generic histogram tuning. const auto sweep = HistogramPrivatizationPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; - return HistogramPolicy{sweep, sweep, sweep, 256 * counter_size_bytes, 0, 0, 0, 0, 0, 0, 0}; + return HistogramPolicy{sweep, sweep, sweep, 1024, 0, 0, 0, 0, 0, 0, 0}; } }; diff --git a/cub/test/catch2_test_device_histogram_custom_policy_hub.cu b/cub/test/catch2_test_device_histogram_custom_policy_hub.cu index f1852f75b997..ac16744e8850 100644 --- a/cub/test/catch2_test_device_histogram_custom_policy_hub.cu +++ b/cub/test/catch2_test_device_histogram_custom_policy_hub.cu @@ -48,8 +48,8 @@ CUB_TEST("DispatchHistogram::DispatchEven: custom policy hub", "[histogram][devi REQUIRE(custom_sm90_policy.gmem.items_per_thread == 8); REQUIRE(custom_sm75_policy.max_privatized_static_smem_single_channel_bytes == 256 * sizeof(unsigned int)); REQUIRE(custom_sm90_policy.max_privatized_dynamic_smem_single_channel_bytes == 0); - REQUIRE(custom_sm75_policy.max_num_bins_for_init_kernel_pdl_trigger == 0); - REQUIRE(custom_sm90_policy.max_num_bins_for_init_kernel_pdl_trigger == 2048); + REQUIRE(custom_sm75_policy.max_output_histogram_bytes_for_init_kernel_pdl_trigger == 0); + REQUIRE(custom_sm90_policy.max_output_histogram_bytes_for_init_kernel_pdl_trigger == 8192); using sample_t = cuda::std::uint8_t; using counter_t = int; diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index 14fc86138b3b..432676357c03 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1832,35 +1832,35 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) # if _CCCL_STD_VER >= 2020 // designated init constexpr auto p2 = cub::HistogramPolicy{ - .gmem = {.threads_per_block = 128, - .items_per_thread = 7, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .static_smem = {.threads_per_block = 96, - .items_per_thread = 3, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .dynamic_smem = {.threads_per_block = 128, - .items_per_thread = 7, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .max_privatized_static_smem_single_channel_bytes = 2052, - .max_privatized_dynamic_smem_single_channel_bytes = 12345, - .static_smem_min_blocks_per_sm = 2, - .max_privatized_dynamic_smem_multi_channel_range_bytes = 1024, - .max_privatized_dynamic_smem_2_channel_even_bytes = 4096, - .max_privatized_dynamic_smem_3_channel_even_bytes = 8192, - .max_privatized_dynamic_smem_4_channel_even_bytes = 16384, - .max_num_bins_for_init_kernel_pdl_trigger = 2048}; + .gmem = {.threads_per_block = 128, + .items_per_thread = 7, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .static_smem = {.threads_per_block = 96, + .items_per_thread = 3, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .dynamic_smem = {.threads_per_block = 128, + .items_per_thread = 7, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .max_privatized_static_smem_single_channel_bytes = 2052, + .max_privatized_dynamic_smem_single_channel_bytes = 12345, + .static_smem_min_blocks_per_sm = 2, + .max_privatized_dynamic_smem_multi_channel_range_bytes = 1024, + .max_privatized_dynamic_smem_2_channel_even_bytes = 4096, + .max_privatized_dynamic_smem_3_channel_even_bytes = 8192, + .max_privatized_dynamic_smem_4_channel_even_bytes = 16384, + .max_output_histogram_bytes_for_init_kernel_pdl_trigger = 2048}; # else // _CCCL_STD_VER >= 2020 constexpr auto p2 = p1; # endif // _CCCL_STD_VER >= 2020 @@ -1892,17 +1892,17 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" == 256); STATIC_REQUIRE(cub::detail::histogram::max_privatized_smem_bins( sm100_policy.max_privatized_static_smem_single_channel_bytes) - == 512); + == 256); STATIC_REQUIRE(sm90_policy.max_privatized_dynamic_smem_single_channel_bytes == 0); STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_single_channel_bytes == 228352); STATIC_REQUIRE(sm100_wide_counter_policy.max_privatized_dynamic_smem_single_channel_bytes == 0); STATIC_REQUIRE(cub::detail::histogram::max_privatized_smem_bins( sm100_policy.max_privatized_dynamic_smem_single_channel_bytes) == 57088); - STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_multi_channel_range_bytes == 2048 * 4 * 1); - STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_2_channel_even_bytes == 28544 * 4 * 2); - STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_3_channel_even_bytes == 19029 * 4 * 3); - STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_4_channel_even_bytes == 8192 * 4 * 4); + STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_multi_channel_range_bytes == 8192); + STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_2_channel_even_bytes == 228352); + STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_3_channel_even_bytes == 228348); + STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_4_channel_even_bytes == 131072); STATIC_REQUIRE(sm100_policy.gmem.threads_per_block == 768); STATIC_REQUIRE(sm100_policy.gmem.items_per_thread == 12); STATIC_REQUIRE(sm100_policy.static_smem == sm100_policy.gmem); @@ -1941,9 +1941,9 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" == 256); using cub::detail::histogram::privatization_mode; - STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_policy, 512) + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_policy, 256) == privatization_mode::static_smem); - STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_policy, 513) + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_policy, 257) == privatization_mode::dynamic_smem); STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_policy, 57088) == privatization_mode::dynamic_smem); @@ -1977,9 +1977,15 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" == privatization_mode::dynamic_smem); STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_3ch_policy, 19030) == privatization_mode::gmem); + STATIC_REQUIRE( + cub::detail::histogram::select_privatization_mode(sm100_wide_counter_policy, 128) + == privatization_mode::static_smem); + STATIC_REQUIRE( + cub::detail::histogram::select_privatization_mode(sm100_wide_counter_policy, 129) + == privatization_mode::gmem); STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode_for_counter_size( - sm100_policy, 256, sizeof(cuda::std::uint64_t)) - == privatization_mode::dynamic_smem); + sm100_policy, 128, sizeof(cuda::std::uint64_t)) + == privatization_mode::static_smem); STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode_for_counter_size( sm100_policy, 28545, sizeof(cuda::std::uint64_t)) == privatization_mode::gmem); diff --git a/cub/test/catch2_test_device_histogram_env_api.cu b/cub/test/catch2_test_device_histogram_env_api.cu index becd885a5720..bd1af9bf3e86 100644 --- a/cub/test/catch2_test_device_histogram_env_api.cu +++ b/cub/test/catch2_test_device_histogram_env_api.cu @@ -421,17 +421,17 @@ struct HistogramPolicySelector const auto sweep = cub::HistogramPrivatizationPolicy{ 128, cc > cuda::compute_capability{9, 0} ? 16 : 7, 4, cub::BLOCK_LOAD_DIRECT, cub::LOAD_LDG, false, false}; return { - .gmem = sweep, - .static_smem = sweep, - .dynamic_smem = sweep, - .max_privatized_static_smem_single_channel_bytes = 256 * sizeof(unsigned int), - .max_privatized_dynamic_smem_single_channel_bytes = 0, - .static_smem_min_blocks_per_sm = 0, - .max_privatized_dynamic_smem_multi_channel_range_bytes = 0, - .max_privatized_dynamic_smem_2_channel_even_bytes = 0, - .max_privatized_dynamic_smem_3_channel_even_bytes = 0, - .max_privatized_dynamic_smem_4_channel_even_bytes = 0, - .max_num_bins_for_init_kernel_pdl_trigger = 2048}; + .gmem = sweep, + .static_smem = sweep, + .dynamic_smem = sweep, + .max_privatized_static_smem_single_channel_bytes = 256 * sizeof(unsigned int), + .max_privatized_dynamic_smem_single_channel_bytes = 0, + .static_smem_min_blocks_per_sm = 0, + .max_privatized_dynamic_smem_multi_channel_range_bytes = 0, + .max_privatized_dynamic_smem_2_channel_even_bytes = 0, + .max_privatized_dynamic_smem_3_channel_even_bytes = 0, + .max_privatized_dynamic_smem_4_channel_even_bytes = 0, + .max_output_histogram_bytes_for_init_kernel_pdl_trigger = 8192}; } }; // example-end histogram-even-policy-selector From b4699d6e73c188b634ed7b19df395c73ef876869 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sat, 15 Aug 2026 19:10:27 +0000 Subject: [PATCH 38/45] [cub] Preserve current multi-channel histogram tuning --- .../device/dispatch/tuning/tuning_histogram.cuh | 13 ++++--------- cub/test/catch2_test_device_histogram_env.cu | 16 ++++++---------- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 97e6ac2ab483..49013c62246f 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -242,14 +242,8 @@ public: const bool single_channel = num_channels == 1 && num_active_channels == 1; auto gmem = HistogramPrivatizationPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; - // Multi-channel 32-bit-counter histograms use the wider SM100 sweep tuned by autoresearch. - if (num_channels > 1 && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive) - { - gmem = - HistogramPrivatizationPolicy{1024, t_scale(is_even ? 8 : 16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; - } // Single-channel primitive samples with 32-bit counters use their per-sample-width tuning. - else if (single_channel && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive) + if (single_channel && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive) { // Eight-bit EVEN and RANGE histograms retain the dedicated SM100 tunings already in main. if (sample_size_bytes == 1) @@ -309,11 +303,12 @@ public: constexpr int max_privatized_dynamic_smem_4_channel_even_bytes = 131072; constexpr int max_output_histogram_bytes_for_init_kernel_pdl_trigger = 8192; - // Dynamic-SMEM tuning exists only for the type and channel combinations measured by autoresearch. + // Dynamic-SMEM tuning exists only for combinations that improve on current main. + // Multi-channel EVEN keeps the established global-memory path above the static tier. const bool has_dynamic_smem_tuning = counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive && ((single_channel && (sample_size_bytes == 1 || sample_size_bytes == 4 || sample_size_bytes == 8)) - || num_channels > 1); + || (!is_even && num_channels > 1)); const int init_kernel_pdl_trigger_bytes = single_channel && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive && (sample_size_bytes == 1 || sample_size_bytes == 2 || sample_size_bytes == 4 || sample_size_bytes == 8) diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index 432676357c03..f7c7a7100404 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1931,7 +1931,7 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" constexpr auto sm100_even_4ch_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); - STATIC_REQUIRE(sm100_multi_range_policy.gmem.threads_per_block == 1024); + STATIC_REQUIRE(sm100_multi_range_policy.gmem.threads_per_block == 384); STATIC_REQUIRE(sm100_multi_range_policy.gmem.items_per_thread == 5); STATIC_REQUIRE(sm100_multi_range_policy.static_smem.threads_per_block == 384); STATIC_REQUIRE(sm100_multi_range_policy.static_smem.items_per_thread == 5); @@ -1965,17 +1965,13 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE( cub::detail::histogram::select_privatization_mode(sm100_multi_range_policy, 2049) == privatization_mode::gmem); - STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_4ch_policy, 8192) - == privatization_mode::dynamic_smem); - STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_4ch_policy, 8193) + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_4ch_policy, 256) + == privatization_mode::static_smem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_4ch_policy, 257) == privatization_mode::gmem); - STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_2ch_policy, 28544) - == privatization_mode::dynamic_smem); - STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_2ch_policy, 28545) + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_2ch_policy, 257) == privatization_mode::gmem); - STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_3ch_policy, 19029) - == privatization_mode::dynamic_smem); - STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_3ch_policy, 19030) + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_3ch_policy, 257) == privatization_mode::gmem); STATIC_REQUIRE( cub::detail::histogram::select_privatization_mode(sm100_wide_counter_policy, 128) From 260abea4f88f6b01a51fce8817d7856190a00961 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sat, 15 Aug 2026 19:38:36 +0000 Subject: [PATCH 39/45] [cub] Preserve legacy histogram policy capacity --- .../device/dispatch/dispatch_histogram.cuh | 31 ++++++++++++------- .../dispatch/tuning/tuning_histogram.cuh | 12 +++---- ...test_device_histogram_custom_policy_hub.cu | 14 +++++++-- cub/test/catch2_test_device_histogram_env.cu | 6 ++-- 4 files changed, 41 insertions(+), 22 deletions(-) diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 505c63c496c0..f6cc37236cf2 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -369,7 +369,6 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( num_privatized_levels.begin(), num_privatized_levels.end(), num_privatized_bins_wrapper.begin(), minus_one); ::cuda::std::transform(num_output_levels.begin(), num_output_levels.end(), num_output_bins_wrapper.begin(), minus_one); - constexpr int histogram_init_threads_per_block = 256; int histogram_init_grid_dims = (max_num_output_bins + histogram_init_threads_per_block - 1) / histogram_init_threads_per_block; @@ -845,22 +844,22 @@ CUB_RUNTIME_FUNCTION _CCCL_FORCEINLINE static cudaError_t __dispatch_even_device } // TODO(bgruber): drop in CCCL 4.0 -template +template _CCCL_HOST_DEVICE_API constexpr auto convert_pdl_trigger_bytes(int) -> decltype(ActivePolicy::init_kernel_pdl_trigger_max_bins) { - return ActivePolicy::init_kernel_pdl_trigger_max_bins * int{sizeof(unsigned int)}; + return ActivePolicy::init_kernel_pdl_trigger_max_bins * int{sizeof(CounterT)}; } // TODO(bgruber): drop in CCCL 4.0 -template +template _CCCL_HOST_DEVICE_API constexpr auto convert_pdl_trigger_bytes(long) { return 0; } // TODO(bgruber): drop in CCCL 4.0 -template +template _CCCL_HOST_DEVICE_API constexpr auto convert_legacy_policy() -> HistogramPolicy { using sweep = typename ActivePolicy::AgentHistogramPolicyT; @@ -873,11 +872,21 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_legacy_policy() -> HistogramPolicy sweep::IS_RLE_COMPRESS, sweep::IS_WORK_STEALING}; return { - kernel_config, kernel_config, kernel_config, 1024, 0, 0, 0, 0, 0, 0, convert_pdl_trigger_bytes(0)}; + kernel_config, + kernel_config, + kernel_config, + legacy_privatized_static_smem_bins * int{sizeof(CounterT)}, + 0, + 0, + 0, + 0, + 0, + 0, + convert_pdl_trigger_bytes(0)}; } // TODO(bgruber): drop in CCCL 4.0 -template +template struct policy_selector_from_hub { private: @@ -888,7 +897,7 @@ private: template _CCCL_HOST_DEVICE_API constexpr cudaError_t Invoke() { - policy = convert_legacy_policy(); + policy = convert_legacy_policy(); return cudaSuccess; } }; @@ -903,7 +912,7 @@ public: _CCCL_VERIFY(MaxPolicy::Invoke(cc.get() * 10, dispatch) == cudaSuccess, ""); return policy; }), - ({ return convert_legacy_policy(); })); + ({ return convert_legacy_policy(); })); } }; @@ -1554,7 +1563,7 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc using policy_selector_t = ::cuda::std::_If< uses_default_policy, detail::histogram::policy_selector_from_types, - detail::histogram::policy_selector_from_hub>; + detail::histogram::policy_selector_from_hub>; return detail::histogram::dispatch_range( d_temp_storage, temp_storage_bytes, @@ -1642,7 +1651,7 @@ struct CCCL_DEPRECATED_BECAUSE("Use the tuning API for DeviceHistogram") Dispatc using policy_selector_t = ::cuda::std::_If< uses_default_policy, detail::histogram::policy_selector_from_types, - detail::histogram::policy_selector_from_hub>; + detail::histogram::policy_selector_from_hub>; return detail::histogram::dispatch_even( d_temp_storage, temp_storage_bytes, diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 49013c62246f..0bc84627efc1 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -119,6 +119,9 @@ struct HistogramPolicy namespace detail::histogram { +inline constexpr int histogram_init_threads_per_block = 256; +inline constexpr int legacy_privatized_static_smem_bins = 256; + enum class privatization_mode { gmem, @@ -298,9 +301,6 @@ public: constexpr int max_privatized_static_smem_bytes = 1024; constexpr int max_privatized_dynamic_smem_single_channel_bytes = 228352; constexpr int max_privatized_dynamic_smem_range_bytes_per_channel = 8192; - constexpr int max_privatized_dynamic_smem_2_channel_even_bytes = 228352; - constexpr int max_privatized_dynamic_smem_3_channel_even_bytes = 228348; - constexpr int max_privatized_dynamic_smem_4_channel_even_bytes = 131072; constexpr int max_output_histogram_bytes_for_init_kernel_pdl_trigger = 8192; // Dynamic-SMEM tuning exists only for combinations that improve on current main. @@ -323,9 +323,9 @@ public: has_dynamic_smem_tuning ? max_privatized_dynamic_smem_single_channel_bytes : 0, range_multi_static || range_u64_static ? 3 : 0, has_dynamic_smem_tuning ? max_privatized_dynamic_smem_range_bytes_per_channel * num_active_channels : 0, - has_dynamic_smem_tuning ? max_privatized_dynamic_smem_2_channel_even_bytes : 0, - has_dynamic_smem_tuning ? max_privatized_dynamic_smem_3_channel_even_bytes : 0, - has_dynamic_smem_tuning ? max_privatized_dynamic_smem_4_channel_even_bytes : 0, + 0, + 0, + 0, init_kernel_pdl_trigger_bytes}; } diff --git a/cub/test/catch2_test_device_histogram_custom_policy_hub.cu b/cub/test/catch2_test_device_histogram_custom_policy_hub.cu index ac16744e8850..4d88ad904fb4 100644 --- a/cub/test/catch2_test_device_histogram_custom_policy_hub.cu +++ b/cub/test/catch2_test_device_histogram_custom_policy_hub.cu @@ -39,17 +39,27 @@ CUB_TEST("DispatchHistogram::DispatchEven: custom policy hub", "[histogram][devi { using custom_max_policy_t = typename my_policy_hub::MaxPolicy; const auto custom_sm75_policy = - cub::detail::histogram::policy_selector_from_hub{}(cuda::compute_capability{7, 5}); + cub::detail::histogram::policy_selector_from_hub{}(cuda::compute_capability{7, 5}); const auto custom_sm90_policy = - cub::detail::histogram::policy_selector_from_hub{}(cuda::compute_capability{9, 0}); + cub::detail::histogram::policy_selector_from_hub{}(cuda::compute_capability{9, 0}); + const auto custom_wide_counter_policy = + cub::detail::histogram::policy_selector_from_hub{}( + cuda::compute_capability{7, 5}); + const auto custom_wide_counter_sm90_policy = + cub::detail::histogram::policy_selector_from_hub{}( + cuda::compute_capability{9, 0}); REQUIRE(custom_sm75_policy.gmem.threads_per_block == 384); REQUIRE(custom_sm75_policy.gmem.items_per_thread == 16); REQUIRE(custom_sm90_policy.gmem.threads_per_block == 256); REQUIRE(custom_sm90_policy.gmem.items_per_thread == 8); REQUIRE(custom_sm75_policy.max_privatized_static_smem_single_channel_bytes == 256 * sizeof(unsigned int)); + REQUIRE( + custom_wide_counter_policy.max_privatized_static_smem_single_channel_bytes == 256 * sizeof(unsigned long long)); REQUIRE(custom_sm90_policy.max_privatized_dynamic_smem_single_channel_bytes == 0); REQUIRE(custom_sm75_policy.max_output_histogram_bytes_for_init_kernel_pdl_trigger == 0); REQUIRE(custom_sm90_policy.max_output_histogram_bytes_for_init_kernel_pdl_trigger == 8192); + REQUIRE(custom_wide_counter_policy.max_output_histogram_bytes_for_init_kernel_pdl_trigger == 0); + REQUIRE(custom_wide_counter_sm90_policy.max_output_histogram_bytes_for_init_kernel_pdl_trigger == 16384); using sample_t = cuda::std::uint8_t; using counter_t = int; diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index f7c7a7100404..3a981a493fd8 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1900,9 +1900,6 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" sm100_policy.max_privatized_dynamic_smem_single_channel_bytes) == 57088); STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_multi_channel_range_bytes == 8192); - STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_2_channel_even_bytes == 228352); - STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_3_channel_even_bytes == 228348); - STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_4_channel_even_bytes == 131072); STATIC_REQUIRE(sm100_policy.gmem.threads_per_block == 768); STATIC_REQUIRE(sm100_policy.gmem.items_per_thread == 12); STATIC_REQUIRE(sm100_policy.static_smem == sm100_policy.gmem); @@ -1931,6 +1928,9 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" constexpr auto sm100_even_4ch_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); + STATIC_REQUIRE(sm100_even_2ch_policy.max_privatized_dynamic_smem_2_channel_even_bytes == 0); + STATIC_REQUIRE(sm100_even_3ch_policy.max_privatized_dynamic_smem_3_channel_even_bytes == 0); + STATIC_REQUIRE(sm100_even_4ch_policy.max_privatized_dynamic_smem_4_channel_even_bytes == 0); STATIC_REQUIRE(sm100_multi_range_policy.gmem.threads_per_block == 384); STATIC_REQUIRE(sm100_multi_range_policy.gmem.items_per_thread == 5); STATIC_REQUIRE(sm100_multi_range_policy.static_smem.threads_per_block == 384); From af286f01808ea10ceffd391e4ff601cd701126d7 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sat, 15 Aug 2026 23:54:42 +0000 Subject: [PATCH 40/45] [cub] Clarify histogram autotuning policy --- cub/benchmarks/bench/histogram/even.cu | 2 +- cub/benchmarks/bench/histogram/histogram_common.cuh | 9 ++++++++- cub/benchmarks/bench/histogram/multi/even.cu | 3 ++- cub/benchmarks/bench/histogram/multi/range.cu | 3 ++- cub/benchmarks/bench/histogram/range.cu | 2 +- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/cub/benchmarks/bench/histogram/even.cu b/cub/benchmarks/bench/histogram/even.cu index 79abc98cebf4..457e58c1750e 100644 --- a/cub/benchmarks/bench/histogram/even.cu +++ b/cub/benchmarks/bench/histogram/even.cu @@ -51,7 +51,7 @@ static void even(nvbench::state& state, nvbench::type_list{}) + cuda::execution::tune(histogram_tuning_policy_selector{}) #endif // !TUNE_BASE ); _CCCL_TRY_CUDA_API( diff --git a/cub/benchmarks/bench/histogram/histogram_common.cuh b/cub/benchmarks/bench/histogram/histogram_common.cuh index f8d735eb672a..7e0735c12156 100644 --- a/cub/benchmarks/bench/histogram/histogram_common.cuh +++ b/cub/benchmarks/bench/histogram/histogram_common.cuh @@ -27,8 +27,15 @@ # define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_STRIPED # endif // TUNE_LOAD_ALGORITHM_ID +// Only generated tuning variants instantiate this selector. The `.base` target used for +// production-policy comparisons defines TUNE_BASE=1 and calls DeviceHistogram without a +// tuning environment, so it exercises the shipping selector unchanged. +// +// A generated tuning point supplies one candidate kernel configuration. Apply that same +// candidate to every privatization mode so the tuner measures the candidate independently +// of the runtime bin count selected by the production storage thresholds. template -struct bench_policy_selector +struct histogram_tuning_policy_selector { _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> cub::HistogramPolicy { diff --git a/cub/benchmarks/bench/histogram/multi/even.cu b/cub/benchmarks/bench/histogram/multi/even.cu index 2d5df84932ac..4bb60a72c26c 100644 --- a/cub/benchmarks/bench/histogram/multi/even.cu +++ b/cub/benchmarks/bench/histogram/multi/even.cu @@ -64,7 +64,8 @@ static void even(nvbench::state& state, nvbench::type_list{}) + cuda::execution::tune( + histogram_tuning_policy_selector{}) #endif // !TUNE_BASE ); _CCCL_TRY_CUDA_API( diff --git a/cub/benchmarks/bench/histogram/multi/range.cu b/cub/benchmarks/bench/histogram/multi/range.cu index 9c16e2d676ee..7be2518dfca8 100644 --- a/cub/benchmarks/bench/histogram/multi/range.cu +++ b/cub/benchmarks/bench/histogram/multi/range.cu @@ -64,7 +64,8 @@ static void range(nvbench::state& state, nvbench::type_list{}) + cuda::execution::tune( + histogram_tuning_policy_selector{}) #endif // !TUNE_BASE ); _CCCL_TRY_CUDA_API( diff --git a/cub/benchmarks/bench/histogram/range.cu b/cub/benchmarks/bench/histogram/range.cu index cd9cbb4f8e3d..938fa11dc4c6 100644 --- a/cub/benchmarks/bench/histogram/range.cu +++ b/cub/benchmarks/bench/histogram/range.cu @@ -50,7 +50,7 @@ static void range(nvbench::state& state, nvbench::type_list{}) + cuda::execution::tune(histogram_tuning_policy_selector{}) #endif // !TUNE_BASE ); _CCCL_TRY_CUDA_API( From 0e652607a41ea960a18e84fb5e5386ddf0a6cafb Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Mon, 17 Aug 2026 02:09:40 +0000 Subject: [PATCH 41/45] Use dynamic SMEM for multi-channel EVEN histograms --- .../device/dispatch/dispatch_histogram.cuh | 51 ++++++-- .../dispatch/tuning/tuning_histogram.cuh | 39 ++++-- cub/test/catch2_test_device_histogram.cu | 3 +- cub/test/catch2_test_device_histogram_env.cu | 119 ++++++++++++++++-- 4 files changed, 186 insertions(+), 26 deletions(-) diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index f6cc37236cf2..7d6fffded1ea 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -271,13 +271,50 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( { dynamic_smem_bytes += (num_privatized_levels[channel] - 1) * static_cast(kernel_source.CounterSize()); } - NV_IF_TARGET(NV_IS_HOST, ({ - if (const auto error = CubDebug(launcher_factory.set_max_dynamic_smem_size_for( - sweep_kernel, dynamic_smem_limit_bytes(active_policy)))) - { - return error; - } - })) + NV_IF_ELSE_TARGET( + NV_IS_HOST, + ({ + if (const auto error = CubDebug(launcher_factory.set_max_dynamic_smem_size_for( + sweep_kernel, dynamic_smem_limit_bytes(active_policy)))) + { + return error; + } + }), + ({ + int max_shared_smem_bytes{}; + if (const auto error = CubDebug(launcher_factory.MaxSharedMemory(max_shared_smem_bytes))) + { + return error; + } + ::cudaFuncAttributes sweep_kernel_attributes{}; + if (const auto error = CubDebug(::cudaFuncGetAttributes(&sweep_kernel_attributes, sweep_kernel))) + { + return error; + } + const int max_dynamic_smem_bytes = + max_shared_smem_bytes - static_cast(sweep_kernel_attributes.sharedSizeBytes); + if (dynamic_smem_bytes > max_dynamic_smem_bytes) + { + return detail::histogram:: + dispatch( + d_temp_storage, + temp_storage_bytes, + d_samples, + d_output_histograms, + num_privatized_levels, + num_output_levels, + first_level_array, + second_level_array, + max_num_output_bins, + num_row_pixels, + num_rows, + row_stride_samples, + stream, + policy_selector, + kernel_source, + launcher_factory); + } + })) } // Get SM count diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 0bc84627efc1..e3d710e916e4 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -301,14 +301,25 @@ public: constexpr int max_privatized_static_smem_bytes = 1024; constexpr int max_privatized_dynamic_smem_single_channel_bytes = 228352; constexpr int max_privatized_dynamic_smem_range_bytes_per_channel = 8192; + constexpr int max_privatized_dynamic_smem_even_bytes_per_channel = 32768; constexpr int max_output_histogram_bytes_for_init_kernel_pdl_trigger = 8192; - // Dynamic-SMEM tuning exists only for combinations that improve on current main. - // Multi-channel EVEN keeps the established global-memory path above the static tier. - const bool has_dynamic_smem_tuning = - counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive - && ((single_channel && (sample_size_bytes == 1 || sample_size_bytes == 4 || sample_size_bytes == 8)) - || (!is_even && num_channels > 1)); + const bool supports_dynamic_smem = + counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive; + const bool has_single_channel_dynamic_smem = + supports_dynamic_smem && single_channel + && (sample_size_bytes == 1 || sample_size_bytes == 4 || sample_size_bytes == 8); + const bool has_multi_channel_dynamic_smem = supports_dynamic_smem && num_channels > 1; + int dynamic_smem_single_channel_bytes = + has_single_channel_dynamic_smem ? max_privatized_dynamic_smem_single_channel_bytes : 0; + int dynamic_smem_multi_channel_range_bytes = + has_single_channel_dynamic_smem || (has_multi_channel_dynamic_smem && !is_even) + ? max_privatized_dynamic_smem_range_bytes_per_channel * num_active_channels + : 0; + int dynamic_smem_multi_channel_even_bytes = + has_multi_channel_dynamic_smem && is_even + ? max_privatized_dynamic_smem_even_bytes_per_channel * num_active_channels + : 0; const int init_kernel_pdl_trigger_bytes = single_channel && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive && (sample_size_bytes == 1 || sample_size_bytes == 2 || sample_size_bytes == 4 || sample_size_bytes == 8) @@ -320,12 +331,18 @@ public: static_smem, gmem, max_privatized_static_smem_bytes, - has_dynamic_smem_tuning ? max_privatized_dynamic_smem_single_channel_bytes : 0, + dynamic_smem_single_channel_bytes, range_multi_static || range_u64_static ? 3 : 0, - has_dynamic_smem_tuning ? max_privatized_dynamic_smem_range_bytes_per_channel * num_active_channels : 0, - 0, - 0, - 0, + dynamic_smem_multi_channel_range_bytes, + has_multi_channel_dynamic_smem && is_even && num_active_channels == 2 + ? dynamic_smem_multi_channel_even_bytes + : 0, + has_multi_channel_dynamic_smem && is_even && num_active_channels == 3 + ? dynamic_smem_multi_channel_even_bytes + : 0, + has_multi_channel_dynamic_smem && is_even && num_active_channels == 4 + ? dynamic_smem_multi_channel_even_bytes + : 0, init_kernel_pdl_trigger_bytes}; } diff --git a/cub/test/catch2_test_device_histogram.cu b/cub/test/catch2_test_device_histogram.cu index cdee59980c4d..86e6e335e770 100644 --- a/cub/test/catch2_test_device_histogram.cu +++ b/cub/test/catch2_test_device_histogram.cu @@ -587,9 +587,10 @@ CUB_TEST("DeviceHistogram::Histogram* dynamic shared-memory privatization", "[hi } using counter_t = unsigned int; - const int num_levels = GENERATE(1025, 4097); + const int num_levels = GENERATE(1025, 4097, 8193); test_even_and_range(num_levels - 1, num_levels, 4096, 4); + test_even_and_range(num_levels - 1, num_levels, 4096, 4); } // Testing only HistogramEven is fine, because HistogramRange shares the loading logic and the different binning diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index 3a981a493fd8..e0c3a3b98c5c 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -739,6 +739,68 @@ CUB_TEST("DeviceHistogram::MultiHistogramEven uses environment", "[histogram][de REQUIRE(d_histogram_b == expected_b); } +CUB_TEST("DeviceHistogram::MultiHistogramEven handles the device-launch dynamic-SMEM boundary", + "[histogram][device]", + CUB_SMALL) +{ + int current_device{}; + REQUIRE(cudaSuccess == cudaGetDevice(¤t_device)); + + cuda::compute_capability cc{}; + REQUIRE(cudaSuccess == cub::detail::ptx_compute_cap(cc, current_device)); + if (cc < cuda::compute_capability{10, 0}) + { + SKIP("The runtime-sized shared-memory histogram policy is currently tuned for SM100"); + } + + constexpr int num_channels = 4; + constexpr int num_active_channels = 3; + // The direct-load kernel has no static shared-memory footprint, so 4,096 + // three-channel counters exactly fill the B200's 48 KiB device-launch limit. + constexpr int num_bins = 4096; + constexpr int num_levels = num_bins + 1; + auto d_samples = c2h::device_vector{0, 1, 2, 3}; + + cuda::std::array levels{num_levels, num_levels, num_levels}; + cuda::std::array lower_levels{0, 0, 0}; + cuda::std::array upper_levels{num_bins, num_bins, num_bins}; + + auto d_histogram_r = c2h::device_vector(num_bins, 0); + auto d_histogram_g = c2h::device_vector(num_bins, 0); + auto d_histogram_b = c2h::device_vector(num_bins, 0); + cuda::std::array d_histograms = { + thrust::raw_pointer_cast(d_histogram_r.data()), + thrust::raw_pointer_cast(d_histogram_g.data()), + thrust::raw_pointer_cast(d_histogram_b.data())}; + + size_t expected_bytes_allocated{}; + REQUIRE( + cudaSuccess + == cub::DeviceHistogram::MultiHistogramEven( + nullptr, + expected_bytes_allocated, + thrust::raw_pointer_cast(d_samples.data()), + d_histograms, + levels, + lower_levels, + upper_levels, + 1)); + + auto env = stdexec::env{expected_allocation_size(expected_bytes_allocated)}; + multi_histogram_even( + thrust::raw_pointer_cast(d_samples.data()), d_histograms, levels, lower_levels, upper_levels, 1, env); + + auto expected_r = c2h::device_vector(num_bins, 0); + auto expected_g = c2h::device_vector(num_bins, 0); + auto expected_b = c2h::device_vector(num_bins, 0); + expected_r[0] = 1; + expected_g[1] = 1; + expected_b[2] = 1; + REQUIRE(d_histogram_r == expected_r); + REQUIRE(d_histogram_g == expected_g); + REQUIRE(d_histogram_b == expected_b); +} + CUB_TEST_CASE("DeviceHistogram::MultiHistogramEven uses custom stream", "[histogram][device]", CUB_SMALL) { [[maybe_unused]] constexpr int NUM_CHANNELS = 4; @@ -1886,6 +1948,8 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" constexpr auto sm100_wide_counter_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); + constexpr int expected_single_channel_policy_bytes = 228352; + constexpr int expected_single_channel_limit_bytes = expected_single_channel_policy_bytes; STATIC_REQUIRE(cub::detail::histogram::max_privatized_smem_bins( sm90_policy.max_privatized_static_smem_single_channel_bytes) @@ -1894,11 +1958,13 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" sm100_policy.max_privatized_static_smem_single_channel_bytes) == 256); STATIC_REQUIRE(sm90_policy.max_privatized_dynamic_smem_single_channel_bytes == 0); - STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_single_channel_bytes == 228352); + STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_single_channel_bytes == expected_single_channel_policy_bytes); + STATIC_REQUIRE( + cub::detail::histogram::dynamic_smem_limit_bytes(sm100_policy) == expected_single_channel_limit_bytes); STATIC_REQUIRE(sm100_wide_counter_policy.max_privatized_dynamic_smem_single_channel_bytes == 0); STATIC_REQUIRE(cub::detail::histogram::max_privatized_smem_bins( sm100_policy.max_privatized_dynamic_smem_single_channel_bytes) - == 57088); + == expected_single_channel_policy_bytes / int{sizeof(unsigned int)}); STATIC_REQUIRE(sm100_policy.max_privatized_dynamic_smem_multi_channel_range_bytes == 8192); STATIC_REQUIRE(sm100_policy.gmem.threads_per_block == 768); STATIC_REQUIRE(sm100_policy.gmem.items_per_thread == 12); @@ -1928,9 +1994,24 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" constexpr auto sm100_even_4ch_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); - STATIC_REQUIRE(sm100_even_2ch_policy.max_privatized_dynamic_smem_2_channel_even_bytes == 0); - STATIC_REQUIRE(sm100_even_3ch_policy.max_privatized_dynamic_smem_3_channel_even_bytes == 0); - STATIC_REQUIRE(sm100_even_4ch_policy.max_privatized_dynamic_smem_4_channel_even_bytes == 0); + constexpr int expected_even_2ch_policy_bytes = 65536; + constexpr int expected_even_3ch_policy_bytes = 98304; + constexpr int expected_even_4ch_policy_bytes = 131072; + constexpr int expected_even_2ch_limit_bytes = expected_even_2ch_policy_bytes; + constexpr int expected_even_3ch_limit_bytes = expected_even_3ch_policy_bytes; + constexpr int expected_even_4ch_limit_bytes = expected_even_4ch_policy_bytes; + STATIC_REQUIRE( + sm100_even_2ch_policy.max_privatized_dynamic_smem_2_channel_even_bytes == expected_even_2ch_policy_bytes); + STATIC_REQUIRE( + sm100_even_3ch_policy.max_privatized_dynamic_smem_3_channel_even_bytes == expected_even_3ch_policy_bytes); + STATIC_REQUIRE( + sm100_even_4ch_policy.max_privatized_dynamic_smem_4_channel_even_bytes == expected_even_4ch_policy_bytes); + STATIC_REQUIRE( + cub::detail::histogram::dynamic_smem_limit_bytes(sm100_even_2ch_policy) == expected_even_2ch_limit_bytes); + STATIC_REQUIRE( + cub::detail::histogram::dynamic_smem_limit_bytes(sm100_even_3ch_policy) == expected_even_3ch_limit_bytes); + STATIC_REQUIRE( + cub::detail::histogram::dynamic_smem_limit_bytes(sm100_even_4ch_policy) == expected_even_4ch_limit_bytes); STATIC_REQUIRE(sm100_multi_range_policy.gmem.threads_per_block == 384); STATIC_REQUIRE(sm100_multi_range_policy.gmem.items_per_thread == 5); STATIC_REQUIRE(sm100_multi_range_policy.static_smem.threads_per_block == 384); @@ -1945,9 +2026,12 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" == privatization_mode::static_smem); STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_policy, 257) == privatization_mode::dynamic_smem); - STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_policy, 57088) + constexpr int sm100_single_channel_max_dynamic_bins = expected_single_channel_limit_bytes / int{sizeof(unsigned int)}; + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode( + sm100_policy, sm100_single_channel_max_dynamic_bins) == privatization_mode::dynamic_smem); - STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_policy, 57089) + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode( + sm100_policy, sm100_single_channel_max_dynamic_bins + 1) == privatization_mode::gmem); STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_range_u64_policy, 256) == privatization_mode::static_smem); @@ -1968,10 +2052,31 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_4ch_policy, 256) == privatization_mode::static_smem); STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_4ch_policy, 257) + == privatization_mode::dynamic_smem); + constexpr int sm100_even_4ch_max_dynamic_bins = expected_even_4ch_limit_bytes / sizeof(unsigned int) / 4; + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode( + sm100_even_4ch_policy, sm100_even_4ch_max_dynamic_bins) + == privatization_mode::dynamic_smem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode( + sm100_even_4ch_policy, sm100_even_4ch_max_dynamic_bins + 1) == privatization_mode::gmem); STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_2ch_policy, 257) + == privatization_mode::dynamic_smem); + constexpr int sm100_even_2ch_max_dynamic_bins = expected_even_2ch_limit_bytes / sizeof(unsigned int) / 2; + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode( + sm100_even_2ch_policy, sm100_even_2ch_max_dynamic_bins) + == privatization_mode::dynamic_smem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode( + sm100_even_2ch_policy, sm100_even_2ch_max_dynamic_bins + 1) == privatization_mode::gmem); STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode(sm100_even_3ch_policy, 257) + == privatization_mode::dynamic_smem); + constexpr int sm100_even_3ch_max_dynamic_bins = expected_even_3ch_limit_bytes / sizeof(unsigned int) / 3; + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode( + sm100_even_3ch_policy, sm100_even_3ch_max_dynamic_bins) + == privatization_mode::dynamic_smem); + STATIC_REQUIRE(cub::detail::histogram::select_privatization_mode( + sm100_even_3ch_policy, sm100_even_3ch_max_dynamic_bins + 1) == privatization_mode::gmem); STATIC_REQUIRE( cub::detail::histogram::select_privatization_mode(sm100_wide_counter_policy, 128) From 39bd606dc80331ffcbda2db030212c9cf085626d Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Wed, 19 Aug 2026 18:11:26 +0000 Subject: [PATCH 42/45] Address histogram review naming and policy feedback --- cub/cub/agent/agent_histogram.cuh | 46 +++--- .../device/dispatch/dispatch_histogram.cuh | 11 +- .../dispatch/kernels/kernel_histogram.cuh | 134 +++++++++--------- .../dispatch/tuning/tuning_histogram.cuh | 56 +++++--- ...test_device_histogram_custom_policy_hub.cu | 8 +- cub/test/catch2_test_device_histogram_env.cu | 63 ++++---- .../catch2_test_device_histogram_env_api.cu | 23 +-- 7 files changed, 179 insertions(+), 162 deletions(-) diff --git a/cub/cub/agent/agent_histogram.cuh b/cub/cub/agent/agent_histogram.cuh index 5ff9ac75d256..48f2fface97f 100644 --- a/cub/cub/agent/agent_histogram.cuh +++ b/cub/cub/agent/agent_histogram.cuh @@ -168,10 +168,10 @@ struct AgentHistogram "Static-SMEM privatization requires room for at least one bin"); static constexpr int vec_size = privatization_policy.vec_size; static constexpr int threads_per_block = privatization_policy.threads_per_block; - static constexpr int pixels_per_thread = privatization_policy.items_per_thread; - static constexpr int samples_per_thread = pixels_per_thread * NumChannels; + static constexpr int items_per_thread = privatization_policy.items_per_thread; + static constexpr int samples_per_thread = items_per_thread * NumChannels; static constexpr int vecs_per_thread = samples_per_thread / vec_size; - static constexpr int tile_pixels = pixels_per_thread * threads_per_block; + static constexpr int tile_pixels = items_per_thread * threads_per_block; static constexpr int tile_samples = samples_per_thread * threads_per_block; static constexpr bool is_rle_compress = privatization_policy.rle_compress; static constexpr bool is_work_stealing = privatization_policy.work_stealing; @@ -192,7 +192,7 @@ struct AgentHistogram using WrappedVecsIteratorT = CacheModifiedInputIterator; using BlockLoadSampleT = BlockLoad; - using BlockLoadPixelT = BlockLoad; + using BlockLoadPixelT = BlockLoad; using BlockLoadVecT = BlockLoad; struct _TempStorage @@ -259,8 +259,8 @@ struct AgentHistogram // Accumulate pixels. Specialized for RLE compression. _CCCL_DEVICE _CCCL_FORCEINLINE void AccumulatePixels( - SampleT samples[pixels_per_thread][NumChannels], - bool is_valid[pixels_per_thread], + SampleT samples[items_per_thread][NumChannels], + bool is_valid[items_per_thread], ::cuda::std::true_type is_rle_compress) { _CCCL_PRAGMA_UNROLL_FULL() @@ -268,10 +268,10 @@ struct AgentHistogram { CounterT* privatized_histogram = PrivatizedHistogram(ch); // Bin pixels - int bins[pixels_per_thread]; + int bins[items_per_thread]; _CCCL_PRAGMA_UNROLL_FULL() - for (int pixel = 0; pixel < pixels_per_thread; ++pixel) + for (int pixel = 0; pixel < items_per_thread; ++pixel) { bins[pixel] = -1; privatized_decode_op[ch].template BinSelect(samples[pixel][ch], bins[pixel], is_valid[pixel]); @@ -280,7 +280,7 @@ struct AgentHistogram CounterT accumulator = 1; _CCCL_PRAGMA_UNROLL_FULL() - for (int pixel = 0; pixel < pixels_per_thread - 1; ++pixel) + for (int pixel = 0; pixel < items_per_thread - 1; ++pixel) { if (bins[pixel] != bins[pixel + 1]) { @@ -295,21 +295,21 @@ struct AgentHistogram } // Last pixel - if (bins[pixels_per_thread - 1] >= 0) + if (bins[items_per_thread - 1] >= 0) { - atomicAdd_block(privatized_histogram + bins[pixels_per_thread - 1], accumulator); + atomicAdd_block(privatized_histogram + bins[items_per_thread - 1], accumulator); } } } // Accumulate pixels. Specialized for individual accumulation of each pixel. _CCCL_DEVICE _CCCL_FORCEINLINE void AccumulatePixels( - SampleT samples[pixels_per_thread][NumChannels], - bool is_valid[pixels_per_thread], + SampleT samples[items_per_thread][NumChannels], + bool is_valid[items_per_thread], ::cuda::std::false_type is_rle_compress) { _CCCL_PRAGMA_UNROLL_FULL() - for (int pixel = 0; pixel < pixels_per_thread; ++pixel) + for (int pixel = 0; pixel < items_per_thread; ++pixel) { _CCCL_PRAGMA_UNROLL_FULL() for (int ch = 0; ch < NumActiveChannels; ++ch) @@ -327,7 +327,7 @@ struct AgentHistogram // Load full, aligned tile using pixel iterator _CCCL_DEVICE _CCCL_FORCEINLINE void - LoadFullAlignedTile(OffsetT block_offset, SampleT (&samples)[pixels_per_thread][NumChannels]) + LoadFullAlignedTile(OffsetT block_offset, SampleT (&samples)[items_per_thread][NumChannels]) { if constexpr (NumActiveChannels == 1) { @@ -338,7 +338,7 @@ struct AgentHistogram } else { - using AliasedPixels = PixelT[pixels_per_thread]; + using AliasedPixels = PixelT[items_per_thread]; WrappedPixelIteratorT d_wrapped_pixels(reinterpret_cast(d_native_samples + block_offset)); // Load using a wrapped pixel iterator BlockLoadPixelT{static_smem_storage.pixel_load}.Load(d_wrapped_pixels, reinterpret_cast(samples)); @@ -347,7 +347,7 @@ struct AgentHistogram template _CCCL_DEVICE _CCCL_FORCEINLINE void - LoadTile(OffsetT block_offset, int valid_samples, SampleT (&samples)[pixels_per_thread][NumChannels]) + LoadTile(OffsetT block_offset, int valid_samples, SampleT (&samples)[items_per_thread][NumChannels]) { if constexpr (IsFullTile) { @@ -368,7 +368,7 @@ struct AgentHistogram if constexpr (IsAligned) { // Load partially-full, aligned tile using the pixel iterator - using AliasedPixels = PixelT[pixels_per_thread]; + using AliasedPixels = PixelT[items_per_thread]; WrappedPixelIteratorT d_wrapped_pixels((PixelT*) (d_native_samples + block_offset)); int valid_pixels = valid_samples / NumChannels; @@ -386,10 +386,10 @@ struct AgentHistogram } template - _CCCL_DEVICE _CCCL_FORCEINLINE void MarkValid(bool (&is_valid)[pixels_per_thread], int valid_samples) + _CCCL_DEVICE _CCCL_FORCEINLINE void MarkValid(bool (&is_valid)[items_per_thread], int valid_samples) { _CCCL_PRAGMA_UNROLL_FULL() - for (int pixel = 0; pixel < pixels_per_thread; ++pixel) + for (int pixel = 0; pixel < items_per_thread; ++pixel) { if constexpr (IsStriped) { @@ -397,7 +397,7 @@ struct AgentHistogram } else { - is_valid[pixel] = IsFullTile || (((threadIdx.x * pixels_per_thread + pixel) * NumChannels) < valid_samples); + is_valid[pixel] = IsFullTile || (((threadIdx.x * items_per_thread + pixel) * NumChannels) < valid_samples); } } } @@ -412,8 +412,8 @@ struct AgentHistogram template _CCCL_DEVICE _CCCL_FORCEINLINE void ConsumeTile(OffsetT block_offset, int valid_samples) { - SampleT samples[pixels_per_thread][NumChannels]; - bool is_valid[pixels_per_thread]; + SampleT samples[items_per_thread][NumChannels]; + bool is_valid[items_per_thread]; LoadTile(block_offset, valid_samples, samples); MarkValid(is_valid, valid_samples); diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index 7d6fffded1ea..a9853fa68568 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -406,21 +406,21 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( num_privatized_levels.begin(), num_privatized_levels.end(), num_privatized_bins_wrapper.begin(), minus_one); ::cuda::std::transform(num_output_levels.begin(), num_output_levels.end(), num_output_bins_wrapper.begin(), minus_one); - int histogram_init_grid_dims = - (max_num_output_bins + histogram_init_threads_per_block - 1) / histogram_init_threads_per_block; + const int histogram_init_grid_dims = + (max_num_output_bins + active_policy.init_threads_per_block - 1) / active_policy.init_threads_per_block; // Log DeviceHistogramInitKernel configuration #ifdef CUB_DEBUG_LOG _CubLog("Invoking DeviceHistogramInitKernel<<<%d, %d, 0, %lld>>>()\n", histogram_init_grid_dims, - histogram_init_threads_per_block, + active_policy.init_threads_per_block, (long long) stream); #endif // CUB_DEBUG_LOG // Invoke histogram_init_kernel if (const auto error = CubDebug( launcher_factory(histogram_init_grid_dims, - histogram_init_threads_per_block, + active_policy.init_threads_per_block, 0, stream, /* dependent_launch */ cc >= ::cuda::compute_capability{9, 0}) @@ -912,7 +912,8 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_legacy_policy() -> HistogramPolicy kernel_config, kernel_config, kernel_config, - legacy_privatized_static_smem_bins * int{sizeof(CounterT)}, + 256, + 256 * int{sizeof(CounterT)}, 0, 0, 0, diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index 2e904f3eb3d5..e33eb4934433 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -26,7 +26,7 @@ CUB_NAMESPACE_BEGIN namespace detail::histogram { -template +template struct Transforms { //--------------------------------------------------------------------- @@ -50,8 +50,8 @@ struct Transforms _CCCL_DEVICE _CCCL_FORCEINLINE void Precompute() {} - template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(SampleT2 sample, int& bin, bool valid) const + template + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(SampleT sample, int& bin, bool valid) const { using WrappedLevelIteratorT = ::cuda::std::_If<::cuda::std::is_pointer_v, @@ -104,21 +104,21 @@ struct Transforms int bin = -1; // cached bin; < 0 means empty }; - BinSelectState mru; + BinSelectState most_recent_bin; LevelIteratorT d_levels; // Pointer to levels array int num_output_levels; // Number of levels in array // Interpolation state shared by all samples processed by a thread. - float m_inv_scale; // num_bins / (float)(last - first); valid iff m_have_precompute - LevelT m_first; // cached d_levels[0] - LevelT m_last; // cached d_levels[num_bins] - bool m_have_precompute; // whether the fields above are valid + float inv_scale; // num_bins / (float)(last - first); valid iff have_precompute + LevelT first; // cached d_levels[0] + LevelT last; // cached d_levels[num_bins] + bool have_precompute; // whether the fields above are valid // Piecewise-linear interpolation state split at the midpoint level. - LevelT m_mid; // cached d_levels[mid_bin] - float m_inv_scale_lo; // mid_bin / (float)(mid - first) - float m_inv_scale_hi; // (num_bins - mid_bin) / (float)(last - mid) - int m_mid_bin; // split bin index (num_bins / 2) + LevelT mid; // cached d_levels[mid_bin] + float inv_scale_lo; // mid_bin / (float)(mid - first) + float inv_scale_hi; // (num_bins - mid_bin) / (float)(last - mid) + int mid_bin; // split bin index (num_bins / 2) //! @brief Initializer //! @@ -128,15 +128,15 @@ struct Transforms { this->d_levels = d_levels_; this->num_output_levels = num_output_levels_; - this->m_have_precompute = false; - this->m_inv_scale = 0.0f; - this->m_first = LevelT{}; - this->m_last = LevelT{}; - this->m_mid = LevelT{}; - this->m_inv_scale_lo = 0.0f; - this->m_inv_scale_hi = 0.0f; - this->m_mid_bin = 0; - this->mru = BinSelectState{}; + this->have_precompute = false; + this->inv_scale = 0.0f; + this->first = LevelT{}; + this->last = LevelT{}; + this->mid = LevelT{}; + this->inv_scale_lo = 0.0f; + this->inv_scale_hi = 0.0f; + this->mid_bin = 0; + this->most_recent_bin = BinSelectState{}; } //! @brief Precomputes interpolation slopes from the device level array. @@ -153,31 +153,31 @@ struct Transforms const LevelT last = wrapped_levels[num_bins]; if (!(first < last)) { - m_have_precompute = false; + have_precompute = false; return; } - m_first = first; - m_last = last; - m_inv_scale = static_cast(num_bins) / static_cast(interpolation_difference(last, first)); - m_have_precompute = true; + this->first = first; + this->last = last; + inv_scale = static_cast(num_bins) / static_cast(interpolation_difference(last, first)); + have_precompute = true; // Use a single secant if the midpoint does not split the level range. - m_mid_bin = 0; - m_inv_scale_lo = m_inv_scale; - m_inv_scale_hi = m_inv_scale; - m_mid = first; - const int mid_bin = num_bins >> 1; - if (mid_bin > 0 && mid_bin < num_bins) + this->mid_bin = 0; + inv_scale_lo = inv_scale; + inv_scale_hi = inv_scale; + mid = first; + const int split = num_bins >> 1; + if (split > 0 && split < num_bins) { - const LevelT mid = wrapped_levels[mid_bin]; - if ((first < mid) && (mid < last)) + const LevelT split_level = wrapped_levels[split]; + if ((first < split_level) && (split_level < last)) { - m_mid = mid; - m_mid_bin = mid_bin; - m_inv_scale_lo = static_cast(mid_bin) / static_cast(interpolation_difference(mid, first)); - m_inv_scale_hi = - static_cast(num_bins - mid_bin) / static_cast(interpolation_difference(last, mid)); + mid = split_level; + mid_bin = split; + inv_scale_lo = static_cast(split) / static_cast(interpolation_difference(split_level, first)); + inv_scale_hi = + static_cast(num_bins - split) / static_cast(interpolation_difference(last, split_level)); } } } @@ -187,8 +187,8 @@ struct Transforms //! A cached-bracket hit returns immediately. Otherwise, this computes and //! verifies an interpolated guess, checks one adjacent bracket, and finally //! falls back to `UpperBound` for arbitrary level distributions. - template - _CCCL_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid) + template + _CCCL_DEVICE _CCCL_FORCEINLINE void BinSelect(SampleT sample, int& bin, bool valid) { using WrappedLevelIteratorT = ::cuda::std::_If<::cuda::std::is_pointer_v, @@ -203,14 +203,14 @@ struct Transforms const LevelT s = static_cast(sample); - if (mru.bin >= 0 && !(s < mru.lo) && (s < mru.hi)) + if (most_recent_bin.bin >= 0 && !(s < most_recent_bin.lo) && (s < most_recent_bin.hi)) { - bin = mru.bin; + bin = most_recent_bin.bin; return; } - const LevelT first_level = m_have_precompute ? m_first : wrapped_levels[0]; - const LevelT last_level = m_have_precompute ? m_last : wrapped_levels[num_bins]; + const LevelT first_level = have_precompute ? first : wrapped_levels[0]; + const LevelT last_level = have_precompute ? last : wrapped_levels[num_bins]; if (!(first_level < last_level)) { @@ -230,23 +230,23 @@ struct Transforms const auto delta = interpolation_difference(s, first_level); int guess; - if (m_have_precompute) + if (have_precompute) { - if (m_mid_bin > 0) + if (mid_bin > 0) { - if (s < m_mid) + if (s < mid) { - guess = static_cast(static_cast(delta) * m_inv_scale_lo); + guess = static_cast(static_cast(delta) * inv_scale_lo); } else { - const auto delta_hi = interpolation_difference(s, m_mid); - guess = m_mid_bin + static_cast(static_cast(delta_hi) * m_inv_scale_hi); + const auto delta_hi = interpolation_difference(s, mid); + guess = mid_bin + static_cast(static_cast(delta_hi) * inv_scale_hi); } } else { - guess = static_cast(static_cast(delta) * m_inv_scale); + guess = static_cast(static_cast(delta) * inv_scale); } } else @@ -269,8 +269,8 @@ struct Transforms if (!(s < lvl_lo) && (s < lvl_hi)) { - bin = guess; - mru = BinSelectState{lvl_lo, lvl_hi, guess}; + bin = guess; + most_recent_bin = BinSelectState{lvl_lo, lvl_hi, guess}; return; } @@ -282,8 +282,8 @@ struct Transforms const LevelT lvl2_lo = wrapped_levels[g2]; if (!(s < lvl2_lo)) { - bin = g2; - mru = BinSelectState{lvl2_lo, lvl_lo, g2}; + bin = g2; + most_recent_bin = BinSelectState{lvl2_lo, lvl_lo, g2}; return; } } @@ -296,8 +296,8 @@ struct Transforms const LevelT lvl2_hi = wrapped_levels[g2 + 1]; if (s < lvl2_hi) { - bin = g2; - mru = BinSelectState{lvl_hi, lvl2_hi, g2}; + bin = g2; + most_recent_bin = BinSelectState{lvl_hi, lvl2_hi, g2}; return; } } @@ -311,7 +311,7 @@ struct Transforms } if (bin >= 0) { - mru = BinSelectState{wrapped_levels[bin], wrapped_levels[bin + 1], bin}; + most_recent_bin = BinSelectState{wrapped_levels[bin], wrapped_levels[bin + 1], bin}; } } }; @@ -319,12 +319,12 @@ struct Transforms // Scales samples to evenly-spaced bins struct ScaleTransform { - using CommonT = ::cuda::std::common_type_t; + using CommonT = ::cuda::std::common_type_t; static_assert(::cuda::std::is_convertible_v, - "The common type of `LevelT` and `SampleT` must be " + "The common type of `LevelT` and `InputSampleT` must be " "convertible to `int`."); static_assert(::cuda::is_trivially_copyable_v, - "The common type of `LevelT` and `SampleT` must be " + "The common type of `LevelT` and `InputSampleT` must be " "trivially copyable."); // An arithmetic type that's used for bin computation of integral types, guaranteed to not @@ -334,7 +334,7 @@ struct Transforms // multiplication result. // If CommonT used to be a 128-bit wide integral type already, we use CommonT's arithmetic using IntArithmeticT = ::cuda::std::_If< // - sizeof(SampleT) + sizeof(CommonT) <= sizeof(uint32_t), // + sizeof(InputSampleT) + sizeof(CommonT) <= sizeof(uint32_t), // uint32_t, // #if _CCCL_HAS_INT128() ::cuda::std::_If< // @@ -511,7 +511,7 @@ struct Transforms // Method for converting samples to bin-ids template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(SampleT sample, int& bin, bool valid) const + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(InputSampleT sample, int& bin, bool valid) const { const CommonT common_sample = static_cast(sample); @@ -544,8 +544,8 @@ struct Transforms _CCCL_DEVICE _CCCL_FORCEINLINE void Precompute() {} // Method for converting samples to bin-ids - template - _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(_SampleT sample, int& bin, bool valid) const + template + _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void BinSelect(SampleT sample, int& bin, bool valid) const { if (valid) { @@ -601,7 +601,7 @@ _CCCL_KERNEL_ATTRIBUTES void DeviceHistogramInitKernel( ::cuda::std::reduce(num_output_bins_wrapper.begin(), num_output_bins_wrapper.end(), ::cuda::std::uint64_t{0}) * sizeof(CounterT); if (output_histogram_bytes - <= static_cast<::cuda::std::uint64_t>(policy.max_output_histogram_bytes_for_init_kernel_pdl_trigger)) + <= static_cast<::cuda::std::uint64_t>(policy.max_output_histogram_bytes_for_init_kernel_pdl)) { _CCCL_PDL_TRIGGER_NEXT_LAUNCH(); } diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index e3d710e916e4..8e659dea13cb 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -64,6 +64,7 @@ struct HistogramPolicy HistogramPrivatizationPolicy gmem; //!< Policy for global-memory privatization HistogramPrivatizationPolicy static_smem; //!< Policy for compile-time-sized shared-memory privatization HistogramPrivatizationPolicy dynamic_smem; //!< Policy for runtime-sized shared-memory privatization + int init_threads_per_block; //!< Number of threads in a histogram initialization block int max_privatized_static_smem_single_channel_bytes; //!< Single-channel compile-time-sized SMEM limit int max_privatized_dynamic_smem_single_channel_bytes; //!< Single-channel runtime-sized SMEM limit int static_smem_min_blocks_per_sm; //!< Minimum blocks per SM requested by the static-SMEM launch bounds @@ -71,12 +72,13 @@ struct HistogramPolicy int max_privatized_dynamic_smem_2_channel_even_bytes; //!< Two-channel HistogramEven SMEM limit int max_privatized_dynamic_smem_3_channel_even_bytes; //!< Three-channel HistogramEven SMEM limit int max_privatized_dynamic_smem_4_channel_even_bytes; //!< Four-channel HistogramEven SMEM limit - int max_output_histogram_bytes_for_init_kernel_pdl_trigger; //!< Largest output allocation for init-kernel PDL + int max_output_histogram_bytes_for_init_kernel_pdl; //!< Largest output allocation for init-kernel PDL [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool operator==(const HistogramPolicy& lhs, const HistogramPolicy& rhs) noexcept { return lhs.gmem == rhs.gmem && lhs.static_smem == rhs.static_smem && lhs.dynamic_smem == rhs.dynamic_smem + && lhs.init_threads_per_block == rhs.init_threads_per_block && lhs.max_privatized_static_smem_single_channel_bytes == rhs.max_privatized_static_smem_single_channel_bytes && lhs.max_privatized_dynamic_smem_single_channel_bytes == rhs.max_privatized_dynamic_smem_single_channel_bytes && lhs.static_smem_min_blocks_per_sm == rhs.static_smem_min_blocks_per_sm @@ -85,8 +87,7 @@ struct HistogramPolicy && lhs.max_privatized_dynamic_smem_2_channel_even_bytes == rhs.max_privatized_dynamic_smem_2_channel_even_bytes && lhs.max_privatized_dynamic_smem_3_channel_even_bytes == rhs.max_privatized_dynamic_smem_3_channel_even_bytes && lhs.max_privatized_dynamic_smem_4_channel_even_bytes == rhs.max_privatized_dynamic_smem_4_channel_even_bytes - && lhs.max_output_histogram_bytes_for_init_kernel_pdl_trigger - == rhs.max_output_histogram_bytes_for_init_kernel_pdl_trigger; + && lhs.max_output_histogram_bytes_for_init_kernel_pdl == rhs.max_output_histogram_bytes_for_init_kernel_pdl; } [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool @@ -100,8 +101,9 @@ struct HistogramPolicy { return os << "HistogramPolicy { .gmem = " << p.gmem << ", .static_smem = " << p.static_smem - << ", .dynamic_smem = " << p.dynamic_smem << ", .max_privatized_static_smem_single_channel_bytes = " - << p.max_privatized_static_smem_single_channel_bytes << ", .max_privatized_dynamic_smem_single_channel_bytes = " + << ", .dynamic_smem = " << p.dynamic_smem << ", .init_threads_per_block = " << p.init_threads_per_block + << ", .max_privatized_static_smem_single_channel_bytes = " << p.max_privatized_static_smem_single_channel_bytes + << ", .max_privatized_dynamic_smem_single_channel_bytes = " << p.max_privatized_dynamic_smem_single_channel_bytes << ", .static_smem_min_blocks_per_sm = " << p.static_smem_min_blocks_per_sm << ", .max_privatized_dynamic_smem_multi_channel_range_bytes = " << p.max_privatized_dynamic_smem_multi_channel_range_bytes @@ -110,18 +112,14 @@ struct HistogramPolicy << ", .max_privatized_dynamic_smem_3_channel_even_bytes = " << p.max_privatized_dynamic_smem_3_channel_even_bytes << ", .max_privatized_dynamic_smem_4_channel_even_bytes = " - << p.max_privatized_dynamic_smem_4_channel_even_bytes - << ", .max_output_histogram_bytes_for_init_kernel_pdl_trigger = " - << p.max_output_histogram_bytes_for_init_kernel_pdl_trigger << " }"; + << p.max_privatized_dynamic_smem_4_channel_even_bytes << ", .max_output_histogram_bytes_for_init_kernel_pdl = " + << p.max_output_histogram_bytes_for_init_kernel_pdl << " }"; } #endif }; namespace detail::histogram { -inline constexpr int histogram_init_threads_per_block = 256; -inline constexpr int legacy_privatized_static_smem_bins = 256; - enum class privatization_mode { gmem, @@ -298,11 +296,12 @@ public: // All storage thresholds are byte budgets. Dispatch derives the corresponding // bin limits from the local counter width and active channel count. - constexpr int max_privatized_static_smem_bytes = 1024; - constexpr int max_privatized_dynamic_smem_single_channel_bytes = 228352; - constexpr int max_privatized_dynamic_smem_range_bytes_per_channel = 8192; - constexpr int max_privatized_dynamic_smem_even_bytes_per_channel = 32768; - constexpr int max_output_histogram_bytes_for_init_kernel_pdl_trigger = 8192; + constexpr int max_privatized_static_smem_bytes = 1024; + constexpr int max_privatized_dynamic_smem_single_channel_bytes = 228352; + constexpr int max_privatized_dynamic_smem_range_bytes_per_channel = 8192; + constexpr int max_privatized_dynamic_smem_even_bytes_per_channel = 32768; + constexpr int init_threads_per_block = 256; + constexpr int max_output_histogram_bytes_for_init_kernel_pdl = 8192; const bool supports_dynamic_smem = counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive; @@ -323,13 +322,14 @@ public: const int init_kernel_pdl_trigger_bytes = single_channel && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive && (sample_size_bytes == 1 || sample_size_bytes == 2 || sample_size_bytes == 4 || sample_size_bytes == 8) - ? max_output_histogram_bytes_for_init_kernel_pdl_trigger + ? max_output_histogram_bytes_for_init_kernel_pdl : 0; return HistogramPolicy{ gmem, static_smem, gmem, + init_threads_per_block, max_privatized_static_smem_bytes, dynamic_smem_single_channel_bytes, range_multi_static || range_u64_static ? 3 : 0, @@ -365,20 +365,32 @@ public: sweep = HistogramPrivatizationPolicy{960, 10, 4, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, true, false}; } } - constexpr int max_privatized_static_smem_bytes = 1024; - constexpr int max_output_histogram_bytes_for_init_kernel_pdl_trigger = 8192; + constexpr int max_privatized_static_smem_bytes = 1024; + constexpr int init_threads_per_block = 256; + constexpr int max_output_histogram_bytes_for_init_kernel_pdl = 8192; const int init_kernel_pdl_trigger_bytes = num_channels == 1 && num_active_channels == 1 && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive && (sample_size_bytes == 1 || sample_size_bytes == 2) - ? max_output_histogram_bytes_for_init_kernel_pdl_trigger + ? max_output_histogram_bytes_for_init_kernel_pdl : 0; return HistogramPolicy{ - sweep, sweep, sweep, max_privatized_static_smem_bytes, 0, 0, 0, 0, 0, 0, init_kernel_pdl_trigger_bytes}; + sweep, + sweep, + sweep, + init_threads_per_block, + max_privatized_static_smem_bytes, + 0, + 0, + 0, + 0, + 0, + 0, + init_kernel_pdl_trigger_bytes}; } // Architectures before SM90 use the longstanding generic histogram tuning. const auto sweep = HistogramPrivatizationPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; - return HistogramPolicy{sweep, sweep, sweep, 1024, 0, 0, 0, 0, 0, 0, 0}; + return HistogramPolicy{sweep, sweep, sweep, 256, 1024, 0, 0, 0, 0, 0, 0, 0}; } }; diff --git a/cub/test/catch2_test_device_histogram_custom_policy_hub.cu b/cub/test/catch2_test_device_histogram_custom_policy_hub.cu index 4d88ad904fb4..335edddccb5f 100644 --- a/cub/test/catch2_test_device_histogram_custom_policy_hub.cu +++ b/cub/test/catch2_test_device_histogram_custom_policy_hub.cu @@ -56,10 +56,10 @@ CUB_TEST("DispatchHistogram::DispatchEven: custom policy hub", "[histogram][devi REQUIRE( custom_wide_counter_policy.max_privatized_static_smem_single_channel_bytes == 256 * sizeof(unsigned long long)); REQUIRE(custom_sm90_policy.max_privatized_dynamic_smem_single_channel_bytes == 0); - REQUIRE(custom_sm75_policy.max_output_histogram_bytes_for_init_kernel_pdl_trigger == 0); - REQUIRE(custom_sm90_policy.max_output_histogram_bytes_for_init_kernel_pdl_trigger == 8192); - REQUIRE(custom_wide_counter_policy.max_output_histogram_bytes_for_init_kernel_pdl_trigger == 0); - REQUIRE(custom_wide_counter_sm90_policy.max_output_histogram_bytes_for_init_kernel_pdl_trigger == 16384); + REQUIRE(custom_sm75_policy.max_output_histogram_bytes_for_init_kernel_pdl == 0); + REQUIRE(custom_sm90_policy.max_output_histogram_bytes_for_init_kernel_pdl == 8192); + REQUIRE(custom_wide_counter_policy.max_output_histogram_bytes_for_init_kernel_pdl == 0); + REQUIRE(custom_wide_counter_sm90_policy.max_output_histogram_bytes_for_init_kernel_pdl == 16384); using sample_t = cuda::std::uint8_t; using counter_t = int; diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index e0c3a3b98c5c..a2dd366c8375 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1690,7 +1690,7 @@ struct histogram_tuning { constexpr auto sweep = cub::HistogramPrivatizationPolicy{BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; - return {sweep, sweep, sweep, 256 * sizeof(unsigned int), 0, 0, 0, 0, 0, 0, 0}; + return {sweep, sweep, sweep, 256, 256 * sizeof(unsigned int), 0, 0, 0, 0, 0, 0, 0}; } }; @@ -1712,6 +1712,7 @@ struct mixed_counter_histogram_tuning sweep, sweep, sweep, + 256, 512 * sizeof(unsigned int), 228352, 0, @@ -1882,6 +1883,7 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) {128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, {96, 3, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, {128, 7, 4, cub::BLOCK_LOAD_DIRECT, cub::CacheLoadModifier::LOAD_LDG, false, false}, + 256, 2052, 12345, 2, @@ -1894,35 +1896,36 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) # if _CCCL_STD_VER >= 2020 // designated init constexpr auto p2 = cub::HistogramPolicy{ - .gmem = {.threads_per_block = 128, - .items_per_thread = 7, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .static_smem = {.threads_per_block = 96, - .items_per_thread = 3, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .dynamic_smem = {.threads_per_block = 128, - .items_per_thread = 7, - .vec_size = 4, - .load_algorithm = cub::BLOCK_LOAD_DIRECT, - .load_modifier = cub::CacheLoadModifier::LOAD_LDG, - .rle_compress = false, - .work_stealing = false}, - .max_privatized_static_smem_single_channel_bytes = 2052, - .max_privatized_dynamic_smem_single_channel_bytes = 12345, - .static_smem_min_blocks_per_sm = 2, - .max_privatized_dynamic_smem_multi_channel_range_bytes = 1024, - .max_privatized_dynamic_smem_2_channel_even_bytes = 4096, - .max_privatized_dynamic_smem_3_channel_even_bytes = 8192, - .max_privatized_dynamic_smem_4_channel_even_bytes = 16384, - .max_output_histogram_bytes_for_init_kernel_pdl_trigger = 2048}; + .gmem = {.threads_per_block = 128, + .items_per_thread = 7, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .static_smem = {.threads_per_block = 96, + .items_per_thread = 3, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .dynamic_smem = {.threads_per_block = 128, + .items_per_thread = 7, + .vec_size = 4, + .load_algorithm = cub::BLOCK_LOAD_DIRECT, + .load_modifier = cub::CacheLoadModifier::LOAD_LDG, + .rle_compress = false, + .work_stealing = false}, + .init_threads_per_block = 256, + .max_privatized_static_smem_single_channel_bytes = 2052, + .max_privatized_dynamic_smem_single_channel_bytes = 12345, + .static_smem_min_blocks_per_sm = 2, + .max_privatized_dynamic_smem_multi_channel_range_bytes = 1024, + .max_privatized_dynamic_smem_2_channel_even_bytes = 4096, + .max_privatized_dynamic_smem_3_channel_even_bytes = 8192, + .max_privatized_dynamic_smem_4_channel_even_bytes = 16384, + .max_output_histogram_bytes_for_init_kernel_pdl = 2048}; # else // _CCCL_STD_VER >= 2020 constexpr auto p2 = p1; # endif // _CCCL_STD_VER >= 2020 diff --git a/cub/test/catch2_test_device_histogram_env_api.cu b/cub/test/catch2_test_device_histogram_env_api.cu index bd1af9bf3e86..07210b3a2481 100644 --- a/cub/test/catch2_test_device_histogram_env_api.cu +++ b/cub/test/catch2_test_device_histogram_env_api.cu @@ -421,17 +421,18 @@ struct HistogramPolicySelector const auto sweep = cub::HistogramPrivatizationPolicy{ 128, cc > cuda::compute_capability{9, 0} ? 16 : 7, 4, cub::BLOCK_LOAD_DIRECT, cub::LOAD_LDG, false, false}; return { - .gmem = sweep, - .static_smem = sweep, - .dynamic_smem = sweep, - .max_privatized_static_smem_single_channel_bytes = 256 * sizeof(unsigned int), - .max_privatized_dynamic_smem_single_channel_bytes = 0, - .static_smem_min_blocks_per_sm = 0, - .max_privatized_dynamic_smem_multi_channel_range_bytes = 0, - .max_privatized_dynamic_smem_2_channel_even_bytes = 0, - .max_privatized_dynamic_smem_3_channel_even_bytes = 0, - .max_privatized_dynamic_smem_4_channel_even_bytes = 0, - .max_output_histogram_bytes_for_init_kernel_pdl_trigger = 8192}; + .gmem = sweep, + .static_smem = sweep, + .dynamic_smem = sweep, + .init_threads_per_block = 256, + .max_privatized_static_smem_single_channel_bytes = 256 * sizeof(unsigned int), + .max_privatized_dynamic_smem_single_channel_bytes = 0, + .static_smem_min_blocks_per_sm = 0, + .max_privatized_dynamic_smem_multi_channel_range_bytes = 0, + .max_privatized_dynamic_smem_2_channel_even_bytes = 0, + .max_privatized_dynamic_smem_3_channel_even_bytes = 0, + .max_privatized_dynamic_smem_4_channel_even_bytes = 0, + .max_output_histogram_bytes_for_init_kernel_pdl = 8192}; } }; // example-end histogram-even-policy-selector From 03e18959b883336ce58033af562fb5460be1c18d Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Thu, 20 Aug 2026 18:14:02 +0000 Subject: [PATCH 43/45] [cub] Enable cached RANGE search for tuned GMEM cases --- .../device/dispatch/dispatch_histogram.cuh | 62 +++++++++++-------- .../dispatch/kernels/kernel_histogram.cuh | 12 ++-- .../dispatch/tuning/tuning_histogram.cuh | 26 +++++++- ...test_device_histogram_custom_policy_hub.cu | 1 + cub/test/catch2_test_device_histogram_env.cu | 31 +++++++++- .../catch2_test_device_histogram_env_api.cu | 1 + 6 files changed, 97 insertions(+), 36 deletions(-) diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index a9853fa68568..a6dd090d1b52 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -920,6 +920,7 @@ _CCCL_HOST_DEVICE_API constexpr auto convert_legacy_policy() -> HistogramPolicy 0, 0, 0, + 0, convert_pdl_trigger_bytes(0)}; } @@ -1101,13 +1102,16 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( } } int max_num_output_bins = max_levels - 1; + const auto privatization = + select_privatization_mode(active_policy, max_num_output_bins); constexpr bool supports_cached_search = ::cuda::std::is_integral_v || ::cuda::std::is_floating_point_v; if constexpr (supports_cached_search) { - if (select_privatization_mode(active_policy, max_num_output_bins) - == privatization_mode::dynamic_smem) + if (privatization == privatization_mode::dynamic_smem + || (privatization == privatization_mode::gmem + && use_cached_search_for_gmem_range(active_policy, max_num_output_bins))) { using PrivatizedDecodeOpT = typename TransformsT::template CachedSearchTransform; ::cuda::std::array privatized_decode_op{}; @@ -1116,28 +1120,35 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( privatized_decode_op[channel].Init(d_levels[channel], num_output_levels[channel]); } - return CubDebug((detail::histogram::dispatch( - d_temp_storage, - temp_storage_bytes, - d_samples, - d_output_histograms, - num_output_levels, - num_output_levels, - output_decode_op, - privatized_decode_op, - max_num_output_bins, - num_row_pixels, - num_rows, - row_stride_samples, - stream, - policy_selector, - kernel_source, - launcher_factory))); + const auto dispatch_with = [&](auto mode) { + using privatization_mode_t = decltype(mode); + return detail::histogram::dispatch( + d_temp_storage, + temp_storage_bytes, + d_samples, + d_output_histograms, + num_output_levels, + num_output_levels, + output_decode_op, + privatized_decode_op, + max_num_output_bins, + num_row_pixels, + num_rows, + row_stride_samples, + stream, + policy_selector, + kernel_source, + launcher_factory); + }; + + return CubDebug(privatization == privatization_mode::dynamic_smem + ? dispatch_with(HistogramPrivatizedDynamicSmem{}) + : dispatch_with(HistogramPrivatizedGmem{})); } } @@ -1149,8 +1160,7 @@ CUB_RUNTIME_FUNCTION static cudaError_t dispatch_range( } // Dispatch - if (select_privatization_mode(active_policy, max_num_output_bins) - != privatization_mode::static_smem) + if (privatization != privatization_mode::static_smem) { // Too many bins to keep in shared memory. if (const auto error = CubDebug( diff --git a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh index e33eb4934433..78b833598ee9 100644 --- a/cub/cub/device/dispatch/kernels/kernel_histogram.cuh +++ b/cub/cub/device/dispatch/kernels/kernel_histogram.cuh @@ -73,12 +73,12 @@ struct Transforms //! @brief Finds a RANGE bin with piecewise-linear interpolation and a per-thread bracket cache. //! - //! This transform is used by the runtime-sized shared-memory kernel. It - //! precomputes interpolation parameters once per thread, verifies each - //! interpolated guess against the level array, and falls back to binary - //! search for irregular levels. `BinSelectState` remembers the most recently - //! resolved bracket so consecutive samples in that bracket require no level - //! loads. + //! This transform is used by the runtime-sized shared-memory kernel and by + //! selected global-memory kernels. It precomputes interpolation parameters + //! once per thread, verifies each interpolated guess against the level array, + //! and falls back to binary search for irregular levels. `BinSelectState` + //! remembers the most recently resolved bracket so consecutive samples in + //! that bracket require no level loads. template struct CachedSearchTransform { diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 8e659dea13cb..12a15b6d0b8f 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -72,6 +72,7 @@ struct HistogramPolicy int max_privatized_dynamic_smem_2_channel_even_bytes; //!< Two-channel HistogramEven SMEM limit int max_privatized_dynamic_smem_3_channel_even_bytes; //!< Three-channel HistogramEven SMEM limit int max_privatized_dynamic_smem_4_channel_even_bytes; //!< Four-channel HistogramEven SMEM limit + int min_cached_search_gmem_range_bins; //!< Minimum RANGE bin count for cached search with GMEM privatization int max_output_histogram_bytes_for_init_kernel_pdl; //!< Largest output allocation for init-kernel PDL [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr bool @@ -87,6 +88,7 @@ struct HistogramPolicy && lhs.max_privatized_dynamic_smem_2_channel_even_bytes == rhs.max_privatized_dynamic_smem_2_channel_even_bytes && lhs.max_privatized_dynamic_smem_3_channel_even_bytes == rhs.max_privatized_dynamic_smem_3_channel_even_bytes && lhs.max_privatized_dynamic_smem_4_channel_even_bytes == rhs.max_privatized_dynamic_smem_4_channel_even_bytes + && lhs.min_cached_search_gmem_range_bins == rhs.min_cached_search_gmem_range_bins && lhs.max_output_histogram_bytes_for_init_kernel_pdl == rhs.max_output_histogram_bytes_for_init_kernel_pdl; } @@ -112,8 +114,10 @@ struct HistogramPolicy << ", .max_privatized_dynamic_smem_3_channel_even_bytes = " << p.max_privatized_dynamic_smem_3_channel_even_bytes << ", .max_privatized_dynamic_smem_4_channel_even_bytes = " - << p.max_privatized_dynamic_smem_4_channel_even_bytes << ", .max_output_histogram_bytes_for_init_kernel_pdl = " - << p.max_output_histogram_bytes_for_init_kernel_pdl << " }"; + << p.max_privatized_dynamic_smem_4_channel_even_bytes + << ", .min_cached_search_gmem_range_bins = " << p.min_cached_search_gmem_range_bins + << ", .max_output_histogram_bytes_for_init_kernel_pdl = " << p.max_output_histogram_bytes_for_init_kernel_pdl + << " }"; } #endif }; @@ -185,6 +189,12 @@ select_privatization_mode(const HistogramPolicy& policy, int num_bins) -> privat return privatization_mode::gmem; } +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool +use_cached_search_for_gmem_range(const HistogramPolicy& policy, int num_bins) +{ + return policy.min_cached_search_gmem_range_bins > 0 && num_bins >= policy.min_cached_search_gmem_range_bins; +} + // The C Parallel API erases CounterT before host dispatch, so its bridge must select from the // preserved runtime counter width. Typed CUB dispatch uses the overload above. template @@ -300,6 +310,8 @@ public: constexpr int max_privatized_dynamic_smem_single_channel_bytes = 228352; constexpr int max_privatized_dynamic_smem_range_bytes_per_channel = 8192; constexpr int max_privatized_dynamic_smem_even_bytes_per_channel = 32768; + constexpr int min_cached_search_gmem_single_channel_range_bins = 1; + constexpr int min_cached_search_gmem_multi_channel_range_bins = 16384; constexpr int init_threads_per_block = 256; constexpr int max_output_histogram_bytes_for_init_kernel_pdl = 8192; @@ -324,6 +336,12 @@ public: && (sample_size_bytes == 1 || sample_size_bytes == 2 || sample_size_bytes == 4 || sample_size_bytes == 8) ? max_output_histogram_bytes_for_init_kernel_pdl : 0; + const int min_cached_search_gmem_range_bins = + !is_even && sample_is_primitive && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} + && (sample_size_bytes == 4 || sample_size_bytes == 8) + ? (single_channel ? min_cached_search_gmem_single_channel_range_bins + : min_cached_search_gmem_multi_channel_range_bins) + : 0; return HistogramPolicy{ gmem, @@ -343,6 +361,7 @@ public: has_multi_channel_dynamic_smem && is_even && num_active_channels == 4 ? dynamic_smem_multi_channel_even_bytes : 0, + min_cached_search_gmem_range_bins, init_kernel_pdl_trigger_bytes}; } @@ -385,12 +404,13 @@ public: 0, 0, 0, + 0, init_kernel_pdl_trigger_bytes}; } // Architectures before SM90 use the longstanding generic histogram tuning. const auto sweep = HistogramPrivatizationPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; - return HistogramPolicy{sweep, sweep, sweep, 256, 1024, 0, 0, 0, 0, 0, 0, 0}; + return HistogramPolicy{sweep, sweep, sweep, 256, 1024, 0, 0, 0, 0, 0, 0, 0, 0}; } }; diff --git a/cub/test/catch2_test_device_histogram_custom_policy_hub.cu b/cub/test/catch2_test_device_histogram_custom_policy_hub.cu index 335edddccb5f..494c0828a45c 100644 --- a/cub/test/catch2_test_device_histogram_custom_policy_hub.cu +++ b/cub/test/catch2_test_device_histogram_custom_policy_hub.cu @@ -56,6 +56,7 @@ CUB_TEST("DispatchHistogram::DispatchEven: custom policy hub", "[histogram][devi REQUIRE( custom_wide_counter_policy.max_privatized_static_smem_single_channel_bytes == 256 * sizeof(unsigned long long)); REQUIRE(custom_sm90_policy.max_privatized_dynamic_smem_single_channel_bytes == 0); + REQUIRE(custom_sm90_policy.min_cached_search_gmem_range_bins == 0); REQUIRE(custom_sm75_policy.max_output_histogram_bytes_for_init_kernel_pdl == 0); REQUIRE(custom_sm90_policy.max_output_histogram_bytes_for_init_kernel_pdl == 8192); REQUIRE(custom_wide_counter_policy.max_output_histogram_bytes_for_init_kernel_pdl == 0); diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index a2dd366c8375..13b938f656f0 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1690,7 +1690,7 @@ struct histogram_tuning { constexpr auto sweep = cub::HistogramPrivatizationPolicy{BlockThreads, 1, 1, cub::BLOCK_LOAD_DIRECT, cub::LOAD_DEFAULT, false, false}; - return {sweep, sweep, sweep, 256, 256 * sizeof(unsigned int), 0, 0, 0, 0, 0, 0, 0}; + return {sweep, sweep, sweep, 256, 256 * sizeof(unsigned int), 0, 0, 0, 0, 0, 0, 0, 0}; } }; @@ -1720,6 +1720,7 @@ struct mixed_counter_histogram_tuning 28544 * sizeof(unsigned int) * 2, 19029 * sizeof(unsigned int) * 3, 8192 * sizeof(unsigned int) * 4, + 0, 0}; } }; @@ -1891,6 +1892,7 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) 4096, 8192, 16384, + 32768, 2048}; # if _CCCL_STD_VER >= 2020 @@ -1925,6 +1927,7 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) .max_privatized_dynamic_smem_2_channel_even_bytes = 4096, .max_privatized_dynamic_smem_3_channel_even_bytes = 8192, .max_privatized_dynamic_smem_4_channel_even_bytes = 16384, + .min_cached_search_gmem_range_bins = 32768, .max_output_histogram_bytes_for_init_kernel_pdl = 2048}; # else // _CCCL_STD_VER >= 2020 constexpr auto p2 = p1; @@ -1948,6 +1951,18 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" constexpr auto sm90_policy = selector_t{}(cuda::compute_capability{9, 0}); constexpr auto sm100_policy = selector_t{}(cuda::compute_capability{10, 0}); + constexpr auto sm90_range_u32_policy = + cub::detail::histogram::policy_selector_from_types{}( + cuda::compute_capability{9, 0}); + constexpr auto sm100_range_u32_policy = + cub::detail::histogram::policy_selector_from_types{}( + cuda::compute_capability{10, 0}); + constexpr auto sm100_range_f64_policy = + cub::detail::histogram::policy_selector_from_types{}( + cuda::compute_capability{10, 0}); + constexpr auto sm100_range_wide_counter_policy = + cub::detail::histogram::policy_selector_from_types{}( + cuda::compute_capability{10, 0}); constexpr auto sm100_wide_counter_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); @@ -1981,6 +1996,12 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" STATIC_REQUIRE(sm100_range_u64_policy.static_smem.threads_per_block == 384); STATIC_REQUIRE(sm100_range_u64_policy.static_smem.items_per_thread == 8); STATIC_REQUIRE(sm100_range_u64_policy.static_smem_min_blocks_per_sm == 3); + STATIC_REQUIRE(sm90_range_u32_policy.min_cached_search_gmem_range_bins == 0); + STATIC_REQUIRE(sm100_range_u32_policy.min_cached_search_gmem_range_bins == 1); + STATIC_REQUIRE(sm100_range_f64_policy.min_cached_search_gmem_range_bins == 1); + STATIC_REQUIRE(sm100_range_wide_counter_policy.min_cached_search_gmem_range_bins == 0); + STATIC_REQUIRE(sm100_range_u64_policy.min_cached_search_gmem_range_bins == 1); + STATIC_REQUIRE(cub::detail::histogram::use_cached_search_for_gmem_range(sm100_range_u32_policy, 1)); STATIC_REQUIRE(cub::detail::histogram::max_privatized_smem_bins( sm100_range_u64_policy.max_privatized_static_smem_single_channel_bytes) == 256); @@ -1988,6 +2009,9 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" constexpr auto sm100_multi_range_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); + constexpr auto sm100_multi_range_f64_policy = + cub::detail::histogram::policy_selector_from_types{}( + cuda::compute_capability{10, 0}); constexpr auto sm100_even_2ch_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); @@ -1997,6 +2021,11 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" constexpr auto sm100_even_4ch_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); + STATIC_REQUIRE(sm100_multi_range_policy.min_cached_search_gmem_range_bins == 16384); + STATIC_REQUIRE(sm100_multi_range_f64_policy.min_cached_search_gmem_range_bins == 16384); + STATIC_REQUIRE_FALSE(cub::detail::histogram::use_cached_search_for_gmem_range(sm100_multi_range_policy, 16383)); + STATIC_REQUIRE(cub::detail::histogram::use_cached_search_for_gmem_range(sm100_multi_range_policy, 16384)); + STATIC_REQUIRE(sm100_even_4ch_policy.min_cached_search_gmem_range_bins == 0); constexpr int expected_even_2ch_policy_bytes = 65536; constexpr int expected_even_3ch_policy_bytes = 98304; constexpr int expected_even_4ch_policy_bytes = 131072; diff --git a/cub/test/catch2_test_device_histogram_env_api.cu b/cub/test/catch2_test_device_histogram_env_api.cu index 07210b3a2481..c0759791cfa5 100644 --- a/cub/test/catch2_test_device_histogram_env_api.cu +++ b/cub/test/catch2_test_device_histogram_env_api.cu @@ -432,6 +432,7 @@ struct HistogramPolicySelector .max_privatized_dynamic_smem_2_channel_even_bytes = 0, .max_privatized_dynamic_smem_3_channel_even_bytes = 0, .max_privatized_dynamic_smem_4_channel_even_bytes = 0, + .min_cached_search_gmem_range_bins = 0, .max_output_histogram_bytes_for_init_kernel_pdl = 8192}; } }; From 47e869094b922866162409e4c7623ccb041c70e0 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sun, 30 Aug 2026 19:57:33 +0000 Subject: [PATCH 44/45] [cub] Respect device dynamic SMEM capacity --- .../device/dispatch/dispatch_histogram.cuh | 60 +++++++++++-------- cub/test/catch2_test_device_histogram_env.cu | 4 +- 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/cub/cub/device/dispatch/dispatch_histogram.cuh b/cub/cub/device/dispatch/dispatch_histogram.cuh index a6dd090d1b52..9d1402cbb77e 100644 --- a/cub/cub/device/dispatch/dispatch_histogram.cuh +++ b/cub/cub/device/dispatch/dispatch_histogram.cuh @@ -271,11 +271,12 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( { dynamic_smem_bytes += (num_privatized_levels[channel] - 1) * static_cast(kernel_source.CounterSize()); } + int max_dynamic_smem_bytes{}; NV_IF_ELSE_TARGET( NV_IS_HOST, ({ - if (const auto error = CubDebug(launcher_factory.set_max_dynamic_smem_size_for( - sweep_kernel, dynamic_smem_limit_bytes(active_policy)))) + if (const auto error = + CubDebug(launcher_factory.max_dynamic_smem_size_for(max_dynamic_smem_bytes, sweep_kernel))) { return error; } @@ -291,30 +292,39 @@ CUB_RUNTIME_FUNCTION _CCCL_VISIBILITY_HIDDEN _CCCL_FORCEINLINE auto dispatch( { return error; } - const int max_dynamic_smem_bytes = - max_shared_smem_bytes - static_cast(sweep_kernel_attributes.sharedSizeBytes); - if (dynamic_smem_bytes > max_dynamic_smem_bytes) - { - return detail::histogram:: - dispatch( - d_temp_storage, - temp_storage_bytes, - d_samples, - d_output_histograms, - num_privatized_levels, - num_output_levels, - first_level_array, - second_level_array, - max_num_output_bins, - num_row_pixels, - num_rows, - row_stride_samples, - stream, - policy_selector, - kernel_source, - launcher_factory); - } + max_dynamic_smem_bytes = max_shared_smem_bytes - static_cast(sweep_kernel_attributes.sharedSizeBytes); })) + if (dynamic_smem_bytes > max_dynamic_smem_bytes) + { + return detail::histogram:: + dispatch( + d_temp_storage, + temp_storage_bytes, + d_samples, + d_output_histograms, + num_privatized_levels, + num_output_levels, + first_level_array, + second_level_array, + max_num_output_bins, + num_row_pixels, + num_rows, + row_stride_samples, + stream, + policy_selector, + kernel_source, + launcher_factory); + } + NV_IF_TARGET(NV_IS_HOST, ({ + const int opt_in_dynamic_smem_bytes = + (::cuda::std::min) (dynamic_smem_limit_bytes(active_policy), + max_dynamic_smem_bytes); + if (const auto error = CubDebug( + launcher_factory.set_max_dynamic_smem_size_for(sweep_kernel, opt_in_dynamic_smem_bytes))) + { + return error; + } + })) } // Get SM count diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index 13b938f656f0..8ce5c751e052 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -753,8 +753,8 @@ CUB_TEST("DeviceHistogram::MultiHistogramEven handles the device-launch dynamic- SKIP("The runtime-sized shared-memory histogram policy is currently tuned for SM100"); } - constexpr int num_channels = 4; - constexpr int num_active_channels = 3; + [[maybe_unused]] constexpr int num_channels = 4; + constexpr int num_active_channels = 3; // The direct-load kernel has no static shared-memory footprint, so 4,096 // three-channel counters exactly fill the B200's 48 KiB device-launch limit. constexpr int num_bins = 4096; From e59377ebe2ad1691d33ff630249a6a155465c361 Mon Sep 17 00:00:00 2001 From: Bryce Adelstein Lelbach Date: Sun, 30 Aug 2026 23:20:46 +0000 Subject: [PATCH 45/45] [cub] Add an SM120 histogram SMEM budget --- .../dispatch/tuning/tuning_histogram.cuh | 26 ++++++++++++++----- cub/test/catch2_test_device_histogram_env.cu | 18 ++++++++++++- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh index 12a15b6d0b8f..2af37486d959 100644 --- a/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +++ b/cub/cub/device/dispatch/tuning/tuning_histogram.cuh @@ -247,10 +247,12 @@ private: public: [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto operator()(::cuda::compute_capability cc) const -> HistogramPolicy { - // SM100 and newer use the autoresearch launch shapes and dynamic-SMEM byte budgets. + // SM100 and SM120 use the autoresearch launch shapes. Their dynamic-SMEM budgets differ because SM100 permits + // 227 KiB per block while SM120 permits 99 KiB per block. if (cc >= ::cuda::compute_capability{10, 0}) { - const bool single_channel = num_channels == 1 && num_active_channels == 1; + const bool is_sm120_or_newer = cc >= ::cuda::compute_capability{12, 0}; + const bool single_channel = num_channels == 1 && num_active_channels == 1; auto gmem = HistogramPrivatizationPolicy{384, t_scale(16), 4, BLOCK_LOAD_DIRECT, LOAD_LDG, true, false}; // Single-channel primitive samples with 32-bit counters use their per-sample-width tuning. @@ -307,13 +309,16 @@ public: // All storage thresholds are byte budgets. Dispatch derives the corresponding // bin limits from the local counter width and active channel count. constexpr int max_privatized_static_smem_bytes = 1024; - constexpr int max_privatized_dynamic_smem_single_channel_bytes = 228352; + constexpr int max_privatized_dynamic_smem_sm100_bytes = 228352; + constexpr int max_privatized_dynamic_smem_sm120_bytes = 99 * 1024; constexpr int max_privatized_dynamic_smem_range_bytes_per_channel = 8192; constexpr int max_privatized_dynamic_smem_even_bytes_per_channel = 32768; constexpr int min_cached_search_gmem_single_channel_range_bins = 1; constexpr int min_cached_search_gmem_multi_channel_range_bins = 16384; constexpr int init_threads_per_block = 256; constexpr int max_output_histogram_bytes_for_init_kernel_pdl = 8192; + const int max_privatized_dynamic_smem_single_channel_bytes = + is_sm120_or_newer ? max_privatized_dynamic_smem_sm120_bytes : max_privatized_dynamic_smem_sm100_bytes; const bool supports_dynamic_smem = counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive; @@ -327,10 +332,17 @@ public: has_single_channel_dynamic_smem || (has_multi_channel_dynamic_smem && !is_even) ? max_privatized_dynamic_smem_range_bytes_per_channel * num_active_channels : 0; - int dynamic_smem_multi_channel_even_bytes = - has_multi_channel_dynamic_smem && is_even - ? max_privatized_dynamic_smem_even_bytes_per_channel * num_active_channels - : 0; + int dynamic_smem_multi_channel_even_bytes = 0; + if (has_multi_channel_dynamic_smem && is_even) + { + dynamic_smem_multi_channel_even_bytes = + max_privatized_dynamic_smem_even_bytes_per_channel * num_active_channels; + if (is_sm120_or_newer) + { + dynamic_smem_multi_channel_even_bytes = + (::cuda::std::min) (dynamic_smem_multi_channel_even_bytes, max_privatized_dynamic_smem_sm120_bytes); + } + } const int init_kernel_pdl_trigger_bytes = single_channel && counter_size_bytes == int{sizeof(::cuda::std::uint32_t)} && sample_is_primitive && (sample_size_bytes == 1 || sample_size_bytes == 2 || sample_size_bytes == 4 || sample_size_bytes == 8) diff --git a/cub/test/catch2_test_device_histogram_env.cu b/cub/test/catch2_test_device_histogram_env.cu index 8ce5c751e052..1fdb04e8fe78 100644 --- a/cub/test/catch2_test_device_histogram_env.cu +++ b/cub/test/catch2_test_device_histogram_env.cu @@ -1945,12 +1945,13 @@ CUB_TEST("Test HistogramPolicy properties", "[histogram][device]", CUB_SMALL) REQUIRE(to_string(p1) == to_string(p2)); } -CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget", "[histogram][device]", CUB_SMALL) +CUB_TEST("Histogram architecture policies carry their dynamic shared-memory budgets", "[histogram][device]", CUB_SMALL) { using selector_t = cub::detail::histogram::policy_selector_from_types; constexpr auto sm90_policy = selector_t{}(cuda::compute_capability{9, 0}); constexpr auto sm100_policy = selector_t{}(cuda::compute_capability{10, 0}); + constexpr auto sm120_policy = selector_t{}(cuda::compute_capability{12, 0}); constexpr auto sm90_range_u32_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{9, 0}); @@ -2021,6 +2022,15 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" constexpr auto sm100_even_4ch_policy = cub::detail::histogram::policy_selector_from_types{}( cuda::compute_capability{10, 0}); + constexpr auto sm120_even_2ch_policy = + cub::detail::histogram::policy_selector_from_types{}( + cuda::compute_capability{12, 0}); + constexpr auto sm120_even_3ch_policy = + cub::detail::histogram::policy_selector_from_types{}( + cuda::compute_capability{12, 0}); + constexpr auto sm120_even_4ch_policy = + cub::detail::histogram::policy_selector_from_types{}( + cuda::compute_capability{12, 0}); STATIC_REQUIRE(sm100_multi_range_policy.min_cached_search_gmem_range_bins == 16384); STATIC_REQUIRE(sm100_multi_range_f64_policy.min_cached_search_gmem_range_bins == 16384); STATIC_REQUIRE_FALSE(cub::detail::histogram::use_cached_search_for_gmem_range(sm100_multi_range_policy, 16383)); @@ -2044,6 +2054,12 @@ CUB_TEST("Histogram SM100 policy carries the tuned dynamic shared-memory budget" cub::detail::histogram::dynamic_smem_limit_bytes(sm100_even_3ch_policy) == expected_even_3ch_limit_bytes); STATIC_REQUIRE( cub::detail::histogram::dynamic_smem_limit_bytes(sm100_even_4ch_policy) == expected_even_4ch_limit_bytes); + constexpr int expected_sm120_dynamic_smem_bytes = 99 * 1024; + STATIC_REQUIRE(sm120_policy.max_privatized_dynamic_smem_single_channel_bytes == expected_sm120_dynamic_smem_bytes); + STATIC_REQUIRE(sm120_even_2ch_policy.max_privatized_dynamic_smem_2_channel_even_bytes == 65536); + STATIC_REQUIRE(sm120_even_3ch_policy.max_privatized_dynamic_smem_3_channel_even_bytes == 98304); + STATIC_REQUIRE( + sm120_even_4ch_policy.max_privatized_dynamic_smem_4_channel_even_bytes == expected_sm120_dynamic_smem_bytes); STATIC_REQUIRE(sm100_multi_range_policy.gmem.threads_per_block == 384); STATIC_REQUIRE(sm100_multi_range_policy.gmem.items_per_thread == 5); STATIC_REQUIRE(sm100_multi_range_policy.static_smem.threads_per_block == 384);