diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_intent/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_intent/Cargo.lock index c641aba6d..cb8ef2d6d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_intent/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_intent/Cargo.lock @@ -184,6 +184,7 @@ dependencies = [ name = "cuda-macros" version = "0.2.1" dependencies = [ + "oxide-tile-export", "proc-macro2", "quote", "reserved-oxide-symbols", @@ -311,6 +312,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + [[package]] name = "lazy_static" version = "1.5.0" @@ -417,6 +424,15 @@ dependencies = [ "object", ] +[[package]] +name = "oxide-tile-export" +version = "0.2.1" +dependencies = [ + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -531,6 +547,19 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "sha2" version = "0.10.9" @@ -716,3 +745,9 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/cuda-oxide/cuda-oxide-book/gpu-programming/images/inline-intent-mapping.svg b/cuda-oxide/cuda-oxide-book/gpu-programming/images/inline-intent-mapping.svg new file mode 100644 index 000000000..ecbfc22b8 --- /dev/null +++ b/cuda-oxide/cuda-oxide-book/gpu-programming/images/inline-intent-mapping.svg @@ -0,0 +1,125 @@ + + + + + +An inline request now reaches the GPU as the matching LLVM attribute +Only #[inline(always)] used to survive. Now #[inline] is inlinehint, #[inline(always)] is alwaysinline and #[inline(never)] is noinline, as in rustc's own backend, so a never helper stays a real PTX function. +RUST SOURCE (the inline_intent example) +RECORDED, THEN LOWERED (mir.func, then the define line) +WHAT OPT AND LLC MAKE OF IT (PTX, sm_120a) + +one helper of each kind +#[inline(never)] +pub fn never_add(a: f32, b: f32) -> f32 { + a + b +} +#[inline] +pub fn hint_mul(a: f32, b: f32) -> f32 { + a * b +} +#[inline(always)] +pub fn always_sub(a: f32, b: f32) -> f32 { + a - b +} +src/main.rs: plain functions the collector +reaches from the kernel. + +inside the #[cuda_module] +#[inline(never)] +#[device] +pub fn never_dev(a: f32, b: f32) -> f32 { + a * 2.0 + b +} +#[device] +pub fn scale_by<const N: u32>(x: f32) -> f32 { + x * (N as f32) +} +scale_by has its own const parameter, so +#[device] adds an #[inline(never)] of its own +there, for the host-side collector. + +the kernel's arithmetic +let mut v = never_add(x, 1.0); +v = hint_mul(v, 2.0); +v = always_sub(v, 1.0); +v += kernels::never_dev(x, 1.0); +v + kernels::scale_by::<4>(x) +Reached through one #[inline(always)] helper. +MIR inlining is off for this package, so every +helper reaches LLVM as its own definition. + +the request, as the pipeline carries it +source mir.func LLVM +#[inline] inline_intent=hint inlinehint +#[inline(always)] inline_intent=always alwaysinline +#[inline(never)] inline_intent=never noinline +no attribute none none +rustc-codegen-cuda reads codegen_fn_attrs and sets a typed +InlineIntentAttr on mir.func. mir-lower names the keyword in +one place. A kernel entry never carries one: verifier error. + +LLVM IR before opt (inline_intent.ll) +define float @inline_intent__never_add(..) noinline #0 +define float @inline_intent__hint_mul(..) inlinehint #0 +define float @inline_intent__always_sub(..) alwaysinline #0 +define float @never_dev(..) noinline #0 +define float @scale_byKm4_EB4_(float %v0) #0 +(..) stands for (float %v0, float %v1). The last line is +scale_by::<4>: no keyword, on purpose (next box). + +The one request that is not carried +#[device] puts #[inline(never)] on a helper with its own +type or const parameters so each monomorphization stays +visible to the host-side collector. That is a host need, +not a device intent, so the backend drops it and N keeps +folding into the kernel. A user's own attribute on such a +helper still does not reach device code: #1471 stays open. + +PTX, inside .entry intent (opt -O2, then llc) +call.uni (retval0), inline_intent__never_add, (param0); +call.uni (retval0), never_dev, (param0, param1); +fma.rn.f32 %r13, %r8, 0f40800000, %r12; +Two real calls, one per noinline helper. 0f40800000 is +4.0: scale_by::<4> folded into the entry. always_sub +appears nowhere, alwaysinline left no function. hint_mul +was inlined by opt, as the hint allows. + +.func headers left in the module +.func (..) inline_intent__never_add( +.visible .func (..) never_dev( +.visible .func (..) scale_byKm4_EB4_( +(..) stands for (.param .b32 func_retval0). scale_by stays +only because every #[device] function is kept alive through +@llvm.used; the entry does not call it. + +RTX 5090: SUCCESS, 256 of 256 match the host reference. +verify-code-shape.sh checks these lines in compile-only CI. + +Before, on main +No keyword on any of these define lines. opt inlined +the never helpers like the rest, so a helper that asked +to stay a separate GPU function never did, and #[inline] +was dropped. Only alwaysinline came through, as a string +attribute the exporter matched by name. + + + +The libNVVM route needed one more piece (nvvm-transforms, legalize_pointer_returns) +A noinline helper that returns a shared-memory pointer is now a real function. libNVVM declares that 32-bit return slot as .param .b64 while the body stores 32 bits and the call site reads 32, +so ptxas rejected addressof_sharedarray on --emit-nvvm-ir. The pass gives such a definition a generic pointer return and an addrspacecast on both sides of each call; parameters, aggregates, +kernels and declarations are untouched, and a function whose address is taken is reported. The libNVVM defect is tracked with its team; the pass goes when the fix ships. + diff --git a/cuda-oxide/cuda-oxide-book/gpu-programming/kernels-and-device-functions.md b/cuda-oxide/cuda-oxide-book/gpu-programming/kernels-and-device-functions.md index 3d89ead39..303334a50 100644 --- a/cuda-oxide/cuda-oxide-book/gpu-programming/kernels-and-device-functions.md +++ b/cuda-oxide/cuda-oxide-book/gpu-programming/kernels-and-device-functions.md @@ -191,6 +191,30 @@ pub fn magnitude(x: f32, y: f32) -> f32 { | Callable from device code| Yes | Yes | Yes | | Annotation required | Always | Only for standalone/cross-crate/FFI | Never | +### Inlining helpers + +A helper's `#[inline]` request reaches the GPU the same way it reaches the host +backend. `#[inline]` becomes the LLVM `inlinehint`, `#[inline(always)]` +becomes `alwaysinline`, and `#[inline(never)]` becomes `noinline`, so a helper +marked `never` stays a separate `.func` in the PTX and is reached with a real +`call.uni`. One exception: the `#[inline(never)]` that `#[device]` adds to a +helper with its own type or const parameters is a host-side collector boundary, +not a device intent, so it is not carried and const-generic arguments keep +folding into the kernel. + +```{figure} images/inline-intent-mapping.svg +:align: center +:width: 100% + +How an inline request reaches the GPU. Left, one helper of each kind from the +`inline_intent` example. Middle, the request recorded on `mir.func` as a typed +`inline_intent` attribute and lowered to the LLVM function attribute in one +place. Right, what opt and llc make of it: `call.uni` for the `noinline` +helpers, no function left for the `alwaysinline` one, and the const-generic +factor folded to an immediate. Before this mapping only `alwaysinline` +survived, and LLVM inlined a `never` helper anyway. +``` + ## What Rust works on the GPU cuda-oxide compiles standard Rust through `rustc` -- it is not a subset