diff --git a/.github/workflows/examples-compile.yml b/.github/workflows/examples-compile.yml index d9ab5a9548..ecc3c8989a 100644 --- a/.github/workflows/examples-compile.yml +++ b/.github/workflows/examples-compile.yml @@ -256,6 +256,51 @@ jobs: "${RUNNER_TEMP}/cross-crate-sm90-build.log" \ '^(kernel-lib|cross_crate_kernel)$' kernel-lib cross_crate_kernel + # With LTO, rustc reads bitcode from the Rust objects of each rlib and + # removes these objects from the link. The artifact objects of the + # libraries must not be Rust objects. + - name: LTO keeps the PTX bundles of libraries + timeout-minutes: 15 + working-directory: cuda-oxide + env: + CUDA_TOOLKIT_PATH: ${{ steps.cuda-toolkit.outputs.CUDA_PATH }} + CUDA_OXIDE_LLC: /usr/bin/llc-23 + CARGO_TARGET_DIR: ${{ runner.temp }}/examples-shared-target + CARGO_INCREMENTAL: "0" + CARGO_PROFILE_RELEASE_LTO: fat + run: | + set -euo pipefail + sysroot="$(rustc --print sysroot)" + export LIBRARY_PATH="${sysroot}/lib:${LIBRARY_PATH:-}" + export LD_LIBRARY_PATH="${CUDA_TOOLKIT_PATH}/lib64:${sysroot}/lib:${LD_LIBRARY_PATH:-}" + + assert_bundles() { + local file="$1" + shift + local section="${RUNNER_TEMP}/$(basename "${file}").oxart" + objcopy --dump-section ".oxart=${section}" "${file}" + local bundle + for bundle in "$@"; do + grep -aFq "${bundle}" "${section}" + done + } + binary="${CARGO_TARGET_DIR}/release/cuda_module_in_lib" + + cargo oxide build cuda_module_in_lib --arch sm_86 2>&1 \ + | tee "${RUNNER_TEMP}/cross-crate-lto-build.log" + assert_bundles "${binary}" cuda_module_in_lib module-kernels + + # If a session writes an rlib and a dynamic library, the system + # linker gets the artifact objects as link arguments. + manifest=crates/rustc-codegen-cuda/examples/cuda_module_in_lib/kernel-lib/Cargo.toml + printf '\n[lib]\ncrate-type = ["rlib", "cdylib"]\n' >> "${manifest}" + cargo oxide build cuda_module_in_lib --arch sm_86 2>&1 \ + | tee "${RUNNER_TEMP}/cross-crate-lto-cdylib-build.log" + git checkout -- "${manifest}" + assert_bundles "${binary}" cuda_module_in_lib module-kernels + assert_bundles \ + "$(find "${CARGO_TARGET_DIR}/release" -name libmodule_kernels.so)" module-kernels + - name: Upload failure logs if: failure() uses: actions/upload-artifact@v4 diff --git a/Cargo.lock b/Cargo.lock index ca0b7d7ad7..156ca2c807 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -498,8 +498,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-core/Cargo.toml b/cuda-core/Cargo.toml index 98bf503077..5dd40ecf84 100644 --- a/cuda-core/Cargo.toml +++ b/cuda-core/Cargo.toml @@ -17,7 +17,7 @@ cuda-bindings = { version = "=0.5.0", path = "../cuda-bindings" } cuda-core-derive = { version = "=0.5.0", path = "../cuda-core-derive" } anyhow = "1" half = { version = "2" } -oxide-artifacts = { version = "0.2.1", features = ["object-read"] } +oxide-artifacts = { version = "0.2.1", path = "../cuda-oxide/crates/oxide-artifacts", features = ["object-read"] } [dev-dependencies] trybuild = "1" diff --git a/cuda-core/src/simt/embedded.rs b/cuda-core/src/simt/embedded.rs index c146f58753..d63fed04c4 100644 --- a/cuda-core/src/simt/embedded.rs +++ b/cuda-core/src/simt/embedded.rs @@ -46,8 +46,19 @@ impl EmbeddedModule { pub fn load(&self, ctx: &Arc) -> Result, EmbeddedModuleError> { let image = loadable_payload(&self.bundle).expect("EmbeddedModule always has a loadable payload"); - ctx.load_module_from_image(image) - .map_err(EmbeddedModuleError::Driver) + ctx.load_module_from_image_with_launch_facts( + image, + self.bundle + .entries + .iter() + .filter_map(|entry| { + entry + .launch_facts + .map(|facts| (entry.symbol.clone(), facts)) + }) + .collect(), + ) + .map_err(EmbeddedModuleError::Driver) } } diff --git a/cuda-core/src/simt/launch.rs b/cuda-core/src/simt/launch.rs index ff1f37684b..49503c2974 100644 --- a/cuda-core/src/simt/launch.rs +++ b/cuda-core/src/simt/launch.rs @@ -64,6 +64,9 @@ mod sealed { pub trait KernelLaunchConfig: sealed::Sealed + Copy { #[doc(hidden)] fn __raw(self) -> LaunchConfig; + + #[doc(hidden)] + fn __auto_shared(self) -> bool; } /// A one-dimensional launch configuration. @@ -73,10 +76,20 @@ pub trait KernelLaunchConfig: sealed::Sealed + Copy { pub struct LaunchConfig1D { grid_x: u32, block_x: u32, - shared_mem_bytes: u32, + shared_mem_bytes: Option, } impl LaunchConfig1D { + /// Uses the kernel's prepared resource requirement for dynamic shared memory. + /// A range contract selects its minimum, raised by compiled facts if present. + pub const fn auto_shared(grid_x: u32, block_x: u32) -> Self { + Self { + grid_x, + block_x, + shared_mem_bytes: None, + } + } + /// Creates a one-dimensional launch configuration. /// /// Zero dimensions are reported when the configuration is prepared for a @@ -85,7 +98,7 @@ impl LaunchConfig1D { Self { grid_x, block_x, - shared_mem_bytes, + shared_mem_bytes: Some(shared_mem_bytes), } } } @@ -93,11 +106,15 @@ impl LaunchConfig1D { impl sealed::Sealed for LaunchConfig1D {} impl KernelLaunchConfig for LaunchConfig1D { + fn __auto_shared(self) -> bool { + self.shared_mem_bytes.is_none() + } + fn __raw(self) -> LaunchConfig { LaunchConfig { grid_dim: (self.grid_x, 1, 1), block_dim: (self.block_x, 1, 1), - shared_mem_bytes: self.shared_mem_bytes, + shared_mem_bytes: self.shared_mem_bytes.unwrap_or(0), } } } @@ -109,16 +126,26 @@ impl KernelLaunchConfig for LaunchConfig1D { pub struct LaunchConfig2D { grid: (u32, u32), block: (u32, u32), - shared_mem_bytes: u32, + shared_mem_bytes: Option, } impl LaunchConfig2D { + /// Uses the kernel's prepared resource requirement for dynamic shared memory. + /// A range contract selects its minimum, raised by compiled facts if present. + pub const fn auto_shared(grid: (u32, u32), block: (u32, u32)) -> Self { + Self { + grid, + block, + shared_mem_bytes: None, + } + } + /// Creates a two-dimensional launch configuration. pub const fn new(grid: (u32, u32), block: (u32, u32), shared_mem_bytes: u32) -> Self { Self { grid, block, - shared_mem_bytes, + shared_mem_bytes: Some(shared_mem_bytes), } } } @@ -126,11 +153,15 @@ impl LaunchConfig2D { impl sealed::Sealed for LaunchConfig2D {} impl KernelLaunchConfig for LaunchConfig2D { + fn __auto_shared(self) -> bool { + self.shared_mem_bytes.is_none() + } + fn __raw(self) -> LaunchConfig { LaunchConfig { grid_dim: (self.grid.0, self.grid.1, 1), block_dim: (self.block.0, self.block.1, 1), - shared_mem_bytes: self.shared_mem_bytes, + shared_mem_bytes: self.shared_mem_bytes.unwrap_or(0), } } } @@ -140,16 +171,26 @@ impl KernelLaunchConfig for LaunchConfig2D { pub struct LaunchConfig3D { grid: (u32, u32, u32), block: (u32, u32, u32), - shared_mem_bytes: u32, + shared_mem_bytes: Option, } impl LaunchConfig3D { + /// Uses the kernel's prepared resource requirement for dynamic shared memory. + /// A range contract selects its minimum, raised by compiled facts if present. + pub const fn auto_shared(grid: (u32, u32, u32), block: (u32, u32, u32)) -> Self { + Self { + grid, + block, + shared_mem_bytes: None, + } + } + /// Creates a three-dimensional launch configuration. pub const fn new(grid: (u32, u32, u32), block: (u32, u32, u32), shared_mem_bytes: u32) -> Self { Self { grid, block, - shared_mem_bytes, + shared_mem_bytes: Some(shared_mem_bytes), } } } @@ -157,11 +198,15 @@ impl LaunchConfig3D { impl sealed::Sealed for LaunchConfig3D {} impl KernelLaunchConfig for LaunchConfig3D { + fn __auto_shared(self) -> bool { + self.shared_mem_bytes.is_none() + } + fn __raw(self) -> LaunchConfig { LaunchConfig { grid_dim: self.grid, block_dim: self.block, - shared_mem_bytes: self.shared_mem_bytes, + shared_mem_bytes: self.shared_mem_bytes.unwrap_or(0), } } } @@ -188,6 +233,9 @@ pub enum BlockRequirement { /// The dynamic shared-memory sizes accepted by a kernel. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DynamicSharedMemoryRequirement { + /// Use the finalized allocation and alignment from this concrete kernel's + /// compiled artifact. Preparation fails when those facts are absent. + Inferred, /// The launch must allocate exactly `bytes` bytes. Exact { /// Required bytes per block. @@ -429,6 +477,11 @@ pub enum LaunchContractError { /// Number of coordinate values required on this axis. positions: u64, }, + /// An inferred contract was loaded without compiled resource facts. + MissingCompiledLaunchFacts { + /// Kernel being prepared. + kernel: &'static str, + }, /// A shared-memory alignment was not a nonzero power of two. InvalidSharedMemoryAlignment { /// Kernel being prepared. @@ -671,6 +724,8 @@ impl Display for LaunchContractError { f, "{kernel}: Grid.{axis:?} {grid} * Block.{axis:?} {block} requires {positions} coordinates, exceeding the u32 range" ), + Self::MissingCompiledLaunchFacts { kernel } => write!(f, + "{kernel}: compiled launch facts are missing; rebuild this kernel with the CUDA compiler"), Self::InvalidSharedMemoryAlignment { kernel, alignment } => write!( f, "{kernel}: dynamic shared-memory alignment {alignment} is not a nonzero power of two" @@ -893,15 +948,25 @@ impl PreparedLaunch { /// /// `C::SPEC` must truthfully describe the compiled device function, /// including its rank, fixed block assumptions, shared-memory layout, and - /// launch mode. cuda-oxide's macros generate that marker and uphold this - /// relationship. + /// launch mode. Any `function.launch_facts()` must truthfully describe this + /// exact loaded image and entry; inferred contracts obtain their allocation + /// from those facts. cuda-oxide's macros and matching artifact loader uphold + /// this relationship. #[doc(hidden)] pub unsafe fn __prepare( function: CudaFunction, config: C::Config, ) -> Result { - let raw = config.__raw(); - validate_static(C::SPEC, raw)?; + let spec = resolve_compiled_spec(C::SPEC, function.launch_facts())?; + let mut raw = config.__raw(); + if config.__auto_shared() { + raw.shared_mem_bytes = match spec.dynamic_shared_memory { + DynamicSharedMemoryRequirement::Exact { bytes, .. } => bytes, + DynamicSharedMemoryRequirement::Range { min_bytes, .. } => min_bytes, + DynamicSharedMemoryRequirement::Inferred => unreachable!("resolved above"), + }; + } + validate_static(spec, raw)?; let context = function.context(); let limits = context.launch_limits()?; @@ -909,17 +974,17 @@ impl PreparedLaunch { let static_shared = function.static_shared_memory_bytes()?; let function_max_dynamic = function.max_dynamic_shared_memory_bytes()?; - validate_live_shape(C::SPEC, raw, limits, function_max_threads)?; + validate_live_shape(spec, raw, limits, function_max_threads)?; // Configure the immutable contract maximum, not this particular // launch's chosen size. Concurrent preparations of two range values // therefore write the same function attribute and cannot lower the // maximum underneath an already prepared launch. - let contract_dynamic_max = dynamic_shared_memory_max(C::SPEC.dynamic_shared_memory); + let contract_dynamic_max = dynamic_shared_memory_max(spec.dynamic_shared_memory); let total_shared = - shared_memory_total(C::SPEC.kernel_name, static_shared, contract_dynamic_max)?; + shared_memory_total(spec.kernel_name, static_shared, contract_dynamic_max)?; if validate_shared_memory_limit( - C::SPEC.kernel_name, + spec.kernel_name, total_shared, limits.max_shared_memory_per_block, false, @@ -927,10 +992,10 @@ impl PreparedLaunch { .is_err() { let opt_in_max = context.max_opt_in_shared_memory_per_block()?; - validate_shared_memory_limit(C::SPEC.kernel_name, total_shared, opt_in_max, true)?; + validate_shared_memory_limit(spec.kernel_name, total_shared, opt_in_max, true)?; } - if let Some(required) = C::SPEC.min_compute_capability { + if let Some(required) = spec.min_compute_capability { let (major, minor) = context.compute_capability()?; let actual = ( u32::try_from(major).map_err(|_| { @@ -940,23 +1005,20 @@ impl PreparedLaunch { DriverError(cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_VALUE) })?, ); - validate_compute_capability(C::SPEC.kernel_name, required, actual)?; + validate_compute_capability(spec.kernel_name, required, actual)?; } - if let Some(cluster) = C::SPEC.cluster { - validate_cluster_support(C::SPEC.kernel_name, context.supports_cluster_launch()?)?; + if let Some(cluster) = spec.cluster { + validate_cluster_support(spec.kernel_name, context.supports_cluster_launch()?)?; validate_required_cluster( - C::SPEC.kernel_name, + spec.kernel_name, cluster, function.required_cluster_dimensions()?, )?; } - if C::SPEC.cooperative { - validate_cooperative_support( - C::SPEC.kernel_name, - context.supports_cooperative_launch()?, - )?; + if spec.cooperative { + validate_cooperative_support(spec.kernel_name, context.supports_cooperative_launch()?)?; } // Perform the only persistent function mutation after all checks that @@ -973,15 +1035,15 @@ impl PreparedLaunch { // `cuOccupancyMaxActiveClusters`. let mut clustered_resident_blocks: Option = None; - if let Some(cluster) = C::SPEC.cluster { + if let Some(cluster) = spec.cluster { let max_cluster_size = function.max_potential_cluster_size( raw.grid_dim, raw.block_dim, raw.shared_mem_bytes, )?; let cluster_blocks = - shape_product(C::SPEC.kernel_name, LaunchDimension::Cluster, cluster)?; - validate_cluster_size(C::SPEC.kernel_name, cluster_blocks, max_cluster_size)?; + shape_product(spec.kernel_name, LaunchDimension::Cluster, cluster)?; + validate_cluster_size(spec.kernel_name, cluster_blocks, max_cluster_size)?; let active_clusters = match function.max_active_clusters( raw.grid_dim, @@ -994,13 +1056,13 @@ impl PreparedLaunch { if error.0 == cuda_bindings::cudaError_enum_CUDA_ERROR_INVALID_CLUSTER_SIZE => { return Err(LaunchContractError::ClusterShapeUnsupported { - kernel: C::SPEC.kernel_name, + kernel: spec.kernel_name, cluster, }); } Err(error) => return Err(error.into()), }; - validate_cluster_residency(C::SPEC.kernel_name, cluster, active_clusters)?; + validate_cluster_residency(spec.kernel_name, cluster, active_clusters)?; // Deliberate error reuse: `active_clusters * cluster_blocks` is a // device-capacity product over the cluster dimension, and // `DimensionProductOverflow { Cluster }` is the closest existing @@ -1012,19 +1074,19 @@ impl PreparedLaunch { u64::from(active_clusters) .checked_mul(cluster_blocks) .ok_or(LaunchContractError::DimensionProductOverflow { - kernel: C::SPEC.kernel_name, + kernel: spec.kernel_name, dimension: LaunchDimension::Cluster, })?, ); } - if C::SPEC.cooperative { - let blocks = shape_product(C::SPEC.kernel_name, LaunchDimension::Grid, raw.grid_dim)?; + if spec.cooperative { + let blocks = shape_product(spec.kernel_name, LaunchDimension::Grid, raw.grid_dim)?; let resident_capacity = if let Some(capacity) = clustered_resident_blocks { capacity } else { let block_threads = - shape_product(C::SPEC.kernel_name, LaunchDimension::Block, raw.block_dim)?; + shape_product(spec.kernel_name, LaunchDimension::Block, raw.block_dim)?; let active_per_sm = function.max_active_blocks_per_multiprocessor( block_threads as u32, raw.shared_mem_bytes, @@ -1032,7 +1094,7 @@ impl PreparedLaunch { let multiprocessors = context.multiprocessor_count()?; u64::from(active_per_sm) * u64::from(multiprocessors) }; - validate_cooperative_residency(C::SPEC.kernel_name, blocks, resident_capacity)?; + validate_cooperative_residency(spec.kernel_name, blocks, resident_capacity)?; } Ok(Self { @@ -1072,6 +1134,85 @@ impl PreparedLaunch { } } +fn resolve_compiled_spec( + mut spec: LaunchContractSpec, + facts: Option, +) -> Result { + let Some(facts) = facts else { + return if spec.dynamic_shared_memory == DynamicSharedMemoryRequirement::Inferred { + Err(LaunchContractError::MissingCompiledLaunchFacts { + kernel: spec.kernel_name, + }) + } else { + Ok(spec) + }; + }; + if let Some(required) = facts.exact_block { + match spec.block { + BlockRequirement::Exact(declared) if declared != required => { + return Err(LaunchContractError::BlockShapeMismatch { + kernel: spec.kernel_name, + required, + actual: declared, + }); + } + BlockRequirement::MaxThreads(max) => { + let actual = shape_product(spec.kernel_name, LaunchDimension::Block, required)?; + if actual > u64::from(max) { + return Err(LaunchContractError::BlockThreadsExceedContract { + kernel: spec.kernel_name, + actual, + max, + }); + } + } + _ => {} + } + spec.block = BlockRequirement::Exact(required); + } + spec.dynamic_shared_memory = match spec.dynamic_shared_memory { + DynamicSharedMemoryRequirement::Inferred => DynamicSharedMemoryRequirement::Exact { + bytes: facts.dynamic_shared_bytes, + min_alignment: facts.dynamic_shared_alignment, + }, + DynamicSharedMemoryRequirement::Exact { + bytes, + min_alignment, + } => { + if bytes < facts.dynamic_shared_bytes { + return Err(LaunchContractError::DynamicSharedMemoryExactMismatch { + kernel: spec.kernel_name, + required: facts.dynamic_shared_bytes, + actual: bytes, + }); + } + DynamicSharedMemoryRequirement::Exact { + bytes, + min_alignment: min_alignment.max(facts.dynamic_shared_alignment), + } + } + DynamicSharedMemoryRequirement::Range { + min_bytes, + max_bytes, + min_alignment, + } => { + if max_bytes < facts.dynamic_shared_bytes { + return Err(LaunchContractError::DynamicSharedMemoryExactMismatch { + kernel: spec.kernel_name, + required: facts.dynamic_shared_bytes, + actual: max_bytes, + }); + } + DynamicSharedMemoryRequirement::Range { + min_bytes: min_bytes.max(facts.dynamic_shared_bytes), + max_bytes, + min_alignment: min_alignment.max(facts.dynamic_shared_alignment), + } + } + }; + Ok(spec) +} + fn validate_static( spec: LaunchContractSpec, config: LaunchConfig, @@ -1111,6 +1252,11 @@ fn validate_static( } match spec.dynamic_shared_memory { + DynamicSharedMemoryRequirement::Inferred => { + return Err(LaunchContractError::MissingCompiledLaunchFacts { + kernel: spec.kernel_name, + }) + } DynamicSharedMemoryRequirement::Exact { bytes, min_alignment, @@ -1250,6 +1396,9 @@ fn shared_memory_total( const fn dynamic_shared_memory_max(requirement: DynamicSharedMemoryRequirement) -> u32 { match requirement { + DynamicSharedMemoryRequirement::Inferred => { + panic!("inferred requirements must be resolved before validation") + } DynamicSharedMemoryRequirement::Exact { bytes, .. } => bytes, DynamicSharedMemoryRequirement::Range { max_bytes, .. } => max_bytes, } @@ -1482,6 +1631,120 @@ mod tests { } } + #[test] + fn compiled_facts_resolve_inference_and_enforce_shape_and_size() { + let source = LaunchContractSpec::new( + "fixed", + BlockRequirement::MaxThreads(32), + DynamicSharedMemoryRequirement::Inferred, + ); + assert!(matches!( + resolve_compiled_spec(source, None), + Err(LaunchContractError::MissingCompiledLaunchFacts { .. }) + )); + let facts = oxide_artifacts::KernelLaunchFacts { + exact_block: Some((8, 4, 1)), + dynamic_shared_bytes: 512, + dynamic_shared_alignment: 64, + }; + let effective = resolve_compiled_spec(source, Some(facts)).unwrap(); + let config = LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (8, 4, 1), + shared_mem_bytes: 512, + }; + assert!(validate_static(effective, config).is_ok()); + assert!(matches!( + validate_static( + effective, + LaunchConfig { + block_dim: (32, 1, 1), + ..config + } + ), + Err(LaunchContractError::BlockShapeMismatch { .. }) + )); + assert!(matches!( + validate_static( + effective, + LaunchConfig { + shared_mem_bytes: 511, + ..config + } + ), + Err(LaunchContractError::DynamicSharedMemoryExactMismatch { .. }) + )); + let empty = resolve_compiled_spec( + source, + Some(oxide_artifacts::KernelLaunchFacts { + dynamic_shared_bytes: 0, + ..facts + }), + ) + .unwrap(); + assert!(validate_static( + empty, + LaunchConfig { + shared_mem_bytes: 0, + ..config + } + ) + .is_ok()); + assert!(LaunchConfig2D::auto_shared((1, 1), (8, 4)).__auto_shared()); + assert!(!LaunchConfig2D::new((1, 1), (8, 4), 0).__auto_shared()); + } + + #[test] + fn compiled_facts_strengthen_explicit_contracts() { + let facts = oxide_artifacts::KernelLaunchFacts { + exact_block: Some((8, 4, 1)), + dynamic_shared_bytes: 128, + dynamic_shared_alignment: 64, + }; + let resolve = |block, memory| { + resolve_compiled_spec(LaunchContractSpec::new("fixed", block, memory), Some(facts)) + }; + let block = BlockRequirement::MaxThreads(32); + let exact = |bytes| DynamicSharedMemoryRequirement::Exact { + bytes, + min_alignment: 16, + }; + for bytes in [0, 127] { + assert!(matches!( + resolve(block, exact(bytes)), + Err(LaunchContractError::DynamicSharedMemoryExactMismatch { .. }) + )); + } + assert_eq!( + resolve(block, exact(256)).unwrap().dynamic_shared_memory, + DynamicSharedMemoryRequirement::Exact { + bytes: 256, + min_alignment: 64, + } + ); + let range = |max_bytes| DynamicSharedMemoryRequirement::Range { + min_bytes: 32, + max_bytes, + min_alignment: 16, + }; + assert!(matches!( + resolve(block, range(127)), + Err(LaunchContractError::DynamicSharedMemoryExactMismatch { .. }) + )); + assert_eq!( + resolve(block, range(256)).unwrap().dynamic_shared_memory, + DynamicSharedMemoryRequirement::Range { + min_bytes: 128, + max_bytes: 256, + min_alignment: 64, + } + ); + assert!(matches!( + resolve(BlockRequirement::MaxThreads(16), exact(128)), + Err(LaunchContractError::BlockThreadsExceedContract { .. }) + )); + } + #[test] fn typed_configs_fix_trailing_dimensions() { let one = LaunchConfig1D::new(7, 64, 16).__raw(); diff --git a/cuda-core/src/simt/mod.rs b/cuda-core/src/simt/mod.rs index 7f07b4c361..7921737eb7 100644 --- a/cuda-core/src/simt/mod.rs +++ b/cuda-core/src/simt/mod.rs @@ -67,7 +67,7 @@ pub use crate::error::{DriverError, IntoResult}; pub use crate::init; pub use crate::launch_kernel; pub use cuda_bindings as sys; -/// Artifact-bundle metadata, from the published crate cuda-oxide releases. +/// Artifact-bundle metadata shared with the CUDA compiler and loaders. pub use oxide_artifacts as artifacts; /// Launches a CUDA kernel on a specific [`CudaStream`], binding its context first. diff --git a/cuda-core/src/simt/module.rs b/cuda-core/src/simt/module.rs index a2b4cb646f..4ec96999fc 100644 --- a/cuda-core/src/simt/module.rs +++ b/cuda-core/src/simt/module.rs @@ -29,7 +29,9 @@ use crate::error::{DriverError, IntoResult}; use crate::simt::context::CudaContext; +use oxide_artifacts::KernelLaunchFacts; use std::borrow::Cow; +use std::collections::BTreeMap; use std::ffi::{c_void, CString}; use std::mem::MaybeUninit; use std::sync::Arc; @@ -44,6 +46,7 @@ pub struct CudaModule { pub(crate) cu_module: cuda_bindings::CUmodule, /// Owning context. Kept alive for the lifetime of this module. pub(crate) ctx: Arc, + kernel_launch_facts: BTreeMap, } /// # Safety @@ -92,6 +95,7 @@ impl CudaContext { Ok(Arc::new(CudaModule { cu_module, ctx: self.clone(), + kernel_launch_facts: BTreeMap::new(), })) } @@ -104,6 +108,22 @@ impl CudaContext { pub fn load_module_from_image( self: &Arc, image: &[u8], + ) -> Result, DriverError> { + self.load_module_from_image_with_launch_facts(image, BTreeMap::new()) + } + + /// Loads an image together with its compiler-produced per-entry facts. + /// + /// The facts must come from the same artifact bundle as `image`. This is + /// intended for artifact loaders; raw image loading leaves facts absent. + /// Loading does not validate the facts against device instructions. Binding + /// this module to a checked launch API is unsafe and requires truthful facts + /// for the exact loaded image and entry. + #[doc(hidden)] + pub fn load_module_from_image_with_launch_facts( + self: &Arc, + image: &[u8], + kernel_launch_facts: BTreeMap, ) -> Result, DriverError> { self.bind_to_thread()?; let image = null_terminated_image(image); @@ -116,6 +136,7 @@ impl CudaContext { Ok(Arc::new(CudaModule { cu_module, ctx: self.clone(), + kernel_launch_facts, })) } @@ -141,6 +162,7 @@ impl CudaContext { Ok(Arc::new(CudaModule { cu_module, ctx: self.clone(), + kernel_launch_facts: BTreeMap::new(), })) } } @@ -168,6 +190,7 @@ pub struct CudaFunction { /// Owning module. Prevents unloading while this function handle exists. #[allow(unused)] pub(crate) module: Arc, + launch_facts: Option, } /// # Safety @@ -254,6 +277,7 @@ impl CudaModule { Ok(CudaFunction { cu_function, module: self.clone(), + launch_facts: self.kernel_launch_facts.get(fn_name).copied(), }) } } @@ -445,6 +469,11 @@ impl CudaModule { } impl CudaFunction { + /// Returns finalized requirements attached to this concrete kernel artifact. + pub fn launch_facts(&self) -> Option { + self.launch_facts + } + fn attribute( &self, attribute: cuda_bindings::CUfunction_attribute, diff --git a/cuda-oxide/Cargo.lock b/cuda-oxide/Cargo.lock index adf2d64339..09d948355f 100644 --- a/cuda-oxide/Cargo.lock +++ b/cuda-oxide/Cargo.lock @@ -211,7 +211,7 @@ dependencies = [ "cuda-artifact-finalizer", "libnvvm-sys", "nvjitlink-sys", - "oxide-artifacts 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "oxide-artifacts", "oxide-tile-export", "oxide-tilebc", "ptx-parse", @@ -417,7 +417,7 @@ dependencies = [ "cuda-bindings", "cuda-core-derive", "half", - "oxide-artifacts 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "oxide-artifacts", ] [[package]] @@ -449,7 +449,7 @@ dependencies = [ "half", "libc", "libloading", - "oxide-artifacts 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "oxide-artifacts", "ptx-parse", "sha2", "thiserror 2.0.21", @@ -484,6 +484,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn 2.0.117", "trybuild", ] @@ -1064,8 +1066,9 @@ dependencies = [ "linkme", "llvm-export", "mir-lower", + "mir-transforms", "once_cell", - "oxide-artifacts 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "oxide-artifacts", "pliron", "pliron-derive", "reserved-oxide-symbols", @@ -1095,6 +1098,8 @@ name = "mir-transforms" version = "0.2.1" dependencies = [ "dialect-mir", + "dialect-nvvm", + "oxide-artifacts", "pliron", "reserved-oxide-symbols", "rustc-hash 2.1.3", @@ -1164,15 +1169,6 @@ dependencies = [ "object", ] -[[package]] -name = "oxide-artifacts" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" -dependencies = [ - "object", -] - [[package]] name = "oxide-tile-export" version = "0.2.1" @@ -1303,7 +1299,7 @@ version = "0.1.0" dependencies = [ "clap", "dialect-ptx", - "oxide-artifacts 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "oxide-artifacts", "ptx-parse", "serde", "serde_json", diff --git a/cuda-oxide/Cargo.toml b/cuda-oxide/Cargo.toml index 0854cb991d..752d4d4de6 100644 --- a/cuda-oxide/Cargo.toml +++ b/cuda-oxide/Cargo.toml @@ -105,13 +105,10 @@ cuda-async = { path = "../cuda-async" } libnvvm-sys = { path = "crates/libnvvm-sys" } nvjitlink-sys = { path = "crates/nvjitlink-sys" } reserved-oxide-symbols = { path = "crates/reserved-oxide-symbols" } -# The crates.io release, not the in-tree path: the shared cuda-core (cutile-rs) -# reads artifact bundles through crates.io oxide-artifacts, and a path copy of -# the same version is a second crate to rustc. crates/oxide-artifacts -# stays in the workspace as the source this release is published from; bump, -# publish, then move this line. -# scripts/check-oxide-artifacts-parity.sh keeps them equal. -oxide-artifacts = "0.2.1" +# Keep the compiler, shared host runtime, and tools on the same artifact +# implementation. The shared runtime and isolated backend use this path too; +# scripts/check-oxide-artifacts-parity.sh verifies the source and version. +oxide-artifacts = { version = "0.2.1", path = "crates/oxide-artifacts" } # Tile export descriptions shared by `#[tile_export]`, the backend, and # `cargo oxide tilelib`. oxide-tile-export = { path = "crates/oxide-tile-export" } diff --git a/cuda-oxide/crates/cargo-oxide/README.md b/cuda-oxide/crates/cargo-oxide/README.md index f9158e3c39..748ea851b7 100644 --- a/cuda-oxide/crates/cargo-oxide/README.md +++ b/cuda-oxide/crates/cargo-oxide/README.md @@ -52,6 +52,7 @@ cargo oxide update --force # inside the workspace, run setup via update | Flag | Applies to | Description | |------------------------------|----------------------------------|-------------------------------------------------| +| `--link-ltoir ` | run, sanitize, build, test, pipeline, debug | Add an ordered external LTO-IR file to `--materialize-cubin` (repeatable) | | `--materialize-cubin` | run, sanitize, build, test, pipeline, debug | Finalize and embed target-specific native GPU code during the host build | | `--emit-nvvm-ir` | run, build, pipeline | Generate NVVM IR for libNVVM | | `--arch ` | run, sanitize, build, test, pipeline, emit-ltoir, tilelib, inspect, debug | Target architecture override | @@ -121,9 +122,32 @@ rechecks them inside the backend; use the wrapper flag instead of setting `CUDA_OXIDE_MATERIALIZE_CUBIN` around raw Cargo. That variable is an internal wrapper/backend signal, not a supported user interface: raw Cargo may reuse a previous artifact without running the backend, and a backend invocation without -the wrapper's fingerprint is rejected. Generic `#[cuda_module]` kernels and -`#[device] extern` declarations are rejected because their run-time -multi-artifact linking is not yet represented by this single-cubin path. +the wrapper's fingerprint is rejected. Generic `#[cuda_module]` kernels remain +unsupported by this single-cubin path because they merge PTX bundles at run time. + +For non-generic kernels calling `#[device] extern` functions, pass the definitions +as ordered, repeatable `--link-ltoir ` inputs alongside `--materialize-cubin`. +Inputs must be raw CUDA LTO-IR, not object files or fatbins. Each supplied file +is linked into every materialized device artifact in that Cargo invocation. +Keep these inputs separate from generated artifact paths; an overlap is rejected +before cleanup or device output emission. +Paths are relative to the invocation directory. The wrapper fingerprints their +canonical names and contents, and the backend verifies and owns those bytes +before linking. Changing an input rebuilds the device artifact; a change during +the wrapper/backend handoff is an error. The Rust module is linked first, followed +by the supplied files in command-line order. CUDA reports unresolved symbols, +invalid LTO-IR, and unsupported toolkit/target combinations during the build. +Use compatible CUDA compiler and linker versions when generating the inputs. + +```bash +cargo oxide run --materialize-cubin --arch sm_120 \ + --link-ltoir build/device-libs/scale.ltoir --link-ltoir build/device-libs/add.ltoir +``` + +The [`external_ltoir` example](../rustc-codegen-cuda/examples/external_ltoir/README.md) +includes C++ sources and commands to generate these files and run the ordinary +embedded-artifact loader. Generated binaries are not source dependencies. + Metadata-interop projects keep rejecting this global flag; they can instead request one independently finalized cubin per declared device crate with `artifact-kind = "cubin"`, as described below. Materialization is also a @@ -326,6 +350,18 @@ links against. It builds the crate in NVVM IR mode, then runs the emitted `compute_XX`, or a bare `XX`. The default output path is `/.ltoir`; `-o/--output` overrides it. +For a standalone project, omit `` to use its Cargo target name. The +directory name may differ from the package name, and hyphens in the target name +become underscores in artifact filenames. A package-named target is preferred; +otherwise the package must have one binary or library target, or its target name +must be supplied explicitly. The positional `` is this Cargo target name; +`default-run` does not select artifacts. This also applies when invoked from a +member of a user Cargo workspace: that member supplies the target, and the IR +and default LTO-IR output are placed in the invocation directory. An explicit +`CUDA_OXIDE_PTX_DIR` selects the intermediate IR directory for both writing and +reading; relative values are resolved from the Cargo workspace root. The process +environment takes precedence over project configuration. + ```bash cargo oxide emit-ltoir standalone_device_fn --arch sm_100 cargo oxide emit-ltoir my_simt_crate --arch sm_120 -o build/simt.ltoir diff --git a/cuda-oxide/crates/cargo-oxide/src/commands/artifacts.rs b/cuda-oxide/crates/cargo-oxide/src/commands/artifacts.rs index 94e2caaa36..37b49a0072 100644 --- a/cuda-oxide/crates/cargo-oxide/src/commands/artifacts.rs +++ b/cuda-oxide/crates/cargo-oxide/src/commands/artifacts.rs @@ -134,12 +134,25 @@ pub(super) fn generated_artifact_paths(project_dir: &Path, package_name: &str) - /// Remove stale generated artifacts (`.ptx`, `.ll`, `.ltoir`, `.cubin`) from a /// previous run so we can verify the build produces fresh output. -pub(super) fn clean_generated_files(example_dir: &Path, example: &str) { - for file in generated_artifact_paths(example_dir, example) { +pub(super) fn clean_generated_files( + example_dir: &Path, + example: &str, + external_inputs: &[PathBuf], +) -> Result<(), String> { + let outputs = generated_artifact_paths(example_dir, example); + // Check the whole deletion set before removing even the first file. + for output in &outputs { + for input in external_inputs { + cuda_artifact_finalizer::check_ltoir_input_output(input, output) + .map_err(|error| error.to_string())?; + } + } + for file in outputs { if file.exists() { let _ = std::fs::remove_file(file); } } + Ok(()) } /// Human-readable label for the selected output format. diff --git a/cuda-oxide/crates/cargo-oxide/src/commands/build_run.rs b/cuda-oxide/crates/cargo-oxide/src/commands/build_run.rs index 27e7640d2b..0b74d36daf 100644 --- a/cuda-oxide/crates/cargo-oxide/src/commands/build_run.rs +++ b/cuda-oxide/crates/cargo-oxide/src/commands/build_run.rs @@ -87,7 +87,10 @@ pub fn codegen_run( std::process::exit(2); } - clean_generated_files(&example_dir, example); + clean_generated_files(&example_dir, example, &ctx.link_ltoir).unwrap_or_else(|error| { + eprintln!("Error: {error}"); + std::process::exit(2); + }); println!("========================================="); println!("RUSTC-CODEGEN-CUDA: {}", example); @@ -256,7 +259,10 @@ pub fn codegen_sanitize( return; } - clean_generated_files(&example_dir, example); + clean_generated_files(&example_dir, example, &ctx.link_ltoir).unwrap_or_else(|error| { + eprintln!("Error: {error}"); + std::process::exit(2); + }); println!("========================================="); println!("RUSTC-CODEGEN-CUDA SANITIZE: {}", example); @@ -381,7 +387,10 @@ pub fn codegen_build( std::process::exit(2); } - clean_generated_files(&example_dir, example); + clean_generated_files(&example_dir, example, &ctx.link_ltoir).unwrap_or_else(|error| { + eprintln!("Error: {error}"); + std::process::exit(2); + }); println!("========================================="); println!("RUSTC-CODEGEN-CUDA BUILD: {}", example); diff --git a/cuda-oxide/crates/cargo-oxide/src/commands/context.rs b/cuda-oxide/crates/cargo-oxide/src/commands/context.rs index 2e45a019bc..67df45fb83 100644 --- a/cuda-oxide/crates/cargo-oxide/src/commands/context.rs +++ b/cuda-oxide/crates/cargo-oxide/src/commands/context.rs @@ -25,6 +25,7 @@ pub struct OxideConfig { /// /// Built once at startup by [`resolve_context`] and passed by reference to /// every command handler. Avoids repeated filesystem walks and backend builds. +#[derive(Clone)] pub struct Context { /// Absolute path to the workspace root (contains top-level `Cargo.toml`). pub workspace_root: PathBuf, @@ -39,6 +40,8 @@ pub struct Context { pub is_workspace: bool, /// Project-local cuda-oxide defaults. pub config: OxideConfig, + /// Ordered external LTO-IR inputs from the command line. + pub link_ltoir: Vec, } impl Context { @@ -74,6 +77,7 @@ pub fn resolve_context() -> Context { let config = load_oxide_config(&workspace_root); let backend_so = backend::find_or_build_backend(&workspace_root, config.backend.as_deref()); return Context { + link_ltoir: Vec::new(), workspace_root, codegen_crate, examples_dir, @@ -92,6 +96,7 @@ pub fn resolve_context() -> Context { let config = load_oxide_config(&cwd); let backend_so = backend::find_or_build_backend(&cwd, config.backend.as_deref()); return Context { + link_ltoir: Vec::new(), workspace_root: cwd.clone(), codegen_crate: cwd.clone(), examples_dir: cwd.clone(), @@ -124,6 +129,7 @@ pub fn resolve_passive_context() -> Context { let config = load_oxide_config_lenient(&workspace_root); let backend_so = backend::backend_so_candidate(&workspace_root, config.backend.as_deref()); return Context { + link_ltoir: Vec::new(), workspace_root, codegen_crate, examples_dir, @@ -142,6 +148,7 @@ pub fn resolve_passive_context() -> Context { let config = load_oxide_config_lenient(&cwd); let backend_so = backend::backend_so_candidate(&cwd, config.backend.as_deref()); return Context { + link_ltoir: Vec::new(), workspace_root: cwd.clone(), codegen_crate: cwd.clone(), examples_dir: cwd.clone(), diff --git a/cuda-oxide/crates/cargo-oxide/src/commands/fingerprint.rs b/cuda-oxide/crates/cargo-oxide/src/commands/fingerprint.rs index c37e7c3435..3c00510b3b 100644 --- a/cuda-oxide/crates/cargo-oxide/src/commands/fingerprint.rs +++ b/cuda-oxide/crates/cargo-oxide/src/commands/fingerprint.rs @@ -78,6 +78,7 @@ pub(super) fn passthrough_codegen_fingerprint_with_env( // Descriptor identity only accelerates verification; artifact identity is // already represented by the content-derived provenance above. effective_env.remove(MATERIALIZER_HANDSHAKE_ENV); + effective_env.remove(EXTERNAL_LTOIR_ENV); if opts.verbose { effective_env.insert("CUDA_OXIDE_VERBOSE".to_string(), b"1".to_vec()); @@ -95,6 +96,10 @@ pub(super) fn passthrough_codegen_fingerprint_with_env( effective_env.insert("CUDA_OXIDE_EMIT_NVVM_IR".to_string(), b"1".to_vec()); } if let Some(prepared) = &materialization.prepared { + effective_env.insert( + EXTERNAL_LTOIR_ENV.to_string(), + prepared.external_ltoir_json.as_bytes().to_vec(), + ); effective_env.insert(MATERIALIZE_ENV.to_string(), b"1".to_vec()); effective_env.insert( EXPECTED_PROVENANCE_ENV.to_string(), diff --git a/cuda-oxide/crates/cargo-oxide/src/commands/host_cargo.rs b/cuda-oxide/crates/cargo-oxide/src/commands/host_cargo.rs index 179ad7bb6f..2bf3bd2e28 100644 --- a/cuda-oxide/crates/cargo-oxide/src/commands/host_cargo.rs +++ b/cuda-oxide/crates/cargo-oxide/src/commands/host_cargo.rs @@ -268,6 +268,62 @@ pub(super) struct CargoExecutableArtifact { pub(super) path: PathBuf, } +/// Infer the artifact stem for a standalone project from Cargo's targets. +/// Keep the package-named target when present, as for workspace examples; +/// otherwise a sole binary or library target supplies its own crate name. +pub fn standalone_crate_name(manifest_dir: &Path) -> Result { + let metadata = cargo_metadata(manifest_dir)?; + let manifest_path = manifest_dir.join("Cargo.toml"); + let manifest_path = manifest_path + .canonicalize() + .map_err(|error| format!("could not resolve {}: {error}", manifest_path.display()))?; + let packages = metadata + .get("packages") + .and_then(|packages| packages.as_array()) + .ok_or_else(|| "Cargo metadata did not include packages".to_string())?; + let selected = cargo_selected_packages(&metadata, packages, &manifest_path)?; + let [package] = selected.as_slice() else { + return Err("run from a single Cargo package to select its device artifact".to_string()); + }; + let package_name = cargo_package_name(package)?; + let targets = package + .get("targets") + .and_then(|targets| targets.as_array()) + .ok_or_else(|| "Cargo metadata did not include package targets".to_string())?; + let mut names: Vec<_> = targets + .iter() + .filter(|target| { + target + .get("kind") + .and_then(|kind| kind.as_array()) + .is_some_and(|kinds| { + kinds.iter().any(|kind| { + matches!( + kind.as_str(), + Some("bin" | "lib" | "rlib" | "staticlib" | "cdylib" | "dylib") + ) + }) + }) + }) + .filter_map(|target| target.get("name").and_then(|name| name.as_str())) + .map(artifact_stem) + .collect(); + names.sort(); + names.dedup(); + let package_stem = artifact_stem(package_name); + if names.contains(&package_stem) { + return Ok(package_stem); + } + if let [name] = names.as_slice() { + return Ok(name.clone()); + } + Err(format!( + "could not choose a device artifact for package `{package_name}`; \ + pass the binary or library target name explicitly (available: {})", + names.join(", ") + )) +} + pub(super) fn cargo_executable_selection( manifest_dir: &Path, explicit_bin: Option<&str>, @@ -365,7 +421,7 @@ fn cargo_package_manifest_matches(package: &serde_json::Value, manifest_path: &P .is_some_and(|path| path == manifest_path) } -fn cargo_metadata(manifest_dir: &Path) -> Result { +pub(super) fn cargo_metadata(manifest_dir: &Path) -> Result { let output = Command::new("cargo") .env("RUSTC_BOOTSTRAP", "1") .args(["metadata", "--format-version=1", "--no-deps"]) diff --git a/cuda-oxide/crates/cargo-oxide/src/commands/interop.rs b/cuda-oxide/crates/cargo-oxide/src/commands/interop.rs index 13d8c532ec..4ea4603dbd 100644 --- a/cuda-oxide/crates/cargo-oxide/src/commands/interop.rs +++ b/cuda-oxide/crates/cargo-oxide/src/commands/interop.rs @@ -573,7 +573,10 @@ fn build_interop_device_crate( } let artifact_name = interop_device_artifact_name(&manifest_path, device_crate); - clean_generated_files(&artifact_dir, &artifact_name); + clean_generated_files(&artifact_dir, &artifact_name, &ctx.link_ltoir).unwrap_or_else(|error| { + eprintln!("Error: {error}"); + std::process::exit(2); + }); if let Some(target) = &binary_target { touch_source_file(&target.source_path); } else { diff --git a/cuda-oxide/crates/cargo-oxide/src/commands/ltoir.rs b/cuda-oxide/crates/cargo-oxide/src/commands/ltoir.rs index feecbef80e..de1f612339 100644 --- a/cuda-oxide/crates/cargo-oxide/src/commands/ltoir.rs +++ b/cuda-oxide/crates/cargo-oxide/src/commands/ltoir.rs @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -use std::path::Path; +use std::path::{Path, PathBuf}; use super::*; @@ -56,12 +56,26 @@ pub fn emit_ltoir( }); let sm_arch = parsed_arch.sm(); + // Cargo invokes rustc from its workspace root even when the selected + // package is a member. Pin this command's IR directory explicitly instead + // of deriving it from the compiler's working directory. + let (build_ctx, ir_dir) = + ltoir_build_context(ctx, &example_dir, std::env::var_os("CUDA_OXIDE_PTX_DIR")) + .unwrap_or_else(|error| { + eprintln!("Error: {error}"); + std::process::exit(1); + }); + clean_generated_files(&ir_dir, example, &ctx.link_ltoir).unwrap_or_else(|error| { + eprintln!("Error: {error}"); + std::process::exit(2); + }); + // Step 1: build in NVVM IR mode so the backend writes `.ll` as // libNVVM-ready NVVM IR. codegen_build exits on build failure. Pass // quiet=true so the intermediate "✓ Build succeeded" line is suppressed; // emit_ltoir prints its own unified summary at the end. codegen_build( - ctx, + &build_ctx, example, verbose, true, @@ -76,7 +90,7 @@ pub fn emit_ltoir( ); // Step 2: compile that NVVM IR to LTOIR via libNVVM -gen-lto. - let ll_path = emitted_ll_path(&example_dir, example); + let ll_path = emitted_ll_path(&ir_dir, example); let ir = std::fs::read(&ll_path).unwrap_or_else(|e| { eprintln!( "Error: could not read emitted NVVM IR at {}: {e}", @@ -165,6 +179,51 @@ pub fn emit_ltoir( ); } +/// Select the same IR directory for the build and subsequent reader. Keep an +/// inherited override strongest, followed by project config. Relative explicit +/// paths are relative to rustc's Cargo workspace cwd, matching existing backend +/// semantics. Without an override, emit-ltoir owns its invocation directory. +pub(super) fn ltoir_build_context( + ctx: &Context, + example_dir: &Path, + inherited_dir: Option, +) -> Result<(Context, PathBuf), String> { + let configured = inherited_dir + .or_else(|| project_config_env(ctx, "CUDA_OXIDE_PTX_DIR").map(std::ffi::OsString::from)); + let ir_dir = match configured { + None => example_dir.to_path_buf(), + Some(path) => { + let path = PathBuf::from(path); + if path.is_absolute() { + path + } else { + let metadata = cargo_metadata(example_dir)?; + let root = metadata + .get("workspace_root") + .and_then(|path| path.as_str()) + .ok_or_else(|| "Cargo metadata did not include workspace_root".to_string())?; + Path::new(root).join(path) + } + } + }; + let value = ir_dir.to_str().ok_or_else(|| { + format!( + "NVVM IR output directory is not Unicode: {}", + ir_dir.display() + ) + })?; + let mut build_ctx = ctx.clone(); + build_ctx + .config + .env + .retain(|(key, _)| key != "CUDA_OXIDE_PTX_DIR"); + build_ctx + .config + .env + .push(("CUDA_OXIDE_PTX_DIR".into(), value.into())); + Ok((build_ctx, ir_dir)) +} + /// Normalize a target architecture to libNVVM's `compute_XX` form. /// /// Accepts `sm_XX` (the form `--arch` and the rest of cargo-oxide use), diff --git a/cuda-oxide/crates/cargo-oxide/src/commands/materialize.rs b/cuda-oxide/crates/cargo-oxide/src/commands/materialize.rs index 80dee3ee40..6b3b5beb56 100644 --- a/cuda-oxide/crates/cargo-oxide/src/commands/materialize.rs +++ b/cuda-oxide/crates/cargo-oxide/src/commands/materialize.rs @@ -9,6 +9,7 @@ use std::process::Command; use super::*; +pub(super) const EXTERNAL_LTOIR_ENV: &str = reserved_oxide_symbols::EXTERNAL_LTOIR_ENV; pub(super) const MATERIALIZE_ENV: &str = reserved_oxide_symbols::MATERIALIZE_CUBIN_ENV; pub(super) const EXPECTED_PROVENANCE_ENV: &str = reserved_oxide_symbols::MATERIALIZER_PROVENANCE_ENV; @@ -31,6 +32,7 @@ pub(super) struct MaterializationMode { pub(super) struct PreparedMaterialization { pub(super) provenance_sha256_hex: String, pub(super) tool_identity_handshake_json: String, + pub(super) external_ltoir_json: String, } impl MaterializationMode { @@ -39,10 +41,13 @@ impl MaterializationMode { } pub(super) fn apply(&self, cmd: &mut Command) { + // Never honor an inherited internal external-input list. + cmd.env_remove(EXTERNAL_LTOIR_ENV); if let Some(prepared) = &self.prepared { // These override inherited/project values: they are a single // wrapper-generated handshake tied to this Cargo invocation. - cmd.env(MATERIALIZE_ENV, "1") + cmd.env(EXTERNAL_LTOIR_ENV, &prepared.external_ltoir_json) + .env(MATERIALIZE_ENV, "1") .env(EXPECTED_PROVENANCE_ENV, &prepared.provenance_sha256_hex) .env( MATERIALIZER_HANDSHAKE_ENV, @@ -205,6 +210,14 @@ fn prepare_materialization_result_with_env( .parse() .map_err(|error| format!("invalid materialization target {arch:?}: {error}"))?; + let external_inputs = ctx + .link_ltoir + .iter() + .map(|path| cuda_artifact_finalizer::LtoirFile::capture(path)) + .collect::, _>>() + .map_err(|error| format!("could not capture external LTO-IR: {error}"))?; + let external_ltoir_json = serde_json::to_string(&external_inputs) + .map_err(|error| format!("could not encode external LTO-IR inputs: {error}"))?; let handshake = discover_materializer_handshake(ctx)?; let handshake_json = serde_json::to_string(&handshake) .map_err(|error| format!("could not encode materializer handshake: {error}"))?; @@ -212,6 +225,7 @@ fn prepare_materialization_result_with_env( prepared: Some(PreparedMaterialization { provenance_sha256_hex: digest_hex(&handshake.provenance_sha256), tool_identity_handshake_json: handshake_json, + external_ltoir_json, }), }) } diff --git a/cuda-oxide/crates/cargo-oxide/src/commands/mod.rs b/cuda-oxide/crates/cargo-oxide/src/commands/mod.rs index 10bc374ee6..81e29f322a 100644 --- a/cuda-oxide/crates/cargo-oxide/src/commands/mod.rs +++ b/cuda-oxide/crates/cargo-oxide/src/commands/mod.rs @@ -40,6 +40,7 @@ pub use doctor::*; pub use examples_list::*; use fingerprint::*; pub use fmt::*; +pub use host_cargo::standalone_crate_name; use host_cargo::*; use interop::*; pub use ltoir::*; diff --git a/cuda-oxide/crates/cargo-oxide/src/commands/pipeline_debug.rs b/cuda-oxide/crates/cargo-oxide/src/commands/pipeline_debug.rs index f84a54f918..43543c2436 100644 --- a/cuda-oxide/crates/cargo-oxide/src/commands/pipeline_debug.rs +++ b/cuda-oxide/crates/cargo-oxide/src/commands/pipeline_debug.rs @@ -41,7 +41,10 @@ pub fn codegen_show_pipeline( reject_interop_output_mode(emit_nvvm_ir, &materialization); } - clean_generated_files(&example_dir, example); + clean_generated_files(&example_dir, example, &ctx.link_ltoir).unwrap_or_else(|error| { + eprintln!("Error: {error}"); + std::process::exit(2); + }); println!("========================================="); println!("RUSTC-CODEGEN-CUDA PIPELINE: {}", example); @@ -196,7 +199,10 @@ pub fn codegen_debug( println!("Detected GPU arch: {dev} (via nvidia-smi)"); } - clean_generated_files(&example_dir, example); + clean_generated_files(&example_dir, example, &ctx.link_ltoir).unwrap_or_else(|error| { + eprintln!("Error: {error}"); + std::process::exit(2); + }); touch_main_rs(&example_dir); diff --git a/cuda-oxide/crates/cargo-oxide/src/commands/tests.rs b/cuda-oxide/crates/cargo-oxide/src/commands/tests.rs index 2ddcaeb911..7324ac290c 100644 --- a/cuda-oxide/crates/cargo-oxide/src/commands/tests.rs +++ b/cuda-oxide/crates/cargo-oxide/src/commands/tests.rs @@ -177,6 +177,7 @@ fn cargo_artifact_freshness( fn test_context(config: OxideConfig) -> Context { Context { + link_ltoir: Vec::new(), workspace_root: PathBuf::from("/tmp/cargo-oxide-test-workspace"), codegen_crate: PathBuf::from("/tmp/cargo-oxide-test-codegen"), examples_dir: PathBuf::from("/tmp/cargo-oxide-test-examples"), @@ -463,6 +464,127 @@ fn emit_ltoir_paths_use_normalized_crate_stem() { assert_eq!(default_ltoir_path(dir, "vecadd"), dir.join("vecadd.ltoir")); } +#[test] +fn standalone_artifact_name_comes_from_cargo_targets() { + for (target, source, expected) in [ + ("", "src/main.rs", "device_export"), + ( + "[[bin]]\nname = \"custom-bin\"\npath = \"src/main.rs\"\n", + "src/main.rs", + "custom_bin", + ), + ( + "[lib]\nname = \"custom_library\"\n", + "src/lib.rs", + "custom_library", + ), + ] { + let root = unique_temp_dir("directory_unrelated_to_package"); + std::fs::create_dir_all(root.join("src")).unwrap(); + std::fs::write( + root.join("Cargo.toml"), + format!( + "[package]\nname = \"device-export\"\nversion = \"0.1.0\"\n\ + edition = \"2024\"\n[workspace]\n{target}" + ), + ) + .unwrap(); + std::fs::write(root.join(source), "fn main() {}\n").unwrap(); + + let name = standalone_crate_name(&root).unwrap(); + assert_eq!(name, expected); + assert_eq!( + emitted_ll_path(&root, &name), + root.join(format!("{expected}.ll")) + ); + assert_eq!( + default_ltoir_path(&root, &name), + root.join(format!("{expected}.ltoir")) + ); + std::fs::remove_dir_all(root).unwrap(); + } +} + +#[test] +fn standalone_artifact_name_preserves_package_target_and_rejects_ambiguity() { + let root = unique_temp_dir("cargo_oxide_artifact_multi_target"); + std::fs::create_dir_all(root.join("src")).unwrap(); + std::fs::write(root.join("src/main.rs"), "fn main() {}\n").unwrap(); + std::fs::write(root.join("src/other.rs"), "fn main() {}\n").unwrap(); + for name in ["device-export", "other-package"] { + std::fs::write( + root.join("Cargo.toml"), + format!( + "[package]\nname = \"{name}\"\nversion = \"0.1.0\"\n\ + edition = \"2024\"\ndefault-run = \"other\"\n[workspace]\n\ + [[bin]]\nname = \"device-export\"\npath = \"src/main.rs\"\n\ + [[bin]]\nname = \"other\"\npath = \"src/other.rs\"\n" + ), + ) + .unwrap(); + let result = standalone_crate_name(&root); + if name == "device-export" { + // `cargo build` builds both bins; default-run does not select the IR. + assert_eq!(result.unwrap(), "device_export"); + } else { + let error = result.unwrap_err(); + assert!( + error.contains("pass the binary or library target name explicitly"), + "{error}" + ); + } + } + std::fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn standalone_artifact_name_uses_workspace_package_selection() { + let root = unique_temp_dir("cargo_oxide_artifact_workspace"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write( + root.join("Cargo.toml"), + "[workspace]\nmembers = [\"first\", \"second\"]\ndefault-members = [\"first\"]\nresolver = \"3\"\n", + ) + .unwrap(); + for member in ["first", "second"] { + let dir = root.join(member); + std::fs::create_dir_all(dir.join("src")).unwrap(); + std::fs::write( + dir.join("Cargo.toml"), + format!( + "[package]\nname = \"{member}-package\"\nversion = \"0.1.0\"\nedition = \"2024\"\n" + ), + ) + .unwrap(); + std::fs::write(dir.join("src/main.rs"), "fn main() {}\n").unwrap(); + } + assert_eq!(standalone_crate_name(&root).unwrap(), "first_package"); + let member = root.join("second"); + assert_eq!(standalone_crate_name(&member).unwrap(), "second_package"); + let ctx = test_context(OxideConfig::default()); + let (build_ctx, ir_dir) = ltoir_build_context(&ctx, &member, None).unwrap(); + assert_eq!(ir_dir, member); + assert_eq!( + project_config_env(&build_ctx, "CUDA_OXIDE_PTX_DIR"), + member.to_str() + ); + + // An explicit relative backend override retains Cargo workspace semantics, + // even though this command was invoked from a different selected member. + let mut ctx = ctx; + ctx.config + .env + .push(("CUDA_OXIDE_PTX_DIR".into(), "configured-ir".into())); + let (_, configured) = ltoir_build_context(&ctx, &member, None).unwrap(); + assert_eq!( + configured, + root.canonicalize().unwrap().join("configured-ir") + ); + let (_, inherited) = ltoir_build_context(&ctx, &member, Some("inherited-ir".into())).unwrap(); + assert_eq!(inherited, root.canonicalize().unwrap().join("inherited-ir")); + std::fs::remove_dir_all(root).unwrap(); +} + #[test] fn generated_file_cleanup_preserves_ltoir_cubin_cache() { let unique = std::time::SystemTime::now() @@ -482,7 +604,7 @@ fn generated_file_cleanup_preserves_ltoir_cubin_cache() { std::fs::create_dir_all(cached_cubin.parent().unwrap()).unwrap(); std::fs::write(&cached_cubin, b"persistent cache entry").unwrap(); - clean_generated_files(&root, "my-kernel"); + clean_generated_files(&root, "my-kernel", &[]).unwrap(); for extension in ["ptx", "ll", "ltoir", "cubin", "target"] { assert!(!root.join(format!("my_kernel.{extension}")).exists()); @@ -522,6 +644,7 @@ edition = "2024" std::fs::write(&cached_cubin, b"persistent cache").unwrap(); let ctx = Context { + link_ltoir: Vec::new(), workspace_root: root.clone(), codegen_crate: root.clone(), examples_dir: root.clone(), @@ -584,6 +707,7 @@ fn workspace_clean_covers_example_and_model_roots_in_flat_and_nested_layouts() { std::fs::write(model_dir.join("decoder.cubin"), b"generated").unwrap(); std::fs::write(cute_dir.join("other.cubin"), b"preserve").unwrap(); let ctx = Context { + link_ltoir: Vec::new(), workspace_root: root.clone(), codegen_crate: codegen_crate.clone(), examples_dir, @@ -642,6 +766,7 @@ edition = "2024" symlink(&external, root.join("target")).unwrap(); let ctx = Context { + link_ltoir: Vec::new(), workspace_root: root.clone(), codegen_crate: root.clone(), examples_dir: root.clone(), @@ -3153,6 +3278,7 @@ device-owner = { path = "../device-owner" } .unwrap(); let ctx = Context { + link_ltoir: Vec::new(), workspace_root: root.clone(), codegen_crate: root.join("unused-codegen-source"), examples_dir: root.join("unused-examples"), @@ -3316,6 +3442,7 @@ shared-dep = { path = "../shared-dep" } fn scratch_workspace_context(root: &Path) -> Context { Context { + link_ltoir: Vec::new(), workspace_root: root.to_path_buf(), codegen_crate: root.join("unused-codegen-source"), examples_dir: root.join("unused-examples"), @@ -3595,6 +3722,7 @@ fn passthrough_fingerprint_tracks_output_affecting_settings() { let materialized = MaterializationMode { prepared: Some(PreparedMaterialization { provenance_sha256_hex: "ab".repeat(32), + external_ltoir_json: "[]".to_string(), tool_identity_handshake_json: "{\"version\":1}".to_string(), }), }; @@ -3612,6 +3740,96 @@ fn passthrough_fingerprint_tracks_output_affecting_settings() { ); } +#[test] +fn cleanup_preserves_all_artifacts_when_a_link_input_overlaps() { + let root = unique_temp_dir("cargo_oxide_input_output_overlap"); + fs::create_dir_all(&root).unwrap(); + let first_output = root.join("my_kernel.ptx"); + let input = root.join("my_kernel.ltoir"); + fs::write(&first_output, b"previous output").unwrap(); + fs::write(&input, b"external input").unwrap(); + + let error = + clean_generated_files(&root, "my-kernel", std::slice::from_ref(&input)).unwrap_err(); + assert!(error.contains("move the external input"), "{error}"); + assert_eq!(fs::read(&input).unwrap(), b"external input"); + assert_eq!(fs::read(&first_output).unwrap(), b"previous output"); + + #[cfg(unix)] + for hard_link in [false, true] { + let alias = root.join("external.ltoir"); + if hard_link { + fs::hard_link(&input, &alias).unwrap(); + } else { + std::os::unix::fs::symlink(&input, &alias).unwrap(); + } + assert!(clean_generated_files(&root, "my-kernel", std::slice::from_ref(&alias)).is_err()); + assert_eq!(fs::read(&alias).unwrap(), b"external input"); + assert_eq!(fs::read(&first_output).unwrap(), b"previous output"); + fs::remove_file(alias).unwrap(); + } + + let separate = root.join("separate.ltoir"); + fs::rename(&input, &separate).unwrap(); + clean_generated_files(&root, "my-kernel", std::slice::from_ref(&separate)).unwrap(); + assert!(!first_output.exists()); + assert_eq!(fs::read(&separate).unwrap(), b"external input"); + fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn external_ltoir_order_and_contents_change_the_codegen_fingerprint() { + let ctx = test_context(OxideConfig::default()); + let opts = CargoPassthroughOptions { + verbose: false, + emit_nvvm_ir: false, + arch: Some("sm_120"), + features: None, + cargo_target_dir: None, + device_codegen_crate: None, + device_cfgs: &[], + no_fmad: false, + unchecked_indexing: false, + materialize_cubin: true, + device_debug: DeviceDebug::Off, + debug_assertions: false, + }; + let root = unique_temp_dir("cargo_oxide_external_ltoir"); + fs::create_dir_all(&root).unwrap(); + let first = root.join("first.ltoir"); + let second = root.join("second.ltoir"); + fs::write(&first, b"first").unwrap(); + fs::write(&second, b"other").unwrap(); + let fingerprint = |paths: &[&Path]| { + let inputs: Vec<_> = paths + .iter() + .map(|path| cuda_artifact_finalizer::LtoirFile::capture(path).unwrap()) + .collect(); + let mode = MaterializationMode { + prepared: Some(PreparedMaterialization { + provenance_sha256_hex: "ab".repeat(32), + tool_identity_handshake_json: "{}".into(), + external_ltoir_json: serde_json::to_string(&inputs).unwrap(), + }), + }; + passthrough_codegen_fingerprint_with_env( + &ctx, + &opts, + None, + Some("sm_120"), + &mode, + &BTreeMap::new(), + ) + }; + let original = fingerprint(&[&first, &second]); + assert_eq!(original, fingerprint(&[&first, &second])); + assert_ne!(original, fingerprint(&[&second, &first])); + fs::write(&first, b"later").unwrap(); + assert_ne!(original, fingerprint(&[&first, &second])); + assert_ne!(original, fingerprint(&[&second])); + fs::remove_dir_all(root).unwrap(); +} + #[test] fn passthrough_fingerprint_tracks_the_managed_cutlass_library_digest() { let ctx = test_context(OxideConfig::default()); @@ -3937,6 +4155,7 @@ fn materialization_forces_nvvm_ir_and_exact_provenance_handshake() { let materialization = MaterializationMode { prepared: Some(PreparedMaterialization { provenance_sha256_hex: "42".repeat(32), + external_ltoir_json: "[]".to_string(), tool_identity_handshake_json: "{\"version\":1}".to_string(), }), }; @@ -5474,6 +5693,7 @@ fn read_ptx_artifact_reports_missing_file() { #[test] fn nvvm_ir_requested_reads_project_configuration() { let ctx = Context { + link_ltoir: Vec::new(), workspace_root: PathBuf::from("/tmp/project"), codegen_crate: PathBuf::from("/tmp/project"), examples_dir: PathBuf::from("/tmp/project"), @@ -5491,6 +5711,7 @@ fn nvvm_ir_requested_reads_project_configuration() { #[test] fn nvvm_ir_requested_accepts_disabled_project_configuration() { let ctx = Context { + link_ltoir: Vec::new(), workspace_root: PathBuf::from("/tmp/project"), codegen_crate: PathBuf::from("/tmp/project"), examples_dir: PathBuf::from("/tmp/project"), @@ -5508,6 +5729,7 @@ fn nvvm_ir_requested_accepts_disabled_project_configuration() { #[test] fn nvvm_ir_requested_env_disable_overrides_enabled_project_configuration() { let ctx = Context { + link_ltoir: Vec::new(), workspace_root: PathBuf::from("/tmp/project"), codegen_crate: PathBuf::from("/tmp/project"), examples_dir: PathBuf::from("/tmp/project"), @@ -6643,7 +6865,7 @@ fn tilelib_artifacts_are_cleaned_between_builds() { for file in &files { std::fs::write(file, b"stale").unwrap(); } - clean_generated_files(&root, "my-lib"); + clean_generated_files(&root, "my-lib", &[]).unwrap(); for file in &files { assert!(!file.exists(), "{} survived the clean", file.display()); } diff --git a/cuda-oxide/crates/cargo-oxide/src/main.rs b/cuda-oxide/crates/cargo-oxide/src/main.rs index bfd4524b61..fddc102931 100644 --- a/cuda-oxide/crates/cargo-oxide/src/main.rs +++ b/cuda-oxide/crates/cargo-oxide/src/main.rs @@ -77,6 +77,10 @@ enum Commands { DebugPolicy, /// Build and run an example, model, or project Run { + /// Link an external LTO-IR file, in argument order (repeatable). + /// Requires --materialize-cubin; paths are relative to the invocation directory. + #[arg(long)] + link_ltoir: Vec, /// Example or model name (required in workspace, optional for standalone projects) example: Option, /// Generate NVVM IR (use with libNVVM -gen-lto) @@ -128,6 +132,10 @@ enum Commands { }, /// Build and run an example or project under NVIDIA Compute Sanitizer Sanitize { + /// Link an external LTO-IR file, in argument order (repeatable). + /// Requires --materialize-cubin; paths are relative to the invocation directory. + #[arg(long)] + link_ltoir: Vec, /// Example name (required in workspace, optional for standalone projects) example: Option, /// Compute Sanitizer tool to run @@ -173,6 +181,10 @@ enum Commands { }, /// Build an example or project (compile only, don't run) Build { + /// Link an external LTO-IR file, in argument order (repeatable). + /// Requires --materialize-cubin; paths are relative to the invocation directory. + #[arg(long)] + link_ltoir: Vec, /// Example name (required in workspace, optional for standalone projects) example: Option, /// Generate NVVM IR (use with libNVVM -gen-lto) @@ -267,6 +279,10 @@ enum Commands { }, /// Run Cargo tests through the cuda-oxide backend Test { + /// Link an external LTO-IR file, in argument order (repeatable). + /// Requires --materialize-cubin; paths are relative to the invocation directory. + #[arg(long)] + link_ltoir: Vec, /// Target architecture (e.g., sm_90, sm_100, sm_120) #[arg(long)] arch: Option, @@ -406,6 +422,10 @@ enum Commands { }, /// Show the full compilation pipeline (MIR -> PTX/NVVM IR) with verbose output Pipeline { + /// Link an external LTO-IR file, in argument order (repeatable). + /// Requires --materialize-cubin; paths are relative to the invocation directory. + #[arg(long)] + link_ltoir: Vec, /// Example name (required in workspace, optional for standalone projects) example: Option, /// Generate NVVM IR (use with libNVVM -gen-lto) @@ -436,6 +456,10 @@ enum Commands { }, /// Build with debug info and launch cuda-gdb Debug { + /// Link an external LTO-IR file, in argument order (repeatable). + /// Requires --materialize-cubin; paths are relative to the invocation directory. + #[arg(long)] + link_ltoir: Vec, /// Example name (required in workspace, optional for standalone projects) example: Option, /// Target architecture (e.g., sm_90, sm_100, sm_120). When omitted, @@ -626,7 +650,22 @@ fn build_passthrough_trigger( } } +fn external_ltoir_inputs(command: &Commands) -> &[PathBuf] { + match command { + Commands::Run { link_ltoir, .. } + | Commands::Sanitize { link_ltoir, .. } + | Commands::Build { link_ltoir, .. } + | Commands::Test { link_ltoir, .. } + | Commands::Pipeline { link_ltoir, .. } + | Commands::Debug { link_ltoir, .. } => link_ltoir, + _ => &[], + } +} + fn validate_materialization_cli(cli: &Cli) -> Result<(), String> { + if !external_ltoir_inputs(&cli.command).is_empty() && !cli.materialize_cubin { + return Err("--link-ltoir requires --materialize-cubin".to_string()); + } if !cli.materialize_cubin { return Ok(()); } @@ -757,6 +796,12 @@ fn main() { .exit(); } let materialize_cubin = cli.materialize_cubin; + let link_ltoir = external_ltoir_inputs(&cli.command).to_vec(); + let resolve_context = || { + let mut ctx = commands::resolve_context(); + ctx.link_ltoir = link_ltoir.clone(); + ctx + }; match cli.command { Commands::MaterializerHandshake => { @@ -766,6 +811,7 @@ fn main() { commands::print_debug_policy(); } Commands::Run { + link_ltoir: _, example, emit_nvvm_ir, arch, @@ -779,7 +825,7 @@ fn main() { device_debug, app_args, } => { - let ctx = commands::resolve_context(); + let ctx = resolve_context(); let example = resolve_example_name(example, &ctx, "run"); validate_output_arch( &ctx, @@ -805,6 +851,7 @@ fn main() { ); } Commands::Sanitize { + link_ltoir: _, example, tool, arch, @@ -817,7 +864,7 @@ fn main() { device_debug, sanitizer_args, } => { - let ctx = commands::resolve_context(); + let ctx = resolve_context(); let example = resolve_example_name(example, &ctx, "sanitize"); validate_output_arch(&ctx, &example, false, materialize_cubin, arch.as_deref()); let (sanitizer_args, application_args) = @@ -839,6 +886,7 @@ fn main() { ); } Commands::Build { + link_ltoir: _, example, emit_nvvm_ir, arch, @@ -855,7 +903,7 @@ fn main() { device_cfgs, cargo_args, } => { - let ctx = commands::resolve_context(); + let ctx = resolve_context(); let passthrough = use_build_passthrough( explicit_passthrough, cargo_target_dir.is_some(), @@ -945,7 +993,7 @@ fn main() { keep_going, fail_on_finding, } => { - let ctx = commands::resolve_context(); + let ctx = resolve_context(); let (seed_start, seed_end) = ptx_schedule::campaign::parse_seed_range(&seeds) .unwrap_or_else(|error| { eprintln!("Error: {error}"); @@ -1009,6 +1057,7 @@ fn main() { } } Commands::Test { + link_ltoir: _, arch, cargo_target_dir, device_codegen_crate, @@ -1020,7 +1069,7 @@ fn main() { device_debug, cargo_args, } => { - let ctx = commands::resolve_context(); + let ctx = resolve_context(); validate_output_arch( &ctx, "cargo test", @@ -1059,7 +1108,7 @@ fn main() { lineinfo, device_debug, } => { - let ctx = commands::resolve_context(); + let ctx = resolve_context(); let example = resolve_example_name(example, &ctx, "emit-ltoir"); commands::emit_ltoir( &ctx, @@ -1086,7 +1135,7 @@ fn main() { lineinfo, device_debug, } => { - let ctx = commands::resolve_context(); + let ctx = resolve_context(); let example = resolve_example_name(example, &ctx, "tilelib"); commands::tilelib( &ctx, @@ -1103,6 +1152,7 @@ fn main() { ); } Commands::Pipeline { + link_ltoir: _, example, emit_nvvm_ir, arch, @@ -1111,7 +1161,7 @@ fn main() { lineinfo, device_debug, } => { - let ctx = commands::resolve_context(); + let ctx = resolve_context(); let example = resolve_example_name(example, &ctx, "pipeline"); validate_output_arch( &ctx, @@ -1141,7 +1191,7 @@ fn main() { lineinfo, device_debug, } => { - let ctx = commands::resolve_context(); + let ctx = resolve_context(); let example = resolve_example_name(example, &ctx, "inspect"); commands::codegen_inspect_ptx( &ctx, @@ -1155,6 +1205,7 @@ fn main() { ); } Commands::Debug { + link_ltoir: _, example, arch, features, @@ -1162,7 +1213,7 @@ fn main() { cgdb, tui, } => { - let ctx = commands::resolve_context(); + let ctx = resolve_context(); let example = resolve_example_name(example, &ctx, "debug"); validate_output_arch(&ctx, &example, false, materialize_cubin, arch.as_deref()); commands::codegen_debug( @@ -1203,7 +1254,7 @@ fn main() { commands::doctor(&ctx); } Commands::Setup => { - let ctx = commands::resolve_context(); + let ctx = resolve_context(); commands::setup(&ctx); } Commands::Update { force } => { @@ -1240,6 +1291,12 @@ fn resolve_example_name(name: Option, ctx: &commands::Context, subcomman return n; } if !ctx.is_workspace { + if subcommand == "emit-ltoir" { + return commands::standalone_crate_name(&ctx.workspace_root).unwrap_or_else(|error| { + eprintln!("Error: {error}"); + std::process::exit(1); + }); + } return std::env::current_dir() .ok() .and_then(|p| p.file_name().map(|n| n.to_string_lossy().into_owned())) @@ -1427,6 +1484,47 @@ mod tests { assert_eq!(device_features.as_deref(), Some("tensor-cores")); } + #[test] + fn external_ltoir_requires_materialization_and_preserves_order() { + let unmaterialized = + Cli::try_parse_from(["cargo-oxide", "build", "--link-ltoir", "one"]).unwrap(); + assert!(validate_materialization_cli(&unmaterialized).is_err()); + let cli = Cli::try_parse_from([ + "cargo-oxide", + "build", + "--link-ltoir", + "one", + "--materialize-cubin", + "--link-ltoir", + "two", + ]) + .unwrap(); + assert_eq!( + external_ltoir_inputs(&cli.command), + [ + std::path::PathBuf::from("one"), + std::path::PathBuf::from("two") + ] + ); + validate_materialization_cli(&cli).unwrap(); + + let run = Cli::try_parse_from([ + "cargo-oxide", + "run", + "--link-ltoir", + "one", + "external_ltoir", + "--materialize-cubin", + ]) + .unwrap(); + validate_materialization_cli(&run).unwrap(); + assert_eq!(external_ltoir_inputs(&run.command), [PathBuf::from("one")]); + let Commands::Run { example, .. } = run.command else { + panic!("expected run command"); + }; + assert_eq!(example.as_deref(), Some("external_ltoir")); + } + #[test] fn materialize_cubin_is_a_global_codegen_option() { let after_subcommand = Cli::try_parse_from([ diff --git a/cuda-oxide/crates/cuda-artifact-finalizer/src/lib.rs b/cuda-oxide/crates/cuda-artifact-finalizer/src/lib.rs index 6a014422de..6e02f0c34b 100644 --- a/cuda-oxide/crates/cuda-artifact-finalizer/src/lib.rs +++ b/cuda-oxide/crates/cuda-artifact-finalizer/src/lib.rs @@ -12,6 +12,7 @@ mod diagnostics; mod link; +mod ltoir_file; mod nvvm; mod options; mod provenance; @@ -21,6 +22,7 @@ mod validation; pub use diagnostics::KernelResourceUsage; pub use libnvvm_sys::{CudaArch, CudaArchParseError, LibdeviceNotFound, NvvmError, find_libdevice}; pub use link::{LinkReport, LtoLinker}; +pub use ltoir_file::{LtoirFile, check_ltoir_input_output}; pub use nvjitlink_sys::NvJitLinkError; pub use nvvm::NvvmCompiler; pub use options::{DebugPolicy, FinalizationOptions, FinalizerOutput, NamedInput}; @@ -37,6 +39,20 @@ use thiserror::Error; /// Failures while compiling or linking CUDA artifacts. #[derive(Debug, Error)] pub enum FinalizerError { + /// A file path cannot be transported losslessly to the CUDA linker. + #[error("external LTO-IR path is not Unicode: {path:?}")] + InvalidLtoirPath { path: PathBuf }, + + /// A link input changed after the wrapper computed Cargo's fingerprint. + #[error("external LTO-IR input changed after Cargo fingerprinting: {path}; rerun cargo oxide")] + LtoirInputChanged { path: PathBuf }, + + /// An output would delete or overwrite an explicitly supplied link input. + #[error( + "external LTO-IR input {input} overlaps generated artifact {output}; move the external input to a separate directory" + )] + LtoirInputOutputOverlap { input: PathBuf, output: PathBuf }, + /// libNVVM failed to load, validate, or compile. #[error("libnvvm: {0}")] Nvvm(#[from] libnvvm_sys::NvvmError), diff --git a/cuda-oxide/crates/cuda-artifact-finalizer/src/ltoir_file.rs b/cuda-oxide/crates/cuda-artifact-finalizer/src/ltoir_file.rs new file mode 100644 index 0000000000..92aff18147 --- /dev/null +++ b/cuda-oxide/crates/cuda-artifact-finalizer/src/ltoir_file.rs @@ -0,0 +1,163 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Content-bound external inputs shared by Cargo fingerprinting and finalization. + +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::FinalizerError; + +/// Identity of an external LTO-IR file captured before Cargo decides freshness. +/// +/// The absolute path is also the linker's diagnostic input name. Preserve the +/// order of these records when fingerprinting and linking a list of inputs. +/// [`Self::read_verified`] returns owned bytes so a later file change cannot +/// alter the data submitted to the linker. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LtoirFile { + path: PathBuf, + sha256: [u8; 32], +} + +impl LtoirFile { + /// Capture the canonical path and SHA-256 of a nonempty input file. + /// + /// Input paths must be Unicode so the same name can reach nvJitLink and + /// the compiler process without lossy conversion. CUDA validates the + /// LTO-IR format and target compatibility during finalization. + pub fn capture(path: &Path) -> Result { + let path = path.canonicalize().map_err(|source| FinalizerError::Io { + path: path.to_owned(), + source, + })?; + if path.to_str().is_none() { + return Err(FinalizerError::InvalidLtoirPath { path }); + } + let bytes = read(&path)?; + Ok(Self { + path, + sha256: Sha256::digest(&bytes).into(), + }) + } + + /// Diagnostic name included in the ordered compilation identity. + pub fn name(&self) -> Result<&str, FinalizerError> { + self.path + .to_str() + .ok_or_else(|| FinalizerError::InvalidLtoirPath { + path: self.path.clone(), + }) + } + + /// Read the exact captured content or fail if it changed before codegen. + pub fn read_verified(&self) -> Result, FinalizerError> { + let bytes = read(&self.path)?; + let actual: [u8; 32] = Sha256::digest(&bytes).into(); + if actual != self.sha256 { + return Err(FinalizerError::LtoirInputChanged { + path: self.path.clone(), + }); + } + Ok(bytes) + } +} + +/// Reject a generated output that aliases an existing external link input. +/// +/// Call before deleting or writing outputs. Canonical paths cover symlinks; +/// Unix file identity also covers hard links with different canonical paths. +pub fn check_ltoir_input_output(input: &Path, output: &Path) -> Result<(), FinalizerError> { + let input_path = input.canonicalize().map_err(|source| FinalizerError::Io { + path: input.to_owned(), + source, + })?; + let output_path = match output.canonicalize() { + Ok(path) => path, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(source) => { + return Err(FinalizerError::Io { + path: output.to_owned(), + source, + }); + } + }; + if input_path == output_path || same_file(&input_path, &output_path) { + return Err(FinalizerError::LtoirInputOutputOverlap { + input: input.to_owned(), + output: output.to_owned(), + }); + } + Ok(()) +} + +#[cfg(unix)] +fn same_file(a: &Path, b: &Path) -> bool { + use std::os::unix::fs::MetadataExt; + match (std::fs::metadata(a), std::fs::metadata(b)) { + (Ok(a), Ok(b)) => a.dev() == b.dev() && a.ino() == b.ino(), + _ => false, + } +} + +#[cfg(not(unix))] +fn same_file(_a: &Path, _b: &Path) -> bool { + false +} + +fn read(path: &Path) -> Result, FinalizerError> { + let bytes = std::fs::read(path).map_err(|source| FinalizerError::Io { + path: path.to_owned(), + source, + })?; + if bytes.is_empty() { + return Err(FinalizerError::EmptyInput { + name: path.display().to_string(), + }); + } + Ok(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn captured_input_checks_content_and_owns_verified_bytes() { + let path = std::env::temp_dir().join(format!( + "oxide-ltoir-{}-{}.test", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::write(&path, b"first").unwrap(); + let identity = LtoirFile::capture(&path).unwrap(); + let json = serde_json::to_string(&identity).unwrap(); + let restored: LtoirFile = serde_json::from_str(&json).unwrap(); + let owned = restored.read_verified().unwrap(); + std::fs::write(&path, b"other").unwrap(); + assert_eq!(owned, b"first"); + assert!(matches!( + restored.read_verified(), + Err(FinalizerError::LtoirInputChanged { .. }) + )); + assert_ne!(identity, LtoirFile::capture(&path).unwrap()); + std::fs::write(&path, []).unwrap(); + assert!(matches!( + LtoirFile::capture(&path), + Err(FinalizerError::EmptyInput { .. }) + )); + std::fs::remove_file(&path).unwrap(); + assert!(matches!( + restored.read_verified(), + Err(FinalizerError::Io { .. }) + )); + } +} diff --git a/cuda-oxide/crates/cuda-device/src/cluster.rs b/cuda-oxide/crates/cuda-device/src/cluster.rs index 300156c827..394feae617 100644 --- a/cuda-oxide/crates/cuda-device/src/cluster.rs +++ b/cuda-oxide/crates/cuda-device/src/cluster.rs @@ -4,7 +4,7 @@ */ #![allow(non_snake_case)] -//! Thread Block Cluster intrinsics for Hopper (sm_90+). +//! Thread block cluster queries and Hopper (sm_90+) cluster operations. //! //! Thread Block Clusters are a new hierarchy level introduced in NVIDIA Hopper //! that groups multiple thread blocks together. Blocks within a cluster can: @@ -25,9 +25,24 @@ //! //! # Hardware Requirements //! -//! - Minimum: sm_90 (Hopper: H100, H200) +//! - Native cluster operations require sm_90 (Hopper: H100, H200) //! - Blackwell (sm_100/sm_120) extends cluster capabilities //! +//! # Queries on earlier targets +//! +//! When compiling for a known target below sm90, the position and dimension +//! queries in this module treat each block as its own cluster. Block coordinates +//! and [`block_rank`] are zero; cluster dimensions and [`cluster_size`] are one. +//! [`cluster_idx`] returns the linear block index, and [`num_clusters`] returns +//! the number of blocks in the grid. +//! +//! An explicit target (for example, `cargo oxide build --arch sm_80`) takes +//! precedence over a detected device. If a pre-sm90 device hint selects these +//! semantics, the compiler binds that target and rejects incompatible operations. +//! Without a target or device hint, queries retain native sm90+ requirements. +//! Raw `cuda_intrinsics` cluster queries, synchronization, and distributed shared +//! memory operations always retain their hardware requirements. +//! //! # Cluster Dimensions //! //! Clusters are defined as a 3D arrangement of thread blocks: @@ -76,7 +91,7 @@ /// /// # PTX /// -/// Lowers to: `mov.u32 %r, %cluster_ctaid.x` +/// On sm90+: `mov.u32 %r, %cluster_ctaid.x`. On earlier targets: zero. #[inline(never)] pub fn cluster_ctaidX() -> u32 { unreachable!("cluster_ctaidX called outside CUDA kernel context") @@ -88,7 +103,7 @@ pub fn cluster_ctaidX() -> u32 { /// /// # PTX /// -/// Lowers to: `mov.u32 %r, %cluster_ctaid.y` +/// On sm90+: `mov.u32 %r, %cluster_ctaid.y`. On earlier targets: zero. #[inline(never)] pub fn cluster_ctaidY() -> u32 { unreachable!("cluster_ctaidY called outside CUDA kernel context") @@ -100,7 +115,7 @@ pub fn cluster_ctaidY() -> u32 { /// /// # PTX /// -/// Lowers to: `mov.u32 %r, %cluster_ctaid.z` +/// On sm90+: `mov.u32 %r, %cluster_ctaid.z`. On earlier targets: zero. #[inline(never)] pub fn cluster_ctaidZ() -> u32 { unreachable!("cluster_ctaidZ called outside CUDA kernel context") @@ -117,7 +132,7 @@ pub fn cluster_ctaidZ() -> u32 { /// /// # PTX /// -/// Lowers to: `mov.u32 %r, %cluster_nctaid.x` +/// On sm90+: `mov.u32 %r, %cluster_nctaid.x`. On earlier targets: one. #[inline(never)] pub fn cluster_nctaidX() -> u32 { unreachable!("cluster_nctaidX called outside CUDA kernel context") @@ -127,7 +142,7 @@ pub fn cluster_nctaidX() -> u32 { /// /// # PTX /// -/// Lowers to: `mov.u32 %r, %cluster_nctaid.y` +/// On sm90+: `mov.u32 %r, %cluster_nctaid.y`. On earlier targets: one. #[inline(never)] pub fn cluster_nctaidY() -> u32 { unreachable!("cluster_nctaidY called outside CUDA kernel context") @@ -137,7 +152,7 @@ pub fn cluster_nctaidY() -> u32 { /// /// # PTX /// -/// Lowers to: `mov.u32 %r, %cluster_nctaid.z` +/// On sm90+: `mov.u32 %r, %cluster_nctaid.z`. On earlier targets: one. #[inline(never)] pub fn cluster_nctaidZ() -> u32 { unreachable!("cluster_nctaidZ called outside CUDA kernel context") @@ -157,7 +172,8 @@ include!("generated/cluster_sreg.rs"); /// # PTX /// /// Lowers to documented `%clusterid.{x,y,z}` / `%nclusterid.{x,y}` reads and -/// computes `x + y * nx + z * nx * ny`. +/// computes `x + y * nx + z * nx * ny`. Below sm90, uses ordinary block +/// coordinates and grid dimensions in the same expression. #[inline(always)] pub fn cluster_idx() -> u32 { let x = __cluster_idxX(); @@ -206,6 +222,7 @@ pub fn cluster_idx_z() -> u32 { /// # PTX /// /// Lowers to documented `%nclusterid.{x,y,z}` reads and computes `nx * ny * nz`. +/// Below sm90, uses ordinary grid dimensions instead. #[inline(always)] pub fn num_clusters() -> u32 { __cluster_grid_dimX() * __cluster_grid_dimY() * __cluster_grid_dimZ() diff --git a/cuda-oxide/crates/cuda-device/src/disjoint.rs b/cuda-oxide/crates/cuda-device/src/disjoint.rs index bbffd17f08..81a8bebd65 100644 --- a/cuda-oxide/crates/cuda-device/src/disjoint.rs +++ b/cuda-oxide/crates/cuda-device/src/disjoint.rs @@ -80,6 +80,24 @@ pub trait SpaceLayout: space_layout_sealed::Sealed { type Data: Copy; } +/// Collective-only storage with a fixed ownership grain per physical thread. +/// +/// Use as the space of a kernel's `DisjointSlice>`. +/// It has the ordinary pointer/length launch packet, but no safe lane or tile +/// reference accessors. Library wrappers check each operation's bounds and +/// canonical tile at runtime, trapping on failure. The compiler checks that +/// conflicting accesses are separated by explicit completion. The capacity +/// remains fixed when an operation uses a different item count. +/// +/// This type supplies no participation, bounds or synchronization proof by +/// itself. Raw access retains the corresponding unsafe obligations. +pub enum CollectiveTiles {} + +impl space_layout_sealed::Sealed for CollectiveTiles {} +impl SpaceLayout for CollectiveTiles { + type Data = (); +} + impl space_layout_sealed::Sealed for Index1D {} impl SpaceLayout for Index1D { type Data = (); @@ -232,6 +250,11 @@ pub trait __LaunchContractDisjointSlice: impl<'a, T> __LaunchContractDisjointSlice for DisjointSlice<'a, T, Index1D> {} +impl<'a, T, const CAPACITY: usize, const DOMAIN: u8> __LaunchContractDisjointSlice + for DisjointSlice<'a, T, CollectiveTiles> +{ +} + impl<'a, T, const N: usize> __LaunchContractDisjointSlice for DisjointSlice<'a, T, LinearTiles> { diff --git a/cuda-oxide/crates/cuda-device/src/lib.rs b/cuda-oxide/crates/cuda-device/src/lib.rs index 67ee46790e..67411866a2 100644 --- a/cuda-oxide/crates/cuda-device/src/lib.rs +++ b/cuda-oxide/crates/cuda-device/src/lib.rs @@ -41,6 +41,8 @@ pub mod grid; pub mod i16x2; pub mod iket; pub mod int; +#[doc(hidden)] +pub mod library; pub mod mma_frag; pub mod prmt; pub mod ptx; @@ -80,7 +82,7 @@ pub use constant::{ConstantMemory, ConstantMemoryValue}; pub use cusimd::{CuSimd, Float2, Float4, TmemRegs4, TmemRegs32}; #[doc(hidden)] pub use disjoint::{__LaunchContractDisjointSlice, __LaunchContractDisjointSliceAbi}; -pub use disjoint::{DisjointSlice, SpaceLayout}; +pub use disjoint::{CollectiveTiles, DisjointSlice, SpaceLayout}; pub use fence::*; pub use shared::{DynamicSharedArray, SharedArray, SharedPtr}; pub use tcgen05::{ diff --git a/cuda-oxide/crates/cuda-device/src/library.rs b/cuda-oxide/crates/cuda-device/src/library.rs new file mode 100644 index 0000000000..63ca2746f4 --- /dev/null +++ b/cuda-oxide/crates/cuda-device/src/library.rs @@ -0,0 +1,314 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Internal, versioned boundary for native device library calls. +//! +//! This module is an unsafe compiler integration point, not a collective API. + +/// Read a Cargo-provided registration and track its native build dependencies. +/// +/// Pass the environment variable name as a string literal inside the registration +/// getter. Materialized builds query the provider for file, directory and +/// environment dependencies before rustc emits dep-info. Host checks and rustdoc +/// do not query CUDA inputs. Cargo's timestamp freshness rules still apply; +/// replacements with unchanged or older timestamps may not be detected. +/// +/// This snippet requires a build script to set `MY_LIBRARY_REGISTRATION`; the +/// `native_library` example compiles and runs the same registration pattern. +/// +/// ```ignore +/// fn registration() -> &'static str { +/// cuda_device::library::registration!("MY_LIBRARY_REGISTRATION") +/// } +/// ``` +pub use cuda_macros::library_registration as registration; + +/// Invoke a library recipe identified by a constant registration getter. +/// +/// Call this marker directly inside a wrapper; the marker itself is not a +/// device function pointer or callable-trait value. +/// +/// `Registration` must be a function item returning one constant UTF-8 JSON +/// string. The compiler reads its MIR without calling it. `Args` must be a +/// tuple of the exact scalar and raw-pointer types the provider declares. +/// `T` supplies the scalar type, and the const parameters supply the exact +/// one-dimensional block shape `[BLOCK_THREADS, 1, 1]` and per-thread extent. +/// +/// # Safety +/// +/// The caller must meet the provider's pointer, lifetime, initialization, +/// participation, and launch-geometry requirements. Library scratch views in +/// one kernel alias one dynamic shared base; the caller must synchronize before +/// reusing it. This version does not permit mixing native dynamic +/// shared views with library calls. It returns no allocation or ownership token. +#[inline(never)] +pub unsafe fn cuda_oxide_library_v1_246e25db_call< + T: Copy, + Registration: Fn() -> &'static str + Copy, + Args: Copy, + const BLOCK_THREADS: usize, + const ITEMS_PER_THREAD: usize, +>( + _registration: Registration, + _args: Args, +) { + panic!("library::call_v1 called outside device compilation"); +} + +pub use cuda_oxide_library_v1_246e25db_call as call_v1; + +/// Invoke a full-block library operation with compiler-checked participation. +/// +/// The exact block shape comes from the root kernel's launch contract. +/// `uniform` is `(memory_pointer, scalar_controls...)`; `payload` contains +/// lane-local const or mutable payload pointers matching the provider ABI. +/// The compiler checks all call contexts before reconstructing the provider's +/// ordinary scalar/pointer ABI. +/// +/// # Safety +/// +/// The caller supplies the correct provider recipe, live aligned memory and +/// payload pointers, sufficient capacity, and the provider's initialization and +/// aliasing requirements. Group checks do not prove bounds or exclusive memory +/// ownership. Scratch aliases the kernel's dynamic shared base; the caller must +/// synchronize before conflicting scratch or output reuse. Completion never +/// follows from ending a Rust borrow or running a destructor. +#[inline(never)] +pub unsafe fn cuda_oxide_library_v1_246e25db_checked_block_call< + T: Copy, + Registration: Fn() -> &'static str + Copy, + UniformArgs: Copy, + PayloadArgs: Copy, + const ITEMS_PER_THREAD: usize, +>( + _registration: Registration, + _uniform: UniformArgs, + _payload: PayloadArgs, +) { + // A backend which does not recognize this version must fail to link rather + // than silently compiling a panic/trap as the collective implementation. + unsafe extern "C" { + fn cuda_oxide_checked_block_call_requires_supported_backend(); + } + unsafe { cuda_oxide_checked_block_call_requires_supported_backend() } +} + +pub use cuda_oxide_library_v1_246e25db_checked_block_call as checked_block_call_v1; + +/// Invoke a scratch-free full-block operation with a typed memory witness. +/// +/// The compiler accepts `&[T]` for reads, a reference to +/// `DisjointSlice>` for reads, and a mutable reference to +/// that carrier for writes. It tracks the original kernel allocation through +/// helpers and requires a full-block barrier before conflicting reuse. +/// The witness is not passed to the native function. `uniform` and `payload` +/// have the same roles as [`checked_block_call_v1`]. +/// +/// # Safety +/// +/// The native pointer and accessed range must describe `memory`. The wrapper +/// must validate capacity, canonical group ownership, arithmetic, and bounds +/// before forming that pointer. Native memory effects must stay within that +/// range and the lane-local payload; `WRITE = false` forbids writes to `memory`. +/// The recipe, payload initialization and native ABI must be correct. Compiler +/// provenance and participation checks do not verify these obligations of the +/// unsafe wrapper implementation. +#[inline(never)] +pub unsafe fn cuda_oxide_library_v1_246e25db_checked_block_resource_call< + T: Copy, + Registration: Fn() -> &'static str + Copy, + Memory, + UniformArgs: Copy, + PayloadArgs: Copy, + const ITEMS: usize, + const WRITE: bool, +>( + _registration: Registration, + _memory: Memory, + _uniform: UniformArgs, + _payload: PayloadArgs, +) { + unsafe extern "C" { + fn cuda_oxide_checked_block_resource_call_requires_supported_backend(); + } + unsafe { cuda_oxide_checked_block_resource_call_requires_supported_backend() } +} + +pub use cuda_oxide_library_v1_246e25db_checked_block_resource_call as checked_block_resource_call_v1; + +/// Internal full-block scratch policy. Its private token has no address identity. +/// Allocation and synchronization are finalized per rooted static use site. +/// This descriptor is neither a Rust borrow of shared memory nor completion. +pub struct StorageDescriptor< + const CAPACITY: usize = 0, + const ALIGN: usize = 0, + const AUTO_SYNC: bool = false, + const EXCLUSIVE: bool = false, +> { + _token: u8, +} + +/// A compiler-planned typed shared view. No Rust reference is created. +/// Releasing the view does not synchronize or initialize its contents. +#[repr(transparent)] +pub struct StorageReservation { + pointer: *mut T, +} + +impl StorageReservation { + /// Return the raw shared pointer. Dereferencing it requires the caller to + /// prove initialization, bounds, aliasing and phase synchronization. + #[inline(always)] + pub fn as_mut_ptr(&self) -> *mut T { + self.pointer + } +} + +/// Create a compiler-owned descriptor origin at this rooted static call site. +/// Zero capacity/alignment mean inferred; explicit alignment is a power of two. +/// +/// # Safety +/// All block lanes must execute this marker with the same policies and origin. +/// Only a supported compiler may interpret the returned descriptor. +#[inline(never)] +pub unsafe fn cuda_oxide_library_v1_246e25db_storage_new< + const CAPACITY: usize, + const ALIGN: usize, + const AUTO_SYNC: bool, + const EXCLUSIVE: bool, +>() -> StorageDescriptor { + unsafe extern "C" { + fn cuda_oxide_storage_requires_supported_backend(); + } + unsafe { cuda_oxide_storage_requires_supported_backend() } + StorageDescriptor { _token: 0 } +} +pub use cuda_oxide_library_v1_246e25db_storage_new as storage_new_v1; + +/// Consume a descriptor and select its policy before any use. +/// +/// # Safety +/// `Old` and `New` must be genuine storage descriptors. The compiler verifies +/// their policies and preserves the original static identity. +#[inline(never)] +pub unsafe fn cuda_oxide_library_v1_246e25db_storage_reconfigure(_old: Old) -> New { + unsafe extern "C" { + fn cuda_oxide_storage_reconfigure_requires_supported_backend() -> !; + } + unsafe { cuda_oxide_storage_reconfigure_requires_supported_backend() } +} +pub use cuda_oxide_library_v1_246e25db_storage_reconfigure as storage_reconfigure_v1; + +/// Invoke a full-block native operation with planned storage and a memory witness. +/// `storage` is `()` (managed reuse) or a mutable borrow of a genuine descriptor. +/// +/// # Safety +/// The obligations of [`checked_block_resource_call_v1`] apply. Provider scratch +/// effects must fit its declared layout. Explicit manual storage requires a +/// full-block barrier before reuse; automatic storage adds a trailing barrier. +#[inline(never)] +pub unsafe fn cuda_oxide_library_v1_246e25db_checked_block_storage_call< + T: Copy, + Registration: Fn() -> &'static str + Copy, + Memory, + Storage, + UniformArgs: Copy, + PayloadArgs: Copy, + const ITEMS_PER_THREAD: usize, + const WRITE: bool, +>( + _registration: Registration, + _memory: Memory, + _storage: Storage, + _uniform: UniformArgs, + _payload: PayloadArgs, +) { + unsafe extern "C" { + fn cuda_oxide_storage_requires_supported_backend(); + } + unsafe { cuda_oxide_storage_requires_supported_backend() } +} +pub use cuda_oxide_library_v1_246e25db_checked_block_storage_call as checked_block_storage_call_v1; + +/// Reserve a typed view at a static use site; all lanes receive a raw address. +/// +/// # Safety +/// Descriptor and participation must be block-uniform. `T` must be a supported +/// numeric scalar, `N` positive, and alignment zero or a power of two. The caller +/// initializes and accesses shared elements without conflicting raw accesses. +/// Automatic trailing synchronization is incompatible with reservations. +#[inline(never)] +pub unsafe fn cuda_oxide_library_v1_246e25db_storage_reserve< + T: Copy, + Descriptor, + const N: usize, + const ALIGN: usize, +>( + _descriptor: &mut Descriptor, +) -> StorageReservation { + unsafe extern "C" { + fn cuda_oxide_storage_requires_supported_backend(); + } + unsafe { cuda_oxide_storage_requires_supported_backend() } + StorageReservation { + pointer: core::ptr::null_mut(), + } +} +pub use cuda_oxide_library_v1_246e25db_storage_reserve as storage_reserve_v1; + +/// End a reservation view without completing pending shared or output accesses. +/// +/// # Safety +/// No subsequent raw access may use this view. Synchronize the full block after +/// release and before any overlapping reuse. Dropping/forgetting is not release. +#[inline(never)] +pub unsafe fn cuda_oxide_library_v1_246e25db_storage_release( + _reservation: StorageReservation, +) { + unsafe extern "C" { + fn cuda_oxide_storage_requires_supported_backend(); + } + unsafe { cuda_oxide_storage_requires_supported_backend() } +} +pub use cuda_oxide_library_v1_246e25db_storage_release as storage_release_v1; + +/// Invoke a synchronous native operation with a compiler-rooted scalar callback. +/// +/// The thunk must be a function item with signature +/// `unsafe extern "C" fn(*const Environment, i32, i32) -> i32`. +/// `Environment` is a function item or an immutable closure capturing fixed-width +/// numeric scalars by value. The first version supports `T = i32` only. +/// The compiler retains the exact thunk and supplies its symbol to the provider; +/// neither witness is passed as a runtime function pointer. +/// +/// # Safety +/// +/// All requirements of [`call_v1`] apply. Every participant must use the same +/// callable specialization and capture values. The provider may synchronously +/// pass the opaque environment pointer back to the thunk, but must not inspect, +/// mutate, retain, free, or publish it. The environment remains live throughout +/// the native call. The callback must not perform collectives, synchronization, +/// shared-memory access, foreign calls, or indirect calls. This raw boundary +/// does not prove capture-value uniformity or pointer ownership. +#[inline(never)] +pub unsafe fn cuda_oxide_library_v1_246e25db_call_with_callback< + T: Copy, + Registration: Fn() -> &'static str + Copy, + Args: Copy, + Environment: Copy, + Thunk: Copy, + const BLOCK_THREADS: usize, + const ITEMS_PER_THREAD: usize, +>( + _registration: Registration, + _args: Args, + _thunk: Thunk, + _environment: &Environment, +) { + unsafe extern "C" { + fn cuda_oxide_callback_call_requires_supported_backend(); + } + unsafe { cuda_oxide_callback_call_requires_supported_backend() } +} + +pub use cuda_oxide_library_v1_246e25db_call_with_callback as call_with_callback_v1; diff --git a/cuda-oxide/crates/cuda-device/src/shared.rs b/cuda-oxide/crates/cuda-device/src/shared.rs index a6a7ed0c66..7ea7e73128 100644 --- a/cuda-oxide/crates/cuda-device/src/shared.rs +++ b/cuda-oxide/crates/cuda-device/src/shared.rs @@ -496,6 +496,28 @@ impl DynamicSharedArray { unreachable!("DynamicSharedArray::get called outside CUDA kernel context") } + /// Gets the block's dynamic shared-memory base with a fixed `T` extent. + /// + /// The compiler records `size_of::()` and the stronger of `ALIGN` and + /// `align_of::()` for checked launch preparation. Use an array or a + /// structure to describe the complete region. Every call returns the same + /// base address: multiple views share storage, they do not allocate separate + /// regions. Their required extents and alignments combine by maximum. + /// Inference follows Rust views through direct calls. Dynamic-base use in + /// external code, indirect calls, or inline assembly must fit a fixed view + /// declared on that path; the compiler does not inspect those accesses. + /// + /// # Safety + /// + /// Access only the declared `T` region, initialize values before reading, + /// and synchronize conflicting accesses across threads. Concurrent views + /// alias; this function establishes neither exclusive ownership nor a + /// synchronization barrier. Raw launches must allocate sufficient memory. + #[inline(never)] + pub unsafe fn get_fixed() -> *mut T { + unreachable!("DynamicSharedArray::get_fixed called outside CUDA kernel context") + } + /// Get a raw byte pointer to the start of dynamic shared memory. /// /// This is useful for manual memory management or when working @@ -766,3 +788,9 @@ impl SharedPtr { } include!("generated/shared_sreg.rs"); + +/// Selects compiler-produced dynamic shared-memory requirements. +/// Emitted by `#[launch_contract(dynamic_shared = inferred)]`. +#[doc(hidden)] +#[inline(never)] +pub fn __dynamic_shared_inferred() {} diff --git a/cuda-oxide/crates/cuda-device/tests/compile_fail/runtime_tile_wrong_domain.stderr b/cuda-oxide/crates/cuda-device/tests/compile_fail/runtime_tile_wrong_domain.stderr index 44c8f4cf96..ac8d9d6bed 100644 --- a/cuda-oxide/crates/cuda-device/tests/compile_fail/runtime_tile_wrong_domain.stderr +++ b/cuda-oxide/crates/cuda-device/tests/compile_fail/runtime_tile_wrong_domain.stderr @@ -6,6 +6,7 @@ error[E0277]: the trait bound `DisjointSlice<'static, f32, RuntimeRowMajorTiles< | = help: the trait `cuda_device::disjoint::__LaunchContractDisjointSlice` is not implemented for `DisjointSlice<'static, f32, RuntimeRowMajorTiles<1, 1>>` = help: the following other types implement trait `cuda_device::disjoint::__LaunchContractDisjointSlice`: + `DisjointSlice<'_, T, CollectiveTiles>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `DisjointSlice<'_, T, Index2D>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `DisjointSlice<'_, T, Index2D>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `DisjointSlice<'_, T, LinearTiles>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` @@ -13,8 +14,7 @@ error[E0277]: the trait bound `DisjointSlice<'static, f32, RuntimeRowMajorTiles< `DisjointSlice<'_, T, Runtime2DIndex>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `DisjointSlice<'_, T, Runtime2DIndex>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `DisjointSlice<'_, T, RuntimeRowMajorTiles>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` - `DisjointSlice<'_, T, WarpIndex>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` - `DisjointSlice<'_, T>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` + and $N others note: required by a bound in `requires_one_dimensional_contract` --> tests/compile_fail/runtime_tile_wrong_domain.rs:10:41 | diff --git a/cuda-oxide/crates/cuda-device/tests/compile_fail/thread_index_row_major_tiles_not_domain1.stderr b/cuda-oxide/crates/cuda-device/tests/compile_fail/thread_index_row_major_tiles_not_domain1.stderr index 8ad3a83cdc..a462c97f21 100644 --- a/cuda-oxide/crates/cuda-device/tests/compile_fail/thread_index_row_major_tiles_not_domain1.stderr +++ b/cuda-oxide/crates/cuda-device/tests/compile_fail/thread_index_row_major_tiles_not_domain1.stderr @@ -6,6 +6,7 @@ error[E0277]: the trait bound `DisjointSlice<'static, u32, RowMajorTiles<2, 4, 1 | = help: the trait `cuda_device::disjoint::__LaunchContractDisjointSlice` is not implemented for `DisjointSlice<'static, u32, RowMajorTiles<2, 4, 16>>` = help: the following other types implement trait `cuda_device::disjoint::__LaunchContractDisjointSlice`: + `DisjointSlice<'_, T, CollectiveTiles>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `DisjointSlice<'_, T, Index2D>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `DisjointSlice<'_, T, Index2D>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `DisjointSlice<'_, T, LinearTiles>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` @@ -13,8 +14,7 @@ error[E0277]: the trait bound `DisjointSlice<'static, u32, RowMajorTiles<2, 4, 1 `DisjointSlice<'_, T, Runtime2DIndex>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `DisjointSlice<'_, T, Runtime2DIndex>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `DisjointSlice<'_, T, RuntimeRowMajorTiles>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` - `DisjointSlice<'_, T, WarpIndex>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` - `DisjointSlice<'_, T>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` + and $N others note: required by a bound in `requires_one_dimensional_contract` --> tests/compile_fail/thread_index_row_major_tiles_not_domain1.rs:10:41 | diff --git a/cuda-oxide/crates/cuda-host/Cargo.toml b/cuda-oxide/crates/cuda-host/Cargo.toml index 3906514968..f29218e7c9 100644 --- a/cuda-oxide/crates/cuda-host/Cargo.toml +++ b/cuda-oxide/crates/cuda-host/Cargo.toml @@ -33,6 +33,7 @@ async = ["dep:cuda-async", "cuda-macros/async"] [dev-dependencies] libloading = { workspace = true } -# The crates.io release cuda-core re-exports bundle types from; tests -# construct payloads with it so they use the same crate instance. +cuda-device = { workspace = true } +# The same artifact implementation cuda-core re-exports; tests construct +# payloads with it so they use the same crate instance. oxide-artifacts = { workspace = true } diff --git a/cuda-oxide/crates/cuda-host/src/embedded.rs b/cuda-oxide/crates/cuda-host/src/embedded.rs index 48776448f2..e8cc4ff503 100644 --- a/cuda-oxide/crates/cuda-host/src/embedded.rs +++ b/cuda-oxide/crates/cuda-host/src/embedded.rs @@ -12,6 +12,7 @@ pub use cuda_core::embedded::{ embedded_modules_from_current_exe, }; use cuda_core::{CudaContext, CudaModule, DriverError}; +use std::collections::BTreeMap; use std::sync::Arc; use thiserror::Error; @@ -45,6 +46,16 @@ pub enum EmbeddedModuleError { #[error("embedded CUDA module '{name}' contains invalid PTX: {reason}")] InvalidPtx { name: String, reason: String }, + /// Bundles disagree about the requirements of the same concrete entry. + #[error("conflicting compiled launch facts for kernel '{symbol}'")] + ConflictingLaunchFacts { symbol: String }, + + /// Generic loading needs one complete native image or a PTX-only bundle set. + #[error( + "generic CUDA loading requires one complete cubin bundle or only PTX bundles; incompatible embedded bundles: {names:?}" + )] + IncompatibleGenericArtifacts { names: Vec }, + /// NVVM IR or LTOIR payload compilation failed. #[error("failed to build embedded CUDA module: {0}")] Ltoir(#[from] ltoir::LtoirError), @@ -135,9 +146,99 @@ pub fn load_all_ptx_bundles_merged( ctx: &Arc, ) -> Result, EmbeddedModuleError> { let bundles = artifact_bundles_from_current_exe()?; - let merged = merge_ptx_bundles(&bundles)?; + load_merged_ptx(ctx, &bundles) +} + +/// Load generic kernel specializations from the image containing `anchor`. +/// +/// A single materialized cubin contains all required concrete entries and keeps +/// their globals in one CUDA module. Multiple native bundles, or native and PTX +/// bundles together, are rejected because independently loaded images would +/// not preserve shared device globals. A PTX-only set uses the existing merge. +/// Empty anchor-only bundles do not participate in payload selection. +/// +/// `anchor` must belong to the host image whose artifacts are to be loaded. +/// Generated loaders anchor the defining crate's host code; the instantiating +/// crate must be linked into that same image. This applies to PTX builds too. +/// Image discovery has the same platform and file-identity requirements as +/// [`load_embedded_module_from_anchor`]. Entry lookup uses the exact concrete +/// symbol name and retains each entry's compiled launch facts. +pub fn load_generic_embedded_module_from_anchor( + ctx: &Arc, + anchor: &u8, +) -> Result, EmbeddedModuleError> { + let bundles = artifact_bundles_containing(anchor)?; + load_generic_bundles(ctx, &bundles) +} + +/// Load generic kernel specializations from the current executable. +/// +/// Uses the same native-artifact selection and PTX merge as +/// [`load_generic_embedded_module_from_anchor`]. Generated loaders use this +/// executable-only discovery on platforms without mapped-image support. +pub fn load_generic_embedded_module( + ctx: &Arc, +) -> Result, EmbeddedModuleError> { + let bundles = artifact_bundles_from_current_exe()?; + load_generic_bundles(ctx, &bundles) +} - let module = ctx.load_module_from_image(merged.as_bytes())?; +fn load_generic_bundles( + ctx: &Arc, + bundles: &[OwnedArtifactBundle], +) -> Result, EmbeddedModuleError> { + if let Some(bundle) = generic_cubin_bundle(bundles)? { + let module = load_bundle(ctx, bundle)?; + crate::entry_registry::register_module_entries( + &module, + bundle + .entries + .iter() + .filter(|entry| entry.kind == oxide_artifacts::ArtifactEntryKind::Kernel) + .map(|entry| entry.symbol.clone()) + .collect(), + ); + Ok(module) + } else { + load_merged_ptx(ctx, bundles) + } +} + +fn generic_cubin_bundle( + bundles: &[OwnedArtifactBundle], +) -> Result, EmbeddedModuleError> { + let active: Vec<_> = bundles + .iter() + .filter(|bundle| !bundle.payloads.is_empty()) + .collect(); + match active.as_slice() { + [] => Err(EmbeddedModuleError::NoModules), + [bundle] if bundle.payload(ArtifactPayloadKind::Cubin).is_some() => Ok(Some(bundle)), + _ if active.iter().all(|bundle| { + bundle.payload(ArtifactPayloadKind::Cubin).is_none() + && bundle.payload(ArtifactPayloadKind::Ptx).is_some() + }) => + { + Ok(None) + } + _ => Err(EmbeddedModuleError::IncompatibleGenericArtifacts { + names: active.iter().map(|bundle| bundle.name.clone()).collect(), + }), + } +} + +fn load_merged_ptx( + ctx: &Arc, + bundles: &[OwnedArtifactBundle], +) -> Result, EmbeddedModuleError> { + let merged = merge_ptx_bundles(bundles)?; + + let facts = merged_launch_facts( + bundles + .iter() + .filter(|bundle| bundle.payload(ArtifactPayloadKind::Ptx).is_some()), + )?; + let module = ctx.load_module_from_image_with_launch_facts(merged.as_bytes(), facts)?; // Retain the merged module's `.entry` names (a few dozen bytes per // kernel). If a later `_TID_` generic-kernel lookup misses while a // same-base entry exists under a different hash, the launch paths can @@ -147,6 +248,32 @@ pub fn load_all_ptx_bundles_merged( Ok(module) } +fn merged_launch_facts<'a>( + bundles: impl IntoIterator, +) -> Result, EmbeddedModuleError> { + // Keep None in the comparison: absent facts must not silently replace a + // known zero-size plan, or inherit another bundle's contract. + let mut entries = BTreeMap::new(); + for bundle in bundles { + for entry in &bundle.entries { + if entry.kind != oxide_artifacts::ArtifactEntryKind::Kernel { + continue; + } + if let Some(previous) = entries.insert(entry.symbol.clone(), entry.launch_facts) + && previous != entry.launch_facts + { + return Err(EmbeddedModuleError::ConflictingLaunchFacts { + symbol: entry.symbol.clone(), + }); + } + } + } + Ok(entries + .into_iter() + .filter_map(|(symbol, facts)| facts.map(|facts| (symbol, facts))) + .collect()) +} + /// Merge the PTX payloads of `bundles`, in iteration order, into one PTX /// module string: the first PTX bundle keeps its `.version` / `.target` / /// `.address_size` header directives, every later one contributes its body @@ -297,12 +424,13 @@ fn load_bundle( ctx: &Arc, bundle: &OwnedArtifactBundle, ) -> Result, EmbeddedModuleError> { + let facts = merged_launch_facts([bundle])?; if let Some(cubin) = bundle.payload(ArtifactPayloadKind::Cubin) { - return Ok(ctx.load_module_from_image(cubin)?); + return Ok(ctx.load_module_from_image_with_launch_facts(cubin, facts)?); } if let Some(ptx) = bundle.payload(ArtifactPayloadKind::Ptx) { - return Ok(ctx.load_module_from_image(ptx)?); + return Ok(ctx.load_module_from_image_with_launch_facts(ptx, facts)?); } if let Some(nvvm_ir) = bundle.payload(ArtifactPayloadKind::NvvmIr) { @@ -322,7 +450,7 @@ fn load_bundle( bundle.compile_options, )?, }; - return Ok(ctx.load_module_from_image(&image)?); + return Ok(ctx.load_module_from_image_with_launch_facts(&image, facts)?); } if let Some(ltoir) = bundle.payload(ArtifactPayloadKind::Ltoir) { @@ -342,7 +470,7 @@ fn load_bundle( bundle.compile_options, )?, }; - return Ok(ctx.load_module_from_image(&image)?); + return Ok(ctx.load_module_from_image_with_launch_facts(&image, facts)?); } Err(EmbeddedModuleError::UnsupportedPayload { @@ -389,6 +517,59 @@ mod tests { } } + #[test] + fn merged_kernel_facts_preserve_identity_and_reject_conflicts() { + use oxide_artifacts::{ + ArtifactBundleSpec, ArtifactEntryKind, ArtifactEntrySpec, ArtifactPayloadSpec, + KernelLaunchFacts, + }; + let facts = KernelLaunchFacts { + exact_block: Some((32, 1, 1)), + dynamic_shared_bytes: 0, + dynamic_shared_alignment: 16, + }; + let bundle = |symbol: &str, facts| { + let mut entry = ArtifactEntrySpec::new(symbol, ArtifactEntryKind::Kernel); + entry.launch_facts = facts; + let spec = ArtifactBundleSpec::new("test", "sm_120") + .with_payload(ArtifactPayloadSpec::new( + ArtifactPayloadKind::Ptx, + "test.ptx", + b"ptx", + )) + .with_entry(entry); + let bytes = oxide_artifacts::build_artifact_blob(&spec).unwrap(); + OwnedArtifactBundle::from(oxide_artifacts::parse_artifact_blob(&bytes).unwrap()) + }; + let first = bundle("apply_items_1", Some(facts)); + let second = bundle( + "apply_items_4", + Some(KernelLaunchFacts { + dynamic_shared_bytes: 512, + ..facts + }), + ); + let merged = merged_launch_facts([&first, &second]).unwrap(); + assert_eq!(merged["apply_items_1"].dynamic_shared_bytes, 0); + assert_eq!(merged["apply_items_4"].dynamic_shared_bytes, 512); + for conflict in [ + bundle("apply_items_1", None), + bundle( + "apply_items_1", + Some(KernelLaunchFacts { + dynamic_shared_bytes: 128, + ..facts + }), + ), + ] { + assert!(matches!( + merged_launch_facts([&first, &conflict]), + Err(EmbeddedModuleError::ConflictingLaunchFacts { .. }) + )); + } + assert_eq!(merged_launch_facts([&first, &first]).unwrap().len(), 1); + } + #[test] fn target_arch_uses_bundle_sm_target() { assert_eq!( @@ -414,6 +595,49 @@ mod tests { } } + #[test] + fn generic_loading_selects_one_native_image_or_merges_ptx() { + let mut native = ptx_bundle("consumer", "native bytes"); + native.payloads[0].kind = ArtifactPayloadKind::Cubin; + let bundles = [bundle_with_target("sm_120"), native]; + assert!(std::ptr::eq( + generic_cubin_bundle(&bundles).unwrap().unwrap(), + &bundles[1] + )); + assert!( + generic_cubin_bundle(&[ptx_bundle("library", "ptx"), ptx_bundle("consumer", "ptx")]) + .unwrap() + .is_none() + ); + assert!(matches!( + generic_cubin_bundle(&[bundle_with_target("sm_120")]), + Err(EmbeddedModuleError::NoModules) + )); + } + + #[test] + fn generic_loading_rejects_native_ambiguity_and_mixed_payloads() { + for kind in [ + ArtifactPayloadKind::Cubin, + ArtifactPayloadKind::Ptx, + ArtifactPayloadKind::NvvmIr, + ArtifactPayloadKind::Ltoir, + ] { + let mut native = ptx_bundle("consumer", "native bytes"); + native.payloads[0].kind = ArtifactPayloadKind::Cubin; + let mut other = ptx_bundle("library", "other bytes"); + other.payloads[0].kind = kind; + // Entry-free payloads can still define device globals. + assert!(other.entries.is_empty()); + let error = generic_cubin_bundle(&[native, other]).unwrap_err(); + assert!(matches!( + error, + EmbeddedModuleError::IncompatibleGenericArtifacts { names } + if names == ["consumer", "library"] + )); + } + } + /// The merge is order-explicit: exactly one header set (the first PTX /// bundle's), every body present in iteration order, non-PTX bundles /// skipped. #1277's collision-free-namespace example checks the same diff --git a/cuda-oxide/crates/cuda-host/src/entry_registry.rs b/cuda-oxide/crates/cuda-host/src/entry_registry.rs index b12885c3af..ee7751c98f 100644 --- a/cuda-oxide/crates/cuda-host/src/entry_registry.rs +++ b/cuda-oxide/crates/cuda-host/src/entry_registry.rs @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -//! Retained PTX entry names for merged modules, and the host/device +//! Retained entry names for loaded artifacts and merged PTX, and the host/device //! type-identity divergence diagnosis built on them. //! //! # Why this exists @@ -20,7 +20,8 @@ //! opaque `DriverError(500, "named symbol not found")`. //! //! To make that failure self-explaining, [`crate::load_all_ptx_bundles_merged`] -//! retains the `.entry` names it already parses out of the merged PTX. When a +//! retains the `.entry` names it already parses out of the merged PTX. Native +//! generic loading retains exact entry names from its artifact metadata. When a //! generic-kernel lookup later misses, the macro-generated launch paths call //! [`diagnose_generic_kernel_load_error`] / //! [`panic_generic_kernel_load_failed`]: if the module holds the same base @@ -61,8 +62,7 @@ const TYPE_ID_HASH_LEN: usize = 32; /// One retained record: the (weakly held) owning module and its entry names. type RetainedEntries = (Weak, Arc>); -/// Process-wide map from a loaded module to the PTX `.entry` names it was -/// built from. +/// Process-wide map from a loaded module to its retained kernel entry names. /// /// Keyed by allocation identity of the owning [`Arc`]: a [`Weak`] pins the /// allocation (not the value), so pointer equality against a live `Arc` is @@ -119,10 +119,15 @@ fn merged_module_entries() -> &'static EntryRegistry { /// driver was happy with. pub(crate) fn register_merged_module_entries(module: &Arc, merged_ptx: &str) { if let Some(entries) = entry_names_from_ptx(merged_ptx) { - merged_module_entries().register(module, entries); + register_module_entries(module, entries); } } +/// Retains exact entry names from a loaded artifact or parsed merged PTX. +pub(crate) fn register_module_entries(module: &Arc, entries: Vec) { + merged_module_entries().register(module, entries); +} + /// Parses PTX text and returns the names of its `.entry` kernels. /// /// `.func` device helpers are excluded: only `.entry` symbols are launchable, @@ -173,7 +178,7 @@ fn divergent_entries_in(entries: &[String], requested: &str) -> Vec { /// name but carry a different `_TID_` type-identity hash. /// /// Empty when `requested` is not a `_TID_` name, when `module` was not loaded -/// through [`crate::load_all_ptx_bundles_merged`], or when no same-base entry +/// through a generic native or merged-PTX loader, or when no same-base entry /// exists; i.e. whenever the miss is an ordinary miss. pub fn divergent_type_id_entries(module: &Arc, requested: &str) -> Vec { match merged_module_entries().entries_for(module) { diff --git a/cuda-oxide/crates/cuda-host/src/lib.rs b/cuda-oxide/crates/cuda-host/src/lib.rs index 5e6a866a7a..03b3fec847 100644 --- a/cuda-oxide/crates/cuda-host/src/lib.rs +++ b/cuda-oxide/crates/cuda-host/src/lib.rs @@ -135,7 +135,8 @@ pub use cuda_async::simt::launch::{ pub use embedded::{ EmbeddedModuleError, load_all_ptx_bundles_merged, load_embedded_module, - load_embedded_module_from_anchor, load_first_embedded_module, merge_ptx_bundles, + load_embedded_module_from_anchor, load_first_embedded_module, load_generic_embedded_module, + load_generic_embedded_module_from_anchor, merge_ptx_bundles, }; pub use entry_registry::{ diagnose_generic_kernel_load_error, divergent_type_id_entries, panic_generic_kernel_load_failed, diff --git a/cuda-oxide/crates/cuda-host/src/type_id.rs b/cuda-oxide/crates/cuda-host/src/type_id.rs index 6d480a4bf4..2f853f7762 100644 --- a/cuda-oxide/crates/cuda-host/src/type_id.rs +++ b/cuda-oxide/crates/cuda-host/src/type_id.rs @@ -65,7 +65,11 @@ pub fn type_id_u128() -> u128 { /// hashes the concrete `kernel::` function-item type rather than its /// coerced function-pointer signature. Keeping the item uncoerced is important: /// the item type retains both the function definition and its generic arguments. -#[inline] +// rustc-codegen-cuda/src/collector.rs::kernel_root_instance recognizes this +// exact crate/module/item path and recovers the concrete function-item type. +// Keep the call through optimized MIR collection by disabling MIR inlining; +// reifying a function pointer instead would also emit the kernel body on host. +#[inline(never)] pub fn type_id_u128_of_val(_: &T) -> u128 { type_id_u128::() } @@ -137,6 +141,10 @@ mod tests { fn function_item_hash_includes_const_arguments() { fn const_item() {} + assert_eq!( + core::any::type_name_of_val(&type_id_u128_of_val::), + "cuda_host::type_id::type_id_u128_of_val" + ); assert_ne!( type_id_u128_of_val(&const_item::<0>), type_id_u128_of_val(&const_item::<1>) diff --git a/cuda-oxide/crates/cuda-macros/Cargo.toml b/cuda-oxide/crates/cuda-macros/Cargo.toml index 440290c02a..7d0a4a8b0f 100644 --- a/cuda-oxide/crates/cuda-macros/Cargo.toml +++ b/cuda-oxide/crates/cuda-macros/Cargo.toml @@ -38,6 +38,8 @@ host = [] [dependencies] quote = { workspace = true } syn = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } proc-macro2 = { workspace = true } reserved-oxide-symbols = { workspace = true } # `#[tile_export]` builds its description with the shared vocabulary and embeds diff --git a/cuda-oxide/crates/cuda-macros/src/cuda_module/contract.rs b/cuda-oxide/crates/cuda-macros/src/cuda_module/contract.rs index 1281fd70d9..b243f8ff3c 100644 --- a/cuda-oxide/crates/cuda-macros/src/cuda_module/contract.rs +++ b/cuda-oxide/crates/cuda-macros/src/cuda_module/contract.rs @@ -22,6 +22,7 @@ use syn::{ #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum DynamicSharedContract { + Inferred, Exact(u32), Range { min_bytes: u32, max_bytes: u32 }, } @@ -95,8 +96,19 @@ impl Parse for LaunchContractArgs { } "dynamic_shared" => { reject_duplicate(&key, dynamic_shared.is_some())?; - let value: syn::LitInt = input.parse()?; - dynamic_shared = Some(value.base10_parse::()?); + dynamic_shared = Some(if input.peek(syn::LitInt) { + let value: syn::LitInt = input.parse()?; + DynamicSharedContract::Exact(value.base10_parse::()?) + } else { + let value: Ident = input.parse()?; + if value != "inferred" { + return Err(syn::Error::new( + value.span(), + "dynamic_shared expects a byte count or `inferred`", + )); + } + DynamicSharedContract::Inferred + }); } "dynamic_shared_range" => { reject_duplicate(&key, dynamic_shared_range.is_some())?; @@ -149,7 +161,7 @@ impl Parse for LaunchContractArgs { )); } let dynamic_shared = match (dynamic_shared, dynamic_shared_range) { - (Some(bytes), None) => DynamicSharedContract::Exact(bytes), + (Some(requirement), None) => requirement, (None, Some((min_bytes, max_bytes))) if min_bytes <= max_bytes => { DynamicSharedContract::Range { min_bytes, @@ -165,6 +177,12 @@ impl Parse for LaunchContractArgs { (None, None) => DynamicSharedContract::Exact(0), (Some(_), Some(_)) => unreachable!(), }; + if dynamic_shared == DynamicSharedContract::Inferred && dynamic_shared_alignment.is_some() { + return Err(syn::Error::new( + input.span(), + "inferred dynamic shared memory obtains alignment from the compiled layout", + )); + } let dynamic_shared_alignment = dynamic_shared_alignment.unwrap_or_else(|| { if dynamic_shared_max(dynamic_shared) == 0 { 1 @@ -216,6 +234,7 @@ fn parse_requires_relations(input: ParseStream) -> syn::Result> { pub(crate) fn dynamic_shared_max(contract: DynamicSharedContract) -> u32 { match contract { + DynamicSharedContract::Inferred => 0, DynamicSharedContract::Exact(bytes) => bytes, DynamicSharedContract::Range { max_bytes, .. } => max_bytes, } diff --git a/cuda-oxide/crates/cuda-macros/src/cuda_module/launchers.rs b/cuda-oxide/crates/cuda-macros/src/cuda_module/launchers.rs index 279a15cce4..5a15350100 100644 --- a/cuda-oxide/crates/cuda-macros/src/cuda_module/launchers.rs +++ b/cuda-oxide/crates/cuda-macros/src/cuda_module/launchers.rs @@ -239,6 +239,9 @@ pub(super) fn generate_cuda_module_launch_contract_impl( }); let alignment = contract.dynamic_shared_alignment; let dynamic_shared = match contract.dynamic_shared { + DynamicSharedContract::Inferred => { + quote! { ::cuda_core::DynamicSharedMemoryRequirement::Inferred } + } DynamicSharedContract::Exact(bytes) => quote! { ::cuda_core::DynamicSharedMemoryRequirement::Exact { bytes: #bytes, diff --git a/cuda-oxide/crates/cuda-macros/src/cuda_module/mod.rs b/cuda-oxide/crates/cuda-macros/src/cuda_module/mod.rs index 58d0902427..43a2ccaa2d 100644 --- a/cuda-oxide/crates/cuda-macros/src/cuda_module/mod.rs +++ b/cuda-oxide/crates/cuda-macros/src/cuda_module/mod.rs @@ -135,9 +135,9 @@ pub(crate) fn expand_cuda_module_inner( let cfg_attrs = &kernel.effective_cfg_attrs; Some(quote! { #(#cfg_attrs)* - // Consumed by the codegen collector. This enabled generic kernel - // requires run-time PTX bundle merging, which ahead-of-time cubin - // materialization cannot represent yet. + // Retain the existing generic-module identity marker consumed by + // the collector. PTX builds merge bundles; materialized builds + // require one complete native artifact at load time. #[doc(hidden)] #[used] #[allow(dead_code, non_upper_case_globals)] @@ -190,8 +190,21 @@ pub(crate) fn expand_cuda_module_inner( let mut __cuda_oxide_has_enabled_generic_kernel = false; #(#enable_generic_loader_statements)* let module = if __cuda_oxide_has_enabled_generic_kernel { - let _ = name; // merged load ignores the crate-name hint - ::cuda_host::load_all_ptx_bundles_merged(ctx)? + let _ = name; // monomorphizations can belong to the consuming crate + // This identifies the host image; artifact object retention is + // still handled by the compiler's existing embedding path. + #[cfg(any(target_os = "linux", target_os = "android"))] + { + static __CUDA_OXIDE_GENERIC_IMAGE: u8 = 0; + ::cuda_host::load_generic_embedded_module_from_anchor( + ctx, + &__CUDA_OXIDE_GENERIC_IMAGE, + )? + } + #[cfg(not(any(target_os = "linux", target_os = "android")))] + { + ::cuda_host::load_generic_embedded_module(ctx)? + } } else { #embedded_module_loader }; @@ -232,8 +245,12 @@ pub(crate) fn expand_cuda_module_inner( /// For a non-generic module, the selected package bundle must be /// the artifact compiled from this `cuda_module`; package names are /// not yet unique across all library and binary targets. For a - /// generic module, the merged PTX set must contain each matching - /// specialization and no conflicting entry definition. + /// generic module, the selected native artifact or merged PTX set + /// must contain each matching specialization and no conflicting + /// entry definition. + /// On Linux, the defining crate's loader and the crate instantiating + /// generic kernels must be linked into the same host image. Native + /// and PTX discovery read that mapped image, with no fallback. pub unsafe fn load_async( device_id: usize, ) -> ::core::result::Result { @@ -285,8 +302,12 @@ pub(crate) fn expand_cuda_module_inner( /// For a non-generic module, the selected package bundle must be /// the artifact compiled from this `cuda_module`; package names are /// not yet unique across all library and binary targets. For a - /// generic module, the merged PTX set must contain each matching - /// specialization and no conflicting entry definition. + /// generic module, the selected native artifact or merged PTX set + /// must contain each matching specialization and no conflicting + /// entry definition. + /// On Linux, the defining crate's loader and the crate instantiating + /// generic kernels must be linked into the same host image. Native + /// and PTX discovery read that mapped image, with no fallback. pub unsafe fn load( ctx: &::std::sync::Arc<::cuda_core::CudaContext>, ) -> ::core::result::Result { @@ -312,6 +333,9 @@ pub(crate) fn expand_cuda_module_inner( /// Every selected kernel must have the exact ABI and resource /// semantics declared by this `cuda_module`. A matching symbol /// name alone is not sufficient. + /// Generic instantiations must share the defining crate's host + /// image. On Linux, discovery reads that mapped image, without + /// falling back to the executable; the name hint is ignored. pub unsafe fn load_named( ctx: &::std::sync::Arc<::cuda_core::CudaContext>, name: &str, @@ -324,6 +348,9 @@ pub(crate) fn expand_cuda_module_inner( } } else { quote! { + /// Generic instantiations must share the defining crate's host + /// image. On Linux, discovery reads that mapped image, without + /// falling back to the executable; the name hint is ignored. pub fn load_named( ctx: &::std::sync::Arc<::cuda_core::CudaContext>, name: &str, @@ -342,7 +369,8 @@ pub(crate) fn expand_cuda_module_inner( /// /// Every loaded kernel must have the exact ABI and resource /// semantics declared by this `cuda_module`. A matching symbol - /// name alone is not sufficient. + /// name alone is not sufficient. Any attached compiled launch facts + /// must truthfully describe that exact loaded image and entry. pub unsafe fn from_module( module: ::std::sync::Arc<::cuda_core::CudaModule>, ) -> ::core::result::Result { diff --git a/cuda-oxide/crates/cuda-macros/src/kernel/codegen.rs b/cuda-oxide/crates/cuda-macros/src/kernel/codegen.rs index b4e55e7ebf..f32ec35928 100644 --- a/cuda-oxide/crates/cuda-macros/src/kernel/codegen.rs +++ b/cuda-oxide/crates/cuda-macros/src/kernel/codegen.rs @@ -139,8 +139,8 @@ pub(crate) fn generic_kernel_no_instantiation_tokens( /// is bound to the generic parameter `F` at the call site /// without moving the closure — the caller still needs the /// closure value to push as the kernel argument right - /// after. Then forces rustc to emit a CGU entry for the - /// concrete `#kernel_name::<...>` instantiation. Returns + /// after. Retains the concrete `#kernel_name::<...>` + /// function-item type for device collection. Returns /// the PTX export name produced by the kernel's /// `GenericCudaKernel::ptx_name()` impl, which is the /// single source of truth for the on-wire name on the @@ -431,13 +431,11 @@ fn generate_generic_cuda_kernel_impl( /// Retains this concrete kernel specialization and returns its PTX entry name. /// - /// The reify cast is what makes rustc collect the monomorphized kernel - /// item for the device backend; `black_box` keeps the otherwise unused - /// cast from being optimized away without touching any pointer. + /// The type-hash helper retains the concrete kernel function-item type + /// for device collection without emitting its body on the host. #(#cfg_attrs)* #[inline(never)] #vis fn #ptx_name_fn #generics () -> &'static str #where_clause { - ::core::hint::black_box(#kernel_name #kernel_turbofish as *const ()); <#marker_type as ::cuda_host::GenericCudaKernel>::ptx_name() } } diff --git a/cuda-oxide/crates/cuda-macros/src/kernel/scope.rs b/cuda-oxide/crates/cuda-macros/src/kernel/scope.rs index 872e34848a..8fc20d7391 100644 --- a/cuda-oxide/crates/cuda-macros/src/kernel/scope.rs +++ b/cuda-oxide/crates/cuda-macros/src/kernel/scope.rs @@ -584,13 +584,18 @@ pub(crate) fn is_kernel_configuration_marker(statement: &Stmt) -> bool { || segments[0].ident != "cuda_device" || !matches!(segments[0].arguments, PathArguments::None) || !matches!(segments[1].arguments, PathArguments::None) - || !matches!(segments[2].arguments, PathArguments::AngleBracketed(_)) { return false; } let module = &segments[1].ident; let marker = &segments[2].ident; + if module == "shared" && marker == "__dynamic_shared_inferred" { + return !unsafe_wrapped && matches!(segments[2].arguments, PathArguments::None); + } + if !matches!(segments[2].arguments, PathArguments::AngleBracketed(_)) { + return false; + } if unsafe_wrapped { module == "thread" && (marker == "__launch_contract_config" || marker == "__grid_constant_config") diff --git a/cuda-oxide/crates/cuda-macros/src/launch_attrs.rs b/cuda-oxide/crates/cuda-macros/src/launch_attrs.rs index b10f4cd455..411ef20210 100644 --- a/cuda-oxide/crates/cuda-macros/src/launch_attrs.rs +++ b/cuda-oxide/crates/cuda-macros/src/launch_attrs.rs @@ -181,7 +181,14 @@ pub(crate) fn inject_launch_contract_markers(args: &LaunchContractArgs, input: & next += 1; } - if dynamic_shared_max(args.dynamic_shared) != 0 { + if args.dynamic_shared == crate::cuda_module::contract::DynamicSharedContract::Inferred { + input.block.stmts.insert( + next, + parse_quote! { + ::cuda_device::shared::__dynamic_shared_inferred(); + }, + ); + } else if dynamic_shared_max(args.dynamic_shared) != 0 { let alignment = args.dynamic_shared_alignment as usize; let alignment_marker: syn::Stmt = parse_quote! { ::cuda_device::shared::__dynamic_shared_alignment::<#alignment>(); diff --git a/cuda-oxide/crates/cuda-macros/src/lib.rs b/cuda-oxide/crates/cuda-macros/src/lib.rs index 0a3d738b55..3e91b80752 100644 --- a/cuda-oxide/crates/cuda-macros/src/lib.rs +++ b/cuda-oxide/crates/cuda-macros/src/lib.rs @@ -29,7 +29,7 @@ //! so the same switch that turns host emission on also adds the `cuda-host` //! dependency that resolves it. -#![feature(proc_macro_def_site, proc_macro_tracked_env)] +#![feature(proc_macro_def_site, proc_macro_tracked_env, proc_macro_tracked_path)] mod common; mod cuda_module; @@ -37,6 +37,7 @@ mod device; mod kernel; mod launch; mod launch_attrs; +mod library; mod printf; mod ptx_asm; mod tile_export; @@ -649,8 +650,23 @@ pub fn launch_bounds(attr: TokenStream, item: TokenStream) -> TokenStream { /// the rest of the declared contract without preparation. /// /// Dynamic shared memory may be fixed with `dynamic_shared = BYTES` or bounded -/// with `dynamic_shared_range = (MIN, MAX)`. The byte extent remains an author -/// contract because arbitrary pointer arithmetic cannot be inferred. When the +/// with `dynamic_shared_range = (MIN, MAX)`. With `dynamic_shared = inferred`, +/// the compiler records the complete byte extent and alignment of unsafe +/// `DynamicSharedArray::get_fixed` views, including calls through helpers. +/// These views alias one dynamic base region; their extents combine by maximum, +/// not by addition. Mixing inferred views with unconstrained `get`, `get_raw`, +/// or `offset` access is rejected. Prepared launches require the matching +/// compiled artifact facts; `LaunchConfig1D::auto_shared` (and its 2D/3D +/// counterparts) selects the recorded byte count. An explicit byte count is +/// still checked before launch. Inferred alignment comes from the compiled +/// layouts, so it cannot be combined with `dynamic_shared_alignment`. +/// Inference follows Rust views through direct calls. Unsafe external code, +/// indirect calls and inline assembly may use the dynamic base only within a +/// fixed view declared on that path; their accesses are not analyzed. An inferred +/// kernel with no declared views records zero bytes. +/// +/// For raw dynamic pointers, the byte extent remains an author contract because +/// arbitrary pointer arithmetic cannot be inferred. When the /// maximum is non-zero, the macro injects a compiler marker whose call is /// removed before code generation. The declared `dynamic_shared_alignment` /// therefore becomes a minimum alignment in generated PTX without adding @@ -1015,10 +1031,14 @@ pub fn tile_export(attr: TokenStream, item: TokenStream) -> TokenStream { // NVVM Attributes for Device FFI // ============================================================================ -/// Marks a device function as convergent. +/// Marks an external device function declaration as convergent. /// -/// Convergent functions must be called by all threads in a warp/block together. -/// This prevents the optimizer from moving calls across control flow boundaries. +/// LLVM forbids transformations that add control dependencies to the call, +/// including before external device definitions are linked. The annotation +/// does not make threads reach the call or guarantee reconvergence. Callers +/// must satisfy the function's warp or block participation requirements. +/// Every declaration of the same symbol reachable in a device module must +/// agree on this annotation; conflicting declarations are rejected. /// /// # When to Use /// @@ -1039,77 +1059,32 @@ pub fn tile_export(attr: TokenStream, item: TokenStream) -> TokenStream { /// # Generated LLVM IR /// /// ```llvm -/// declare float @cub_block_reduce_sum(float, ptr) #0 -/// attributes #0 = { convergent nounwind } +/// declare float @cub_block_reduce_sum(float, ptr) convergent /// ``` #[proc_macro_attribute] pub fn convergent(_attr: TokenStream, item: TokenStream) -> TokenStream { - // This is a marker attribute - just pass through the item unchanged. - // The collector will read this attribute and apply the LLVM convergent attribute. - item + // The procedural attribute disappears during expansion. A doc alias is + // retained in metadata even for private foreign items, unlike doc comments. + let marker = reserved_oxide_symbols::DEVICE_EXTERN_CONVERGENT_MARKER; + let item = proc_macro2::TokenStream::from(item); + quote::quote!(#[doc(alias = #marker)] #item).into() } -/// Marks a device function as pure (no side effects). -/// -/// Pure functions only depend on their inputs and have no side effects. -/// This enables aggressive optimizations like CSE and dead code elimination. -/// -/// # When to Use -/// -/// - Math functions that don't access memory -/// - Functions that compute results purely from input arguments +/// Compatibility annotation for a pure device function. /// -/// # Example -/// -/// ```ignore -/// #[device] -/// extern "C" { -/// #[pure] -/// fn fast_rsqrt(x: f32) -> f32; -/// } -/// ``` -/// -/// # Generated LLVM IR -/// -/// ```llvm -/// declare float @fast_rsqrt(float) #0 -/// attributes #0 = { nounwind readnone } -/// ``` +/// This annotation is currently inactive: it records no compiler effect and +/// emits no LLVM memory-effect attributes. #[proc_macro_attribute] pub fn pure(_attr: TokenStream, item: TokenStream) -> TokenStream { - // Marker attribute - collector will read and apply LLVM readnone attribute item } -/// Marks a device function as read-only. +/// Compatibility annotation for a read-only device function. /// -/// Read-only functions may read memory but never write to it. -/// This enables optimizations like load hoisting and caching. -/// -/// # When to Use -/// -/// - Lookup table functions -/// - Functions that only read from input arrays -/// -/// # Example -/// -/// ```ignore -/// #[device] -/// extern "C" { -/// #[readonly] -/// fn lookup_table(table: *const f32, idx: i32) -> f32; -/// } -/// ``` -/// -/// # Generated LLVM IR -/// -/// ```llvm -/// declare float @lookup_table(ptr, i32) #0 -/// attributes #0 = { nounwind readonly } -/// ``` +/// This annotation is currently inactive: it records no compiler effect and +/// emits no LLVM memory-effect attributes. #[proc_macro_attribute] pub fn readonly(_attr: TokenStream, item: TokenStream) -> TokenStream { - // Marker attribute - collector will read and apply LLVM readonly attribute item } @@ -1275,3 +1250,16 @@ pub fn cuda_launch_async(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input as CudaLaunchAsyncInput); expand_cuda_launch_async(input).into() } + +/// Embed a native library registration from a tracked environment variable. +/// +/// Re-exported as `cuda_device::library::registration!`. Tracks the provider +/// executable and, during CUBIN materialization, its declared native inputs. +/// Ordinary host and documentation builds do not query CUDA toolkit inputs. +#[proc_macro] +pub fn library_registration(input: TokenStream) -> TokenStream { + let variable = parse_macro_input!(input as syn::LitStr); + library::expand(&variable) + .unwrap_or_else(|error| syn::Error::new(variable.span(), error).to_compile_error()) + .into() +} diff --git a/cuda-oxide/crates/cuda-macros/src/library.rs b/cuda-oxide/crates/cuda-macros/src/library.rs new file mode 100644 index 0000000000..3de32d7759 --- /dev/null +++ b/cuda-oxide/crates/cuda-macros/src/library.rs @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Register native inputs during expansion, before rustc writes dep-info. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +const MAX_JSON: usize = 1024 * 1024; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Registration { + version: u32, + provider: String, + executable: PathBuf, + operation: String, + algorithm: String, +} + +#[derive(Serialize)] +struct InputRequest { + version: u32, + cuda_root: PathBuf, + target: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct InputResponse { + version: u32, + paths: Vec, + environment: Vec, +} + +fn environment(name: &str) -> Result, String> { + match proc_macro::tracked::env_var(name) { + Ok(value) => Ok(Some(value)), + Err(std::env::VarError::NotPresent) => Ok(None), + Err(error) => Err(format!("cannot read {name}: {error}")), + } +} + +fn registration(value: &str) -> Result { + if value.len() > MAX_JSON { + return Err("native library registration exceeds 1 MiB".into()); + } + let registration: Registration = serde_json::from_str(value) + .map_err(|error| format!("invalid native library registration: {error}"))?; + if registration.version != 1 + || registration.provider.is_empty() + || registration.operation.is_empty() + || registration.algorithm.is_empty() + || !registration.executable.is_absolute() + { + return Err("native library registration requires version 1, nonempty names and an absolute executable path".into()); + } + Ok(registration) +} + +fn response(bytes: &[u8]) -> Result { + if bytes.len() > MAX_JSON { + return Err("native library input response exceeds 1 MiB".into()); + } + let response: InputResponse = serde_json::from_slice(bytes) + .map_err(|error| format!("invalid native library input response: {error}"))?; + if response.version != 1 { + return Err("native library input response requires version 1".into()); + } + if response.paths.iter().any(|path| !path.is_absolute()) { + return Err("native library input paths must be absolute".into()); + } + if response + .environment + .iter() + .any(|name| name.is_empty() || name.as_bytes().iter().any(|byte| matches!(byte, b'=' | 0))) + { + return Err( + "native library input response has an invalid environment variable name".into(), + ); + } + Ok(response) +} + +fn query(executable: &Path, request: &InputRequest) -> Result { + let bytes = serde_json::to_vec(request).map_err(|error| error.to_string())?; + if bytes.len() > MAX_JSON { + return Err("native library input request exceeds 1 MiB".into()); + } + let mut child = Command::new(executable) + .arg("--oxide-library-inputs-v1") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .map_err(|error| { + format!( + "cannot query native library {}: {error}", + executable.display() + ) + })?; + let read = (|| { + child.stdin.take().unwrap().write_all(&bytes)?; + let mut output = Vec::new(); + child + .stdout + .take() + .unwrap() + .take((MAX_JSON + 1) as u64) + .read_to_end(&mut output)?; + Ok::<_, std::io::Error>(output) + })(); + let output = match read { + Ok(output) if output.len() <= MAX_JSON => output, + other => { + let _ = child.kill(); + let _ = child.wait(); + return Err(match other { + Err(error) => format!("native library input query failed: {error}"), + _ => "native library input response exceeds 1 MiB".into(), + }); + } + }; + let status = child.wait().map_err(|error| error.to_string())?; + if !status.success() { + return Err(format!("native library input query exited with {status}")); + } + response(&output) +} + +fn track_tree( + path: &Path, + directories: &mut BTreeSet, + track: &mut impl FnMut(&Path), +) -> Result<(), String> { + let metadata = path.metadata().map_err(|error| { + format!( + "cannot track native library input {}: {error}", + path.display() + ) + })?; + if !metadata.is_file() && !metadata.is_dir() { + return Err(format!( + "native library input is not a file or directory: {}", + path.display() + )); + } + track(path); + let canonical = path.canonicalize().map_err(|error| error.to_string())?; + if canonical != path { + track(&canonical); + } + if metadata.is_dir() && directories.insert(canonical) { + for entry in std::fs::read_dir(path).map_err(|error| error.to_string())? { + track_tree( + &entry.map_err(|error| error.to_string())?.path(), + directories, + track, + )?; + } + } + Ok(()) +} + +pub(crate) fn expand(variable: &syn::LitStr) -> Result { + crate::common::track_codegen_environment(); + let value = environment(&variable.value())?.ok_or_else(|| { + format!( + "native library registration variable {} is not set", + variable.value() + ) + })?; + let registration = registration(&value)?; + if !registration.executable.is_file() { + return Err(format!( + "native library provider is not a file: {}", + registration.executable.display() + )); + } + let mut directories = BTreeSet::new(); + let mut track = |path: &Path| proc_macro::tracked::path(path); + track_tree(®istration.executable, &mut directories, &mut track)?; + let cuda_home = environment("CUDA_HOME")?; + let cuda_path = environment("CUDA_PATH")?; + let target = environment("CUDA_OXIDE_TARGET")?; + if environment(reserved_oxide_symbols::MATERIALIZE_CUBIN_ENV)? + .is_some_and(|value| matches!(value.as_str(), "1" | "true")) + { + let cuda_root = cuda_home + .or(cuda_path) + .ok_or("native library inputs require CUDA_HOME or CUDA_PATH")?; + let cuda_root = std::path::absolute(cuda_root) + .map_err(|error| format!("cannot resolve CUDA toolkit path: {error}"))?; + let target = target.ok_or("native library inputs require CUDA_OXIDE_TARGET")?; + let target = target + .strip_prefix("sm_") + .map(|value| format!("compute_{value}")) + .unwrap_or(target); + let inputs = query( + ®istration.executable, + &InputRequest { + version: 1, + cuda_root, + target, + }, + )?; + for name in inputs.environment { + environment(&name)?; + } + for path in inputs.paths { + track_tree(&path, &mut directories, &mut track)?; + } + } + let literal = syn::LitStr::new(&value, variable.span()); + Ok(quote::quote!(#literal)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn input_response_rejects_invalid_contracts() { + for value in [ + r#"{"version":2,"paths":[],"environment":[]}"#, + r#"{"version":1,"paths":["relative"],"environment":[]}"#, + r#"{"version":1,"paths":[],"environment":["A=B"]}"#, + r#"{"version":1,"paths":[],"environment":[],"extra":1}"#, + r#"{"version":1,"paths":[],"environment":[]} trailing"#, + ] { + assert!(response(value.as_bytes()).is_err(), "{value}"); + } + assert!(response(&vec![b' '; MAX_JSON + 1]).is_err()); + assert!(response(br#"{"version":1,"paths":[],"environment":[]}"#).is_ok()); + } +} diff --git a/cuda-oxide/crates/cuda-macros/src/tests/cuda_module.rs b/cuda-oxide/crates/cuda-macros/src/tests/cuda_module.rs index be7a69ac79..cbc93b4d40 100644 --- a/cuda-oxide/crates/cuda-macros/src/tests/cuda_module.rs +++ b/cuda-oxide/crates/cuda-macros/src/tests/cuda_module.rs @@ -381,7 +381,7 @@ fn cfg_gated_generic_uses_the_same_gate_for_marker_and_loader() { "loader selection must inherit the generic kernel's cfg:\n{expanded}" ); assert!( - expanded.contains(&format!("if__cuda_oxide_has_enabled_generic_kernel{{let_=name;::cuda_host::load_all_ptx_bundles_merged(ctx)?}}else{{{ANCHORED_EMBEDDED_LOADER}}}")), + expanded.contains(&format!("if__cuda_oxide_has_enabled_generic_kernel{{let_=name;#[cfg(any(target_os=\"linux\",target_os=\"android\"))]{{static__CUDA_OXIDE_GENERIC_IMAGE:u8=0;::cuda_host::load_generic_embedded_module_from_anchor(ctx,&__CUDA_OXIDE_GENERIC_IMAGE,)?}}#[cfg(not(any(target_os=\"linux\",target_os=\"android\")))]{{::cuda_host::load_generic_embedded_module(ctx)?}}}}else{{{ANCHORED_EMBEDDED_LOADER}}}")), "loader must fall back to the embedded artifact when no generic kernel is enabled:\n{expanded}" ); } @@ -1493,3 +1493,18 @@ fn non_generic_kernel_lookups_never_consult_the_divergence_diagnosis() { "non-generic lookups must keep the ordinary error path: {expanded}" ); } + +#[test] +fn inferred_shared_contract_rejects_a_separate_alignment_promise() { + let module: ItemMod = parse_quote! { + mod kernels { + #[kernel] + #[launch_contract(domain = 1, block = (32, 1, 1), dynamic_shared = inferred, + dynamic_shared_alignment = 128)] + pub fn map() {} + } + }; + let error = + expand_cuda_module(module).expect_err("inferred alignment belongs to the compiler layout"); + assert!(error.to_string().contains("inferred")); +} diff --git a/cuda-oxide/crates/cuda-macros/src/tests/kernel.rs b/cuda-oxide/crates/cuda-macros/src/tests/kernel.rs index caaf588b90..3e13600e74 100644 --- a/cuda-oxide/crates/cuda-macros/src/tests/kernel.rs +++ b/cuda-oxide/crates/cuda-macros/src/tests/kernel.rs @@ -689,6 +689,7 @@ fn generic_kernel_forwards_only_exact_top_level_configuration_markers() { ::cuda_device::thread::__launch_contract_block_config::<64, 1, 1>(); ::cuda_device::cluster::__cluster_config::<2, 1, 1>(); ::cuda_device::shared::__dynamic_shared_alignment::<128>(); + ::cuda_device::shared::__dynamic_shared_inferred(); cuda_device::thread::__launch_bounds_config::<4, 1>(); unrelated(); other::cuda_device::thread::__launch_bounds_config::<32, 1>(); @@ -702,12 +703,13 @@ fn generic_kernel_forwards_only_exact_top_level_configuration_markers() { let markers = top_level_kernel_configuration_markers(&kernel); let forwarded = quote!(#(#markers)*).to_string().replace(' ', ""); - assert_eq!(markers.len(), 5); + assert_eq!(markers.len(), 6); assert!(forwarded.contains("__launch_bounds_config::<64,2>()")); assert!(forwarded.contains("__launch_contract_config::<1,true>()")); assert!(forwarded.contains("__launch_contract_block_config::<64,1,1>()")); assert!(forwarded.contains("__cluster_config::<2,1,1>()")); assert!(forwarded.contains("__dynamic_shared_alignment::<128>()")); + assert!(forwarded.contains("__dynamic_shared_inferred()")); assert!(!forwarded.contains("unrelated")); assert!(!forwarded.contains("::<4,1>()")); assert!(!forwarded.contains("other::cuda_device")); diff --git a/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_fake_disjoint_slice.stderr b/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_fake_disjoint_slice.stderr index 530af8ce95..20ae24d496 100644 --- a/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_fake_disjoint_slice.stderr +++ b/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_fake_disjoint_slice.stderr @@ -10,6 +10,7 @@ help: the trait `for<'__cuda_oxide_disjoint> cuda_device::disjoint::__LaunchCont 9 | pub struct DisjointSlice<'a, T, IndexSpace = Index1D> { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ = help: the following other types implement trait `cuda_device::disjoint::__LaunchContractDisjointSlice`: + `cuda_device::DisjointSlice<'a, T, CollectiveTiles>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `cuda_device::DisjointSlice<'a, T, Index2D>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `cuda_device::DisjointSlice<'a, T, Index2D>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `cuda_device::DisjointSlice<'a, T, LinearTiles>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` @@ -17,8 +18,7 @@ help: the trait `for<'__cuda_oxide_disjoint> cuda_device::disjoint::__LaunchCont `cuda_device::DisjointSlice<'a, T, Runtime2DIndex>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `cuda_device::DisjointSlice<'a, T, Runtime2DIndex>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `cuda_device::DisjointSlice<'a, T, RuntimeRowMajorTiles>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` - `cuda_device::DisjointSlice<'a, T, WarpIndex>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` - `cuda_device::DisjointSlice<'a, T>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` + and $N others note: required by a bound in `LoadedModule::prepare_lookalike` --> tests/compile_fail/launch_contract_fake_disjoint_slice.rs:15:1 | diff --git a/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_misleading_index_alias.stderr b/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_misleading_index_alias.stderr index 50bc23cd6f..16d9a6a12a 100644 --- a/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_misleading_index_alias.stderr +++ b/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_misleading_index_alias.stderr @@ -6,6 +6,7 @@ error[E0277]: the trait bound `for<'__cuda_oxide_disjoint> cuda_device::Disjoint | = help: the trait `for<'__cuda_oxide_disjoint> cuda_device::disjoint::__LaunchContractDisjointSlice` is not implemented for `cuda_device::DisjointSlice<'__cuda_oxide_disjoint, u32>` = help: the following other types implement trait `cuda_device::disjoint::__LaunchContractDisjointSlice`: + `cuda_device::DisjointSlice<'_, T, CollectiveTiles>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `cuda_device::DisjointSlice<'_, T, LinearTiles>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `cuda_device::DisjointSlice<'_, T, RowMajorTiles>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `cuda_device::DisjointSlice<'_, T, Runtime2DIndex>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` @@ -13,8 +14,7 @@ error[E0277]: the trait bound `for<'__cuda_oxide_disjoint> cuda_device::Disjoint `cuda_device::DisjointSlice<'_, T, RuntimeRowMajorTiles>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `cuda_device::DisjointSlice<'_, T, WarpIndex>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `cuda_device::DisjointSlice<'_, T, cuda_device::Index2D>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` - `cuda_device::DisjointSlice<'_, T, cuda_device::Index2D>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` - `cuda_device::DisjointSlice<'_, T>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` + and $N others note: required by a bound in `LoadedModule::prepare_misleading_name` --> tests/compile_fail/launch_contract_misleading_index_alias.rs:7:1 | diff --git a/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_reordered_disjoint_alias.stderr b/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_reordered_disjoint_alias.stderr index 6509665d5d..0d17721ef0 100644 --- a/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_reordered_disjoint_alias.stderr +++ b/cuda-oxide/crates/cuda-macros/tests/compile_fail/launch_contract_reordered_disjoint_alias.stderr @@ -6,6 +6,7 @@ error[E0277]: the trait bound `for<'__cuda_oxide_disjoint> cuda_device::Disjoint | = help: the trait `for<'__cuda_oxide_disjoint> cuda_device::disjoint::__LaunchContractDisjointSlice` is not implemented for `cuda_device::DisjointSlice<'__cuda_oxide_disjoint, u64>` = help: the following other types implement trait `cuda_device::disjoint::__LaunchContractDisjointSlice`: + `cuda_device::DisjointSlice<'_, T, CollectiveTiles>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `cuda_device::DisjointSlice<'_, T, Index2D>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `cuda_device::DisjointSlice<'_, T, Index2D>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `cuda_device::DisjointSlice<'_, T, LinearTiles>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` @@ -13,8 +14,7 @@ error[E0277]: the trait bound `for<'__cuda_oxide_disjoint> cuda_device::Disjoint `cuda_device::DisjointSlice<'_, T, Runtime2DIndex>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `cuda_device::DisjointSlice<'_, T, Runtime2DIndex>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` `cuda_device::DisjointSlice<'_, T, RuntimeRowMajorTiles>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` - `cuda_device::DisjointSlice<'_, T, WarpIndex>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` - `cuda_device::DisjointSlice<'_, T>` implements `cuda_device::disjoint::__LaunchContractDisjointSlice` + and $N others note: required by a bound in `LoadedModule::prepare_reordered` --> tests/compile_fail/launch_contract_reordered_disjoint_alias.rs:11:1 | diff --git a/cuda-oxide/crates/cuda-macros/tests/device-only/Cargo.lock b/cuda-oxide/crates/cuda-macros/tests/device-only/Cargo.lock index 4a78f429b9..6b95509d84 100644 --- a/cuda-oxide/crates/cuda-macros/tests/device-only/Cargo.lock +++ b/cuda-oxide/crates/cuda-macros/tests/device-only/Cargo.lock @@ -17,6 +17,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn 2.0.119", ] diff --git a/cuda-oxide/crates/cuda-macros/tests/kernel_host_retention.rs b/cuda-oxide/crates/cuda-macros/tests/kernel_host_retention.rs new file mode 100644 index 0000000000..f89a896416 --- /dev/null +++ b/cuda-oxide/crates/cuda-macros/tests/kernel_host_retention.rs @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use cuda_macros::kernel; + +unsafe extern "C" { + // Deliberately undefined on the host. Looking up a specialization's name + // must not retain the kernel body or any of its device-only callees. + fn device_only_retention_probe(); +} + +#[kernel] +pub fn guarded() { + // SAFETY: this body is a device-only link probe and is never called. + unsafe { device_only_retention_probe() }; +} + +#[kernel] +pub fn guarded_closure i32 + Copy>(f: F) { + let _ = f(); + // SAFETY: this body is a device-only link probe and is never called. + unsafe { device_only_retention_probe() }; +} + +#[test] +fn names_do_not_require_host_kernel_bodies() { + let first = guarded_ptx_name::<1>(); + let fourth = guarded_ptx_name::<4>(); + assert!(first.starts_with("guarded_TID_")); + assert_ne!(first, fourth); + assert_eq!(first, guarded_ptx_name::<1>()); + assert_eq!( + first, + <__guarded_CudaKernel<1> as cuda_host::GenericCudaKernel>::ptx_name() + ); +} + +#[test] +fn naming_accepts_a_borrowed_closure() { + fn name i32 + Copy>(_: &F) -> &'static str { + guarded_closure_ptx_name::() + } + + let value = 7; + let borrowed = || value; + assert!(name(&borrowed).starts_with("guarded_closure_TID_")); +} diff --git a/cuda-oxide/crates/cuda-macros/tests/library_registration.rs b/cuda-oxide/crates/cuda-macros/tests/library_registration.rs new file mode 100644 index 0000000000..4388dc66cb --- /dev/null +++ b/cuda-oxide/crates/cuda-macros/tests/library_registration.rs @@ -0,0 +1,157 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Ordinary Cargo freshness across a dependency registration and its consumer. +//! The fake provider is a shell process; no CUDA toolkit or GPU is used. +#![cfg(unix)] + +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn build(root: &Path, materialize: bool, option: Option<&str>, expected: Option<&str>) -> String { + let mut command = Command::new(env!("CARGO")); + command + .args(["build", "-vv", "--offline"]) + .current_dir(root) + .env("CARGO_TARGET_DIR", root.join("target")) + .env("CUDA_HOME", root.join("toolkit")) + .env("CUDA_OXIDE_TARGET", "sm_120") + .env_remove("CARGO_ENCODED_RUSTFLAGS") + .env_remove("RUSTFLAGS"); + if let Some(option) = option { + command.env("NATIVE_INPUT_OPTION", option); + } else { + command.env_remove("NATIVE_INPUT_OPTION"); + } + if materialize { + command.env("CUDA_OXIDE_MATERIALIZE_CUBIN", "1"); + } else { + command.env_remove("CUDA_OXIDE_MATERIALIZE_CUBIN"); + } + let output = command.output().unwrap(); + let log = String::from_utf8_lossy(&output.stderr).into_owned(); + if let Some(error) = expected { + assert!(!output.status.success(), "{log}"); + assert!(log.contains(error), "{log}"); + } else { + assert!(output.status.success(), "{log}"); + } + log +} + +fn provider(root: &Path, response: &str) { + fs::write( + root.join("provider"), + format!("#!/bin/sh\ncat >/dev/null\nprintf '%s' '{response}'\n"), + ) + .unwrap(); + fs::set_permissions(root.join("provider"), fs::Permissions::from_mode(0o755)).unwrap(); +} + +#[test] +fn tracks_native_inputs_before_cargo_dep_info() { + let root = std::env::temp_dir().join(format!( + "oxide-library-inputs-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let device = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../cuda-device") + .canonicalize() + .unwrap(); + fs::create_dir_all(root.join("api/src")).unwrap(); + fs::create_dir_all(root.join("consumer/src")).unwrap(); + fs::create_dir_all(root.join("headers")).unwrap(); + fs::write(root.join("headers/value.h"), "initial").unwrap(); + fs::write( + root.join("Cargo.toml"), + "[workspace]\nmembers=['api','consumer']\ndefault-members=['consumer']\nresolver='3'\n", + ) + .unwrap(); + fs::write(root.join("api/Cargo.toml"), format!( + "[package]\nname='native-input-api'\nversion='0.0.0'\nedition='2024'\n[dependencies]\ncuda-device={{path={:?}}}\n", device.to_str().unwrap() + )).unwrap(); + fs::write(root.join("consumer/Cargo.toml"), + "[package]\nname='native-input-consumer'\nversion='0.0.0'\nedition='2024'\n[dependencies]\nnative-input-api={path='../api'}\n").unwrap(); + let registration = serde_json::json!({ + "version":1,"provider":"fixture","executable":root.join("provider"), + "operation":"test","algorithm":"direct" + }) + .to_string(); + fs::write(root.join("api/build.rs"), format!( + "fn main() {{ println!(\"cargo::rerun-if-changed=build.rs\"); println!(\"cargo::rustc-env=REG={{}}\", {registration:?}); }}" + )).unwrap(); + fs::write( + root.join("api/src/lib.rs"), + "pub fn registration() -> &'static str { cuda_device::library::registration!(\"REG\") }\n", + ) + .unwrap(); + fs::write( + root.join("consumer/src/main.rs"), + "fn main() { assert!(native_input_api::registration().contains(\"fixture\")); }\n", + ) + .unwrap(); + let response = serde_json::json!({ + "version":1,"paths":[root.join("headers")],"environment":["NATIVE_INPUT_OPTION"] + }) + .to_string(); + provider(&root, &response); + build(&root, true, None, None); + let warm = build(&root, true, None, None); + assert!(warm.contains("Fresh native-input-api"), "{warm}"); + assert!(warm.contains("Fresh native-input-consumer"), "{warm}"); + + fs::write(root.join("headers/value.h"), "changed").unwrap(); + let changed = build(&root, true, None, None); + assert!(changed.contains("Compiling native-input-api"), "{changed}"); + assert!( + changed.contains("Compiling native-input-consumer"), + "{changed}" + ); + fs::write(root.join("headers/new.h"), "added").unwrap(); + let added = build(&root, true, None, None); + assert!(added.contains("Compiling native-input-api"), "{added}"); + + let option = build(&root, true, Some("changed"), None); + assert!(option.contains("Compiling native-input-api"), "{option}"); + assert!( + option.contains("Compiling native-input-consumer"), + "{option}" + ); + let warm = build(&root, true, Some("changed"), None); + assert!(warm.contains("Fresh native-input-api"), "{warm}"); + + provider(&root, r#"{"version":2,"paths":[],"environment":[]}"#); + build( + &root, + true, + Some("changed"), + Some("input response requires version 1"), + ); + provider( + &root, + r#"{"version":1,"paths":["relative"],"environment":[]}"#, + ); + build( + &root, + true, + Some("changed"), + Some("input paths must be absolute"), + ); + provider(&root, "malformed"); + build( + &root, + true, + Some("changed"), + Some("invalid native library input response"), + ); + // A normal host build expands the registration without querying CUDA or + // attempting to consume the deliberately invalid native response. + build(&root, false, None, None); + fs::remove_dir_all(root).unwrap(); +} diff --git a/cuda-oxide/crates/cuda-oxide-codegen/src/export.rs b/cuda-oxide/crates/cuda-oxide-codegen/src/export.rs index 12fe5ecda2..fc1b5521db 100644 --- a/cuda-oxide/crates/cuda-oxide-codegen/src/export.rs +++ b/cuda-oxide/crates/cuda-oxide-codegen/src/export.rs @@ -44,25 +44,19 @@ pub struct DeviceExternDecl { /// NVVM attributes for device extern declarations. /// -/// NOTE: These attributes are currently **not emitted** to the LLVM IR output. -/// When linking LTOIR via nvJitLink, the external library's LTOIR already contains -/// proper attributes (convergent, nounwind, memory, etc.) on the function DEFINITIONS. -/// nvJitLink uses the definition's attributes during LTO, making attributes on our -/// declarations redundant. -/// -/// This struct is retained for the pipeline API but values are not used in code generation. +/// Convergence is emitted on each marked declaration before libNVVM optimization. +/// Memory-effect annotations remain inactive. #[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] pub struct DeviceExternAttrs { - /// Function is convergent (all threads must execute together). - /// NOTE: Not currently emitted to LLVM IR. + /// Function has convergence requirements for its participating threads. pub is_convergent: bool, /// Function is pure (no side effects, result depends only on inputs). - /// NOTE: Not currently emitted to LLVM IR. + /// Inactive; not emitted to LLVM IR. pub is_pure: bool, /// Function is read-only (only reads memory, doesn't write). - /// NOTE: Not currently emitted to LLVM IR. + /// Inactive; not emitted to LLVM IR. pub is_readonly: bool, } diff --git a/cuda-oxide/crates/cuda-oxide-codegen/src/lib.rs b/cuda-oxide/crates/cuda-oxide-codegen/src/lib.rs index 682d9c2fc1..27b8092767 100644 --- a/cuda-oxide/crates/cuda-oxide-codegen/src/lib.rs +++ b/cuda-oxide/crates/cuda-oxide-codegen/src/lib.rs @@ -127,7 +127,7 @@ pub mod __private { #[doc(hidden)] pub use crate::pipeline::{ ModuleArtifactKind, ModulePipelineOutput, ModulePipelineRequest, OutputFiles, - PipelineTrace, compile_translated_module, + PipelineTrace, PreparedModuleStage, compile_translated_module, prepare_translated_module, }; #[doc(hidden)] pub use crate::verify::verify_operation; diff --git a/cuda-oxide/crates/cuda-oxide-codegen/src/lower.rs b/cuda-oxide/crates/cuda-oxide-codegen/src/lower.rs index 05011e5e58..d08f68f8d5 100644 --- a/cuda-oxide/crates/cuda-oxide-codegen/src/lower.rs +++ b/cuda-oxide/crates/cuda-oxide-codegen/src/lower.rs @@ -84,7 +84,7 @@ pub fn add_device_extern_declarations( region.iter(ctx).next().expect("Module should have a block") }; - let declared_symbols: HashMap<_, _> = block + let mut declared_symbols: HashMap<_, _> = block .deref(ctx) .iter(ctx) .filter_map(|op| { @@ -140,6 +140,7 @@ pub fn add_device_extern_declarations( // Insert at the front of the module (declarations come before definitions) func_op.get_operation().insert_at_front(block, ctx); + declared_symbols.insert(decl.export_name.clone(), func_type); } Ok(()) @@ -196,3 +197,62 @@ fn device_extern_type_to_pliron( } }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::export::DeviceExternAttrs; + use llvm_export::export::{NvvmExportConfig, export_module_with_externs}; + use pliron::builtin::ops::ModuleOp; + use pliron::common_traits::Verify; + + #[test] + fn repeated_externs_keep_one_operation_and_preserve_conflict_checks() { + let mut ctx = Context::new(); + let module = ModuleOp::new(&mut ctx, "externs".try_into().unwrap()); + let declaration = DeviceExternDecl { + export_name: "external".to_string(), + param_types: vec![DeviceExternType::Integer(32)], + return_type: DeviceExternType::Void, + attrs: DeviceExternAttrs::default(), + }; + let mut externs = [declaration.clone(), declaration]; + add_device_extern_declarations(&mut ctx, module.get_operation(), &externs).unwrap(); + let declarations = { + let region = module.get_operation().deref(&ctx).get_region(0).deref(&ctx); + let block = region.iter(&ctx).next().unwrap(); + block + .deref(&ctx) + .iter(&ctx) + .filter(|&op| { + Operation::get_op::(op, &ctx).is_some_and( + |function| function.get_symbol_name(&ctx).to_string() == "external", + ) + }) + .count() + }; + assert_eq!(declarations, 1); + module.get_operation().deref(&ctx).verify(&ctx).unwrap(); + let ir = export_module_with_externs(&ctx, &module, &externs, &NvvmExportConfig::default()) + .unwrap(); + assert_eq!(ir.matches("declare void @external(i32)").count(), 1); + + // Lowering shares only the FuncOp. The complete declarations still + // reach export, where differing convergence requirements are rejected. + externs[1].attrs.is_convergent = true; + add_device_extern_declarations(&mut ctx, module.get_operation(), &externs).unwrap(); + let error = + export_module_with_externs(&ctx, &module, &externs, &NvvmExportConfig::default()) + .unwrap_err(); + assert!(error.contains("convergent is false versus true"), "{error}"); + + externs[1].param_types[0] = DeviceExternType::Float32; + let error = + add_device_extern_declarations(&mut ctx, module.get_operation(), &externs).unwrap_err(); + assert!( + error + .to_string() + .contains("conflicts with the call-site declaration") + ); + } +} diff --git a/cuda-oxide/crates/cuda-oxide-codegen/src/pipeline.rs b/cuda-oxide/crates/cuda-oxide-codegen/src/pipeline.rs index d9b9c863c3..bc6502da74 100644 --- a/cuda-oxide/crates/cuda-oxide-codegen/src/pipeline.rs +++ b/cuda-oxide/crates/cuda-oxide-codegen/src/pipeline.rs @@ -27,7 +27,7 @@ use crate::iket::{has_iket_operations, materialize as materialize_iket, strip as use crate::llvm_tools::LlvmToolchain; use crate::lower::{add_device_extern_declarations, lower_to_llvm}; use crate::options::{BackendOptions, IketInstrumentation}; -use crate::prep::{MirPreparation, prepare_mir_module}; +use crate::prep::{MirPreparation, MirPreparationStage, begin_mir_preparation}; use crate::ptx::{ PtxModule, discover_llvm_toolchain, generate_ptx_discovered, generate_ptx_with_toolchain, }; @@ -143,6 +143,7 @@ impl<'a> ValidatedBackendOptions<'a> { /// Complete request for the shared post-translation pipeline. #[doc(hidden)] +#[derive(Clone, Copy)] pub struct ModulePipelineRequest<'a> { device_externs: &'a [DeviceExternDecl], output_policy: OutputPolicy, @@ -237,6 +238,101 @@ pub fn compile_translated_module( module: Ptr, request: &ModulePipelineRequest<'_>, ) -> Result { + prepare_translated_module(ctx, module, request)?.finish() +} + +/// A borrowed module paused before optional post-mem2reg passes and unrolling. +/// +/// Full variable debug retains unpromoted locals at this same boundary. The +/// context, module, request and validated backend remain bound until consuming +/// [`finish`](Self::finish); no output artifact has been emitted at the pause. +/// Retained library hooks are still typed MIR operations at this boundary. +/// Finishing lowers them first, including when Full debug skips promotion. +#[doc(hidden)] +#[must_use = "finish the stage to emit its artifact"] +pub struct PreparedModuleStage<'ctx, 'inputs> { + ctx: &'ctx mut Context, + module: Ptr, + request: ModulePipelineRequest<'inputs>, + bound_externs: Option>, + validated: ValidatedBackendOptions<'inputs>, + preparation: MirPreparationStage, +} + +impl PreparedModuleStage<'_, '_> { + /// Access the bound MIR and its preparation analysis manager. + /// + /// Callers that mutate IR must invalidate affected cached analyses through + /// this manager before reusing them or finishing. This accessor does not + /// infer which analyses a caller's changes preserve. Do not introduce IKET + /// operations or change the debug/optional-pass preconditions already checked + /// before this pause. + pub fn ir( + &mut self, + ) -> ( + &mut Context, + Ptr, + &mut pliron::pass::AnalysisManager, + ) { + (self.ctx, self.module, &mut self.preparation.analyses) + } + + /// Bind validated native declarations once, retaining the original externs. + /// + /// The host provider owns response and ABI validation. The shared pipeline + /// checks these declarations against the actual module before optional MIR + /// preparation resumes and uses the same combined list for final export. + pub fn bind_device_externs( + mut self, + device_externs: Vec, + ) -> Result { + if self.bound_externs.is_some() { + return Err(PipelineError::Translation( + "native device externs were already bound to this module".into(), + )); + } + let mut combined = self.request.device_externs.to_vec(); + combined.extend(device_externs); + add_device_extern_declarations(self.ctx, self.module, &combined)?; + if !combined.is_empty() { + let mut invalidated = pliron::pass::PassResult::default(); + invalidated.ir_changed = pliron::irbuild::IRStatus::Changed; + self.preparation.analyses.retain_preserved(&invalidated); + } + self.bound_externs = Some(combined); + Ok(self) + } + + /// Consume this stage, complete preparation and emit the requested artifact. + pub fn finish(self) -> Result { + let Self { + ctx, + module, + request, + bound_externs, + validated, + preparation, + } = self; + preparation.finish(ctx, module)?; + let request = ModulePipelineRequest { + device_externs: bound_externs.as_deref().unwrap_or(request.device_externs), + ..request + }; + finish_translated_module(ctx, module, &request, validated) + } +} + +/// Prepare an imported module through promotion and result-bundle forwarding. +/// +/// Validation, IKET stripping and target pinning retain their existing order. +/// Full variable debug applies its current restrictions and returns a stage +/// without promotion. Optional post-mem2reg passes run only when it is finished. +#[doc(hidden)] +pub fn prepare_translated_module<'ctx, 'inputs>( + ctx: &'ctx mut Context, + module: Ptr, + request: &ModulePipelineRequest<'inputs>, +) -> Result, PipelineError> { let mut validated = ValidatedBackendOptions::new(request.backend)?; if request.trace.dump_mir { @@ -306,7 +402,7 @@ pub fn compile_translated_module( .emit("\n=== Skipping mem2reg (full debug keeps locals in memory) ==="); } } - prepare_mir_module( + let preparation = begin_mir_preparation( ctx, module, MirPreparation { @@ -315,10 +411,27 @@ pub fn compile_translated_module( mir_pass_pipeline: backend.mir_pass_pipeline.as_deref(), }, )?; + Ok(PreparedModuleStage { + ctx, + module, + request: *request, + bound_externs: None, + validated, + preparation, + }) +} + +fn finish_translated_module( + ctx: &mut Context, + module: Ptr, + request: &ModulePipelineRequest<'_>, + validated: ValidatedBackendOptions<'_>, +) -> Result { + let backend = validated.options(); if request.trace.verbose { request.trace.emit("dialect-mir preparation successful ✓"); } - if request.trace.dump_mir && promote_and_unroll { + if request.trace.dump_mir && !request.debug_kind.variables_enabled() { request .trace .emit("\n=== dialect-mir module (after preparation) ==="); @@ -1208,6 +1321,88 @@ fn remove_stale_files(paths: &[PathBuf]) -> Result<(), PipelineError> { mod tests { use super::*; + #[test] + fn paused_module_emits_only_when_finished() { + use pliron::op::Op; + for debug_kind in [DebugKind::Off, DebugKind::Full] { + let mut ctx = Context::new(); + dialect_mir::register(&mut ctx); + dialect_nvvm::register(&mut ctx); + let module = + pliron::builtin::ops::ModuleOp::new(&mut ctx, "staged".try_into().unwrap()) + .get_operation(); + let output = tempfile::tempdir().unwrap(); + let llvm_ir = output.path().join("staged.ll"); + let ptx = output.path().join("staged.ptx"); + let cubin = output.path().join("staged.cubin"); + std::fs::write(&ptx, "stale artifact").unwrap(); + let stale = [ptx.clone()]; + let backend = BackendOptions { + target_arch: Some("sm_120".into()), + ..BackendOptions::default() + }; + let original = [DeviceExternDecl { + export_name: "original_extern".into(), + param_types: vec![llvm_export::export::DeviceExternType::Integer(32)], + return_type: llvm_export::export::DeviceExternType::Void, + attrs: Default::default(), + }]; + // Only the referenced inputs must outlive the stage, not this + // temporary request assembled by an importing frontend. + let stage = { + let request = ModulePipelineRequest::for_rust_pipeline( + &original, + true, + &backend, + debug_kind, + None, + &[], + OutputFiles { + llvm_ir: &llvm_ir, + ptx: &ptx, + cubin: &cubin, + mlir: None, + stale_before_export: &stale, + }, + PipelineTrace::default(), + ); + prepare_translated_module(&mut ctx, module, &request).unwrap() + }; + let mut stage = stage + .bind_device_externs(vec![DeviceExternDecl { + export_name: "late_native".into(), + param_types: vec![llvm_export::export::DeviceExternType::pointer_to( + llvm_export::export::DeviceExternType::Integer(8), + 3, + )], + return_type: llvm_export::export::DeviceExternType::Void, + attrs: crate::export::DeviceExternAttrs { + is_convergent: true, + ..Default::default() + }, + }]) + .unwrap(); + let (ctx, bound_module, _) = stage.ir(); + verify_operation(ctx, bound_module, "paused module").unwrap(); + assert!(!llvm_ir.exists(), "preparation must not export"); + assert!( + ptx.exists(), + "preparation must not invalidate output artifacts" + ); + let result = stage.finish().unwrap(); + assert_eq!(result.artifact_kind, ModuleArtifactKind::NvvmIr); + assert_eq!(result.target, "sm_120"); + let ir = std::fs::read_to_string(&llvm_ir).unwrap(); + assert!(ir.contains("declare void @original_extern(i32)"), "{ir}"); + assert!( + ir.contains("declare void @late_native(ptr addrspace(3))"), + "{ir}" + ); + assert!(ir.contains("convergent"), "{ir}"); + assert!(!ptx.exists()); + } + } + #[test] fn primitives_render_uses_the_selected_manifest() { use pliron::op::Op; diff --git a/cuda-oxide/crates/cuda-oxide-codegen/src/prep.rs b/cuda-oxide/crates/cuda-oxide-codegen/src/prep.rs index 5fa8e1d10d..6e8bbeb9e5 100644 --- a/cuda-oxide/crates/cuda-oxide-codegen/src/prep.rs +++ b/cuda-oxide/crates/cuda-oxide-codegen/src/prep.rs @@ -65,16 +65,28 @@ pub struct MirPreparation<'a> { pub mir_pass_pipeline: Option<&'a str>, } -/// Verify and prepare a dialect-mir module before LLVM lowering. +/// Preparation state at the boundary before optional post-mem2reg passes. +/// The same analysis manager continues through the remaining preparation. +/// Library hooks retain their typed operations here, including in Full debug; +/// finishing lowers them before optional passes and unrolling. +pub(crate) struct MirPreparationStage { + selected_passes: Option, + pub(crate) analyses: pliron::pass::AnalysisManager, + verbose: bool, + has_high_level_cute_operations: bool, +} + +/// Verify and prepare dialect MIR through post-mem2reg result forwarding. +/// Full variable debug retains unpromoted locals at this same pause. /// /// The one shared post-translation orchestrator calls this helper for both the /// rustc and standalone frontends. #[doc(hidden)] -pub fn prepare_mir_module( +pub(crate) fn begin_mir_preparation( ctx: &mut Context, module: Ptr, preparation: MirPreparation<'_>, -) -> Result<(), PipelineError> { +) -> Result { verify_operation(ctx, module, "module")?; // Capture this before any optional preparation pass mutates the module. // The extra promotion pass belongs only to the high-level tensor slice; @@ -95,7 +107,12 @@ pub fn prepare_mir_module( .to_string(), )); } - return Ok(()); + return Ok(MirPreparationStage { + selected_passes: None, + analyses: pliron::pass::AnalysisManager::default(), + verbose: preparation.verbose, + has_high_level_cute_operations, + }); } // Validate every requested pass before any transformation runs. This keeps @@ -173,67 +190,97 @@ pub fn prepare_mir_module( "module post-mem2reg-compiler-result-forwarding", )?; - // Formation passes that need promoted SSA values but must still see the - // original loop CFG run here. In particular, a reduction formation pass - // cannot safely infer a source loop once generic unrolling has cloned it. - run_optional_mir_passes( - ctx, - module, - &selected_passes, - MirPassStage::PostMem2Reg, - &mut analyses, - )?; + Ok(MirPreparationStage { + selected_passes: Some(selected_passes), + analyses, + verbose: preparation.verbose, + has_high_level_cute_operations, + }) +} - // An immutable aggregate pointer argument in an always-inline helper can - // still retain dynamic field/array pointer chains after mem2reg. Recover - // bounded read-only accesses in typed MIR before LLVM lowering. - mir_transforms::scalarize_borrowed_aggregate_reads:: - canonicalize_bounded_borrowed_pointer_arguments(module, ctx, preparation.verbose); - verify_operation( - ctx, - module, - "module post-borrowed-pointer-read-canonicalization", - )?; +impl MirPreparationStage { + /// Resume the remaining preparation on the module owned by the pipeline stage. + pub(crate) fn finish( + self, + ctx: &mut Context, + module: Ptr, + ) -> Result<(), PipelineError> { + let mut analyses = self.analyses; + // Native calls retain their source identity through the proof boundary. + // Binding precedes optional post-mem2reg transforms and unrolling; + // Full variable debug uses the same lowering over unpromoted locals. + mir_transforms::library::lower_library_operations(module, ctx, &mut analyses).map_err( + |error| PipelineError::Verification { + name: "library operation lowering".to_string(), + message: error.disp(ctx).to_string(), + operation: None, + }, + )?; + verify_operation(ctx, module, "module post-library-operation-lowering")?; + let Some(selected_passes) = self.selected_passes else { + return Ok(()); + }; + // Formation passes that need promoted SSA values but must still see the + // original loop CFG run here. In particular, a reduction formation pass + // cannot safely infer a source loop once generic unrolling has cloned it. + run_optional_mir_passes( + ctx, + module, + &selected_passes, + MirPassStage::PostMem2Reg, + &mut analyses, + )?; + + // An immutable aggregate pointer argument in an always-inline helper can + // still retain dynamic field/array pointer chains after mem2reg. Recover + // bounded read-only accesses in typed MIR before LLVM lowering. + mir_transforms::scalarize_borrowed_aggregate_reads:: + canonicalize_bounded_borrowed_pointer_arguments(module, ctx, self.verbose); + verify_operation( + ctx, + module, + "module post-borrowed-pointer-read-canonicalization", + )?; + if self.has_high_level_cute_operations { + // Borrowing a local CuTe value (`&mut tensor`) can leave a `mir.cast` + // on its stack-slot pointer. The cast changes only the pointer kind, + // but prevents mem2reg from removing the slot. CuTe ghost values must + // stay in SSA, so remove these casts when every use is a load or store + // address. The next mem2reg pass can then remove the CuTe slot. + fold_ghost_slot_kind_only_casts(ctx, module); + verify_operation(ctx, module, "module post-ghost-slot-kind-cast-folding")?; + + // The first mem2reg pass can remove a temporary pointer slot that was + // the only reason another tensor slot looked as though its address + // escaped. That newly exposed inner slot is promotable only after the + // pointer cleanup above, so give this tensor pipeline one more pass. + pliron::opts::mem2reg::mem2reg(module, ctx, &mut analyses).map_err(|error| { + PipelineError::Verification { + name: "second mem2reg".to_string(), + message: error.disp(ctx).to_string(), + operation: None, + } + })?; + verify_operation(ctx, module, "module post-second-mem2reg")?; + } - if has_high_level_cute_operations { - // Borrowing a local CuTe value (`&mut tensor`) can leave a `mir.cast` - // on its stack-slot pointer. The cast changes only the pointer kind, - // but prevents mem2reg from removing the slot. CuTe ghost values must - // stay in SSA, so remove these casts when every use is a load or store - // address. The next mem2reg pass can then remove the CuTe slot. - fold_ghost_slot_kind_only_casts(ctx, module); - verify_operation(ctx, module, "module post-ghost-slot-kind-cast-folding")?; - - // The first mem2reg pass can remove a temporary pointer slot that was - // the only reason another tensor slot looked as though its address - // escaped. That newly exposed inner slot is promotable only after the - // pointer cleanup above, so give this tensor pipeline one more pass. - pliron::opts::mem2reg::mem2reg(module, ctx, &mut analyses).map_err(|error| { - PipelineError::Verification { - name: "second mem2reg".to_string(), + mir_transforms::unroll::unroll_annotated_loops(module, ctx, &mut analyses).map_err( + |error| PipelineError::Verification { + name: "loop-unroll".to_string(), message: error.disp(ctx).to_string(), operation: None, - } - })?; - verify_operation(ctx, module, "module post-second-mem2reg")?; + }, + )?; + verify_operation(ctx, module, "module post-unroll")?; + + run_optional_mir_passes( + ctx, + module, + &selected_passes, + MirPassStage::PostPreparation, + &mut analyses, + ) } - - mir_transforms::unroll::unroll_annotated_loops(module, ctx, &mut analyses).map_err( - |error| PipelineError::Verification { - name: "loop-unroll".to_string(), - message: error.disp(ctx).to_string(), - operation: None, - }, - )?; - verify_operation(ctx, module, "module post-unroll")?; - - run_optional_mir_passes( - ctx, - module, - &selected_passes, - MirPassStage::PostPreparation, - &mut analyses, - ) } fn has_high_level_cute_operations(ctx: &Context, root: Ptr) -> bool { @@ -503,11 +550,179 @@ mod tests { assert!(has_high_level_cute_operations(&ctx, module_operation)); } + /// A source local feeding an otherwise eligible optional atomic transform. + fn local_atomic_module(ctx: &mut Context) -> (Ptr, Ptr, Ptr) { + use dialect_mir::ops::{ + MirAllocaOp, MirFloatConstantOp, MirFuncOp, MirLoadOp, MirReturnOp, MirStoreOp, + }; + use dialect_mir::types::MirPtrType; + use dialect_nvvm::ops::atomic::{ + AtomicOrdering, AtomicRmwKind, AtomicScope, NvvmAtomicRmwOp, + }; + use pliron::basic_block::BasicBlock; + use pliron::builtin::{ + attributes::{FPSingleAttr, StringAttr, TypeAttr}, + op_interfaces::SymbolOpInterface, + types::{FP32Type, FunctionType}, + }; + dialect_mir::register(ctx); + dialect_nvvm::register(ctx); + let module = ModuleOp::new(ctx, "staged".try_into().unwrap()).get_operation(); + let float = FP32Type::get(ctx); + let ptr = MirPtrType::get_generic(ctx, float.into(), true); + let function_ty = FunctionType::get(ctx, vec![ptr.into()], vec![float.into()]); + let function = Operation::new( + ctx, + MirFuncOp::get_concrete_op_info(), + vec![], + vec![], + vec![], + 1, + ); + MirFuncOp::new(ctx, function, TypeAttr::new(function_ty.into())) + .set_symbol_name(ctx, "staged".try_into().unwrap()); + let entry = BasicBlock::new(ctx, None, vec![ptr.into()]); + entry.insert_at_back(function.deref(ctx).get_region(0), ctx); + let alloca = Operation::new( + ctx, + MirAllocaOp::get_concrete_op_info(), + vec![ptr.into()], + vec![], + vec![], + 0, + ); + alloca.deref_mut(ctx).attributes.set( + "cuda_oxide_debug_local_name".try_into().unwrap(), + StringAttr::new("value".into()), + ); + alloca.insert_at_back(entry, ctx); + let constant = Operation::new( + ctx, + MirFloatConstantOp::get_concrete_op_info(), + vec![float.into()], + vec![], + vec![], + 0, + ); + MirFloatConstantOp::new(constant).set_attr_float_value(ctx, FPSingleAttr::from(1.0)); + constant.insert_at_back(entry, ctx); + let slot = alloca.deref(ctx).get_result(0); + let value = constant.deref(ctx).get_result(0); + Operation::new( + ctx, + MirStoreOp::get_concrete_op_info(), + vec![], + vec![slot, value], + vec![], + 0, + ) + .insert_at_back(entry, ctx); + let load = Operation::new( + ctx, + MirLoadOp::get_concrete_op_info(), + vec![float.into()], + vec![slot], + vec![], + 0, + ); + load.insert_at_back(entry, ctx); + let loaded = load.deref(ctx).get_result(0); + let output = entry.deref(ctx).get_argument(0); + NvvmAtomicRmwOp::build( + ctx, + output, + loaded, + float.into(), + AtomicRmwKind::FAdd, + AtomicOrdering::Relaxed, + AtomicScope::Device, + ) + .get_operation() + .insert_at_back(entry, ctx); + Operation::new( + ctx, + MirReturnOp::get_concrete_op_info(), + vec![], + vec![loaded], + vec![], + 0, + ) + .insert_at_back(entry, ctx); + crate::lower::append_to_module(ctx, module, function); + (module, function, alloca) + } + + #[test] + fn pause_promotes_locals_before_running_optional_passes() { + use pliron::linked_list::ContainsLinkedList; + let mut ctx = Context::new(); + let (module, function, alloca) = local_atomic_module(&mut ctx); + let stage = begin_mir_preparation( + &mut ctx, + module, + MirPreparation { + promote_and_unroll: true, + verbose: false, + mir_pass_pipeline: Some("warp-aggregate-constant-fp-atomics"), + }, + ) + .unwrap(); + assert!( + alloca.try_deref(&ctx).is_err(), + "local must already be promoted at the pause" + ); + let count_matches = |ctx: &Context| { + function + .deref(ctx) + .get_region(0) + .deref(ctx) + .iter(ctx) + .flat_map(|block| block.deref(ctx).iter(ctx).collect::>()) + .filter(|&op| { + Operation::get_op::(op, ctx).is_some() + }) + .count() + }; + assert_eq!( + count_matches(&ctx), + 0, + "optional pass must not run before the pause" + ); + stage.finish(&mut ctx, module).unwrap(); + assert_eq!( + count_matches(&ctx), + 1, + "consuming finish must run the selected optional pass" + ); + } + + #[test] + fn full_debug_stage_preserves_local_slots_through_finish() { + let mut ctx = Context::new(); + let (module, _, alloca) = local_atomic_module(&mut ctx); + let original = module.deref(&ctx).disp(&ctx).to_string(); + let stage = begin_mir_preparation( + &mut ctx, + module, + MirPreparation { + promote_and_unroll: false, + verbose: false, + mir_pass_pipeline: None, + }, + ) + .unwrap(); + assert!(alloca.try_deref(&ctx).is_ok()); + assert_eq!(module.deref(&ctx).disp(&ctx).to_string(), original); + stage.finish(&mut ctx, module).unwrap(); + assert!(alloca.try_deref(&ctx).is_ok()); + assert_eq!(module.deref(&ctx).disp(&ctx).to_string(), original); + } + #[test] fn debug_mode_rejects_requested_mir_passes() { let mut ctx = Context::new(); let module = ModuleOp::new(&mut ctx, "test".try_into().unwrap()); - let error = prepare_mir_module( + let error = begin_mir_preparation( &mut ctx, module.get_operation(), MirPreparation { @@ -516,7 +731,8 @@ mod tests { mir_pass_pipeline: Some("future-pass"), }, ) - .unwrap_err(); + .err() + .expect("invalid pipeline must fail before a stage is returned"); assert!(matches!(error, PipelineError::InvalidMirPassPipeline(_))); } @@ -524,7 +740,7 @@ mod tests { fn invalid_staged_pipeline_is_rejected_before_preparation() { let mut ctx = Context::new(); let module = ModuleOp::new(&mut ctx, "test".try_into().unwrap()); - let error = prepare_mir_module( + let error = begin_mir_preparation( &mut ctx, module.get_operation(), MirPreparation { @@ -533,7 +749,8 @@ mod tests { mir_pass_pipeline: Some("missing-pass"), }, ) - .unwrap_err(); + .err() + .expect("invalid pipeline must fail before a stage is returned"); assert!(matches!(error, PipelineError::InvalidMirPassPipeline(_))); } @@ -656,7 +873,7 @@ mod tests { ) .insert_at_back(entry, &ctx); - prepare_mir_module( + begin_mir_preparation( &mut ctx, module.get_operation(), MirPreparation { @@ -665,6 +882,8 @@ mod tests { mir_pass_pipeline: None, }, ) + .unwrap() + .finish(&mut ctx, module.get_operation()) .unwrap(); assert!( @@ -709,7 +928,7 @@ mod tests { .get_operation() .insert_at_back(block, &ctx); - let error = prepare_mir_module( + let error = begin_mir_preparation( &mut ctx, module_operation, MirPreparation { @@ -718,7 +937,8 @@ mod tests { mir_pass_pipeline: None, }, ) - .unwrap_err(); + .err() + .expect("high-level CuTe must fail before a Full debug stage is returned"); let PipelineError::Lowering(message) = error else { panic!("expected a full-debug lowering rejection"); }; diff --git a/cuda-oxide/crates/dialect-mir/src/lib.rs b/cuda-oxide/crates/dialect-mir/src/lib.rs index 1cedf1a5f0..9b33447834 100644 --- a/cuda-oxide/crates/dialect-mir/src/lib.rs +++ b/cuda-oxide/crates/dialect-mir/src/lib.rs @@ -7,6 +7,7 @@ pub mod attributes; pub mod const_fold; +pub mod library; pub mod ops; pub mod rust_intrinsics; pub mod side_effects; @@ -26,4 +27,5 @@ pub fn register(ctx: &mut Context) { ops::register(ctx); types::register(ctx); attributes::register(ctx); + library::register(ctx); } diff --git a/cuda-oxide/crates/dialect-mir/src/library.rs b/cuda-oxide/crates/dialect-mir/src/library.rs new file mode 100644 index 0000000000..33b4ac52fd --- /dev/null +++ b/cuda-oxide/crates/dialect-mir/src/library.rs @@ -0,0 +1,406 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Closed library roles retained until mandatory MIR lowering. + +use pliron::{attribute::Attribute, context::Context}; +use pliron_derive::{format, pliron_attr}; + +/// Fixed-width numeric source type; pointer fields retain their own types. +#[format] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LibraryScalar { + Signed(u16), + Unsigned(u16), + Float(u16), +} + +/// Native specialization facts independent of any provider response. +#[format] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct NativeSpec { + pub scalar: LibraryScalar, + pub items: u32, +} + +/// Global-memory access selected by a genuine resource marker. +#[format] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LibraryAccess { + Read, + Write, +} + +/// Nominal witness classification captured before its type is erased. +#[format] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LibraryWitness { + ReadOnlySlice, + CollectiveCarrier(u64), +} + +/// Source descriptor policy. Zero capacity/alignment retain inferred meaning. +#[format] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct LibraryStoragePolicy { + pub capacity: u32, + pub alignment: u32, + pub auto_sync: bool, + pub exclusive: bool, +} + +/// Explicit descriptor use is distinct from managed implicit scratch. +#[format] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LibraryStorageArg { + Implicit, + Explicit(LibraryStoragePolicy), +} + +/// Evaluated source array layout, including requested reservation alignment. +#[format] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct LibraryReserveLayout { + pub scalar: LibraryScalar, + pub count: u32, + pub bytes: u32, + pub alignment: u32, +} + +/// Native contracts supported by the checked-Block prefix. +#[format] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LibraryNativeMetadata { + Raw { + spec: NativeSpec, + block_threads: u32, + }, + Callback { + spec: NativeSpec, + block_threads: u32, + }, + CheckedBlock { + spec: NativeSpec, + }, + CheckedBlockResource { + spec: NativeSpec, + access: LibraryAccess, + witness: LibraryWitness, + }, + CheckedBlockStorage { + spec: NativeSpec, + access: LibraryAccess, + witness: LibraryWitness, + storage: LibraryStorageArg, + }, +} +impl LibraryNativeMetadata { + pub fn spec(self) -> NativeSpec { + match self { + Self::Raw { spec, .. } + | Self::Callback { spec, .. } + | Self::CheckedBlock { spec } + | Self::CheckedBlockResource { spec, .. } + | Self::CheckedBlockStorage { spec, .. } => spec, + } + } +} + +/// Genuine definition class, before source calls are expanded or erased. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LibraryCallbackDefinition { + Rust, + Cuda, + Foreign, + Intrinsic, +} + +/// Concrete dispatch identity; the two named shims have separately rooted receivers. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LibraryCallbackDispatch { + Direct, + Indirect, + FunctionItemShim, + ClosureOnceShim, +} + +/// Original monomorphically reachable terminators relevant to callback admission. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum LibraryCallbackCall { + Direct { + definition: LibraryCallbackDefinition, + diagnostic: String, + }, + Indirect, + InlineAssembly, +} + +/// Source distinctions consumed by callback admission, independent of Rust types. +/// Local nominal types include unused declarations; calls include only original +/// monomorphically reachable terminators. This is not a block or value graph. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LibraryCallbackFunction { + pub diagnostic: String, + pub export: Option, + pub definition: LibraryCallbackDefinition, + pub dispatch: LibraryCallbackDispatch, + pub has_body: bool, + pub cuda_local_types: Vec, + pub calls: Vec, + pub callees: Vec, +} + +/// Original selected callback roots and complete collector identities, including +/// callees that have no imported symbol. Missing records cannot imply admission. +#[derive(Clone, Debug, Default)] +pub struct LibraryCallbackSources { + pub roots: std::collections::BTreeMap, + pub functions: std::collections::BTreeMap, +} + +/// Genuine source-native semantics; this role does not prove participation. +#[pliron_attr(name = "mir.library_role", format, verifier = "succ")] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LibraryRoleAttr { + /// Original position before evaluating a native marker's runtime operands. + /// + /// This has the same site/callee as its paired native operation. It carries + /// no participation or completion authority. Lowering consumes both and + /// emits any planned pre-call barrier here, before the operand producers. + NativeOperandBoundary, + /// Original native marker semantics, before tuple flattening or scratch insertion. + Native(LibraryNativeMetadata), + /// Create a descriptor with a new source identity. + StorageNew(LibraryStoragePolicy), + /// Consume a descriptor and change its policy, preserving its identity. + StorageReconfigure { + old: LibraryStoragePolicy, + new: LibraryStoragePolicy, + }, + /// Borrow a descriptor to create a typed reservation token. + StorageReserve { + policy: LibraryStoragePolicy, + layout: LibraryReserveLayout, + }, + /// Consume a reservation token; neither synchronization nor descriptor release. + StorageRelease, +} + +/// One source terminator in a concrete emitted Rust body. +/// +/// `block` is the original MIR block index, not the block's position after +/// transformations. Context-specialized helpers have distinct caller symbols. +#[pliron_attr(name = "mir.library_site", format, verifier = "succ")] +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct LibraryCallSite { + pub caller: String, + pub block: usize, +} + +/// Original source identity used by diagnostics and contextual helper naming. +#[pliron_attr(name = "mir.library_function_source", format, verifier = "succ")] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct LibraryFunctionSourceAttr { + pub export: String, + pub mangled: String, + pub diagnostic: String, + pub checked_ancestor: bool, +} + +/// Authenticated source type class for one original kernel argument. +#[pliron_attr(name = "mir.library_root_argument", format, verifier = "succ")] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LibraryRootArgumentClass { + StorageCandidate, + Scalar, + ResourceCandidate, + Kernel, +} + +/// First-selected source edge, distinct from the complete allowed-target set. +#[format] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LibraryCallKind { + Direct, + Callable, + Drop, +} + +/// Exact source edge selected in the original collector order. +#[pliron_attr(name = "mir.library_call_target", format, verifier = "succ")] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct LibraryCallTargetAttr { + pub kind: LibraryCallKind, + pub export: String, + pub marker: String, +} + +/// Source call classifications whose ordinary expansion erases their identity. +#[format] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LibrarySourceEffectKind { + BlockSync, + BlockQuery, + LaneQuery, + DynamicShared, + UnitHint, + ScalarIntrinsic, + PointerOffset, + UnsupportedIntrinsic, + UnresolvedCall, + NontrivialDrop, + InlineAssembly, + Abort, +} + +/// Genuine source event retained independently of expanded instruction names. +#[pliron_attr(name = "mir.library_source_effect", format, verifier = "succ")] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct LibrarySourceEffectAttr { + pub site: LibraryCallSite, + pub kind: LibrarySourceEffectKind, +} + +/// Only the source mutation distinctions consumed by library validation. +#[format] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LibraryWriteKind { + Bulk, + Indirect { nominal_protected: bool }, +} + +/// Original statement identity and nominal restriction; no alias verdict. +#[pliron_attr(name = "mir.library_write_source", format, verifier = "succ")] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct LibraryWriteSourceAttr { + pub site: LibraryCallSite, + pub statement: usize, + pub kind: LibraryWriteKind, +} + +/// Actual successor ordinal for an original exhaustive-enum impossible edge. +#[pliron_attr(name = "mir.library_impossible_successor", format, verifier = "succ")] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct LibraryImpossibleSuccessorAttr(pub u32); + +/// Stamp an imported terminator with its original, not physical, source block. +pub fn set_library_source_site( + ctx: &mut Context, + op: pliron::context::Ptr, + site: LibraryCallSite, +) { + op.deref_mut(ctx).attributes.set( + pliron::identifier::Identifier::try_new("library_terminator_site".to_string()).unwrap(), + site, + ); +} + +/// Original block identity for an actual imported terminator. +pub fn library_source_site( + ctx: &Context, + op: pliron::context::Ptr, +) -> Option { + op.deref(ctx) + .attributes + .get::( + &pliron::identifier::Identifier::try_new("library_terminator_site".to_string()) + .unwrap(), + ) + .cloned() +} + +/// Mark only the actual successor corresponding to a proved source impossible edge. +pub fn set_library_impossible_successor( + ctx: &mut Context, + op: pliron::context::Ptr, + index: u32, +) { + op.deref_mut(ctx).attributes.set( + pliron::identifier::Identifier::try_new("library_impossible_successor".to_string()) + .unwrap(), + LibraryImpossibleSuccessorAttr(index), + ); +} + +/// The original impossible edge, if one exists on this actual terminator. +pub fn library_impossible_successor( + ctx: &Context, + op: pliron::context::Ptr, +) -> Option { + op.deref(ctx) + .attributes + .get::( + &pliron::identifier::Identifier::try_new("library_impossible_successor".to_string()) + .unwrap(), + ) + .map(|attr| attr.0) +} + +/// Compiler-owned operation at one exact MIR terminator. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum LibraryOperation { + Native, + StorageNew, + StorageReconfigure, + StorageReserve, + StorageRelease, + Redirect, +} + +/// One monomorphized call whose Rust tuple and provider ABI already agree. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LibraryCallPlan { + pub operation: LibraryOperation, + pub storage_witness: bool, + pub scratch_offset: u32, + pub planned_root: Option, + pub barrier_before: bool, + pub barrier_after: bool, + /// Exact external symbol emitted by the native provider. + pub symbol: String, + /// Expected marker identity at this exact MIR terminator. + pub marker: String, + /// Split uniform/payload operands of the checked full-block marker. + pub checked_block: bool, + /// An additional typed memory witness, consumed only by compiler checks. + pub resource_witness: bool, + /// Append the borrowed callback environment after injected scratch. + pub callback: bool, + /// Size of the assigned shared scratch view in bytes. + pub scratch_bytes: u32, + /// Required alignment of the shared scratch base. + pub scratch_alignment: u32, +} + +impl LibraryCallPlan { + pub fn internal(operation: LibraryOperation, marker: String) -> Self { + Self { + operation, + marker, + symbol: String::new(), + storage_witness: false, + scratch_offset: 0, + planned_root: None, + barrier_before: false, + barrier_after: false, + checked_block: false, + resource_witness: false, + callback: false, + scratch_bytes: 0, + scratch_alignment: 1, + } + } +} + +pub(crate) fn register(ctx: &mut Context) { + LibraryRoleAttr::register(ctx); + LibraryCallSite::register(ctx); + LibraryFunctionSourceAttr::register(ctx); + LibraryRootArgumentClass::register(ctx); + LibraryCallTargetAttr::register(ctx); + LibrarySourceEffectAttr::register(ctx); + LibraryWriteSourceAttr::register(ctx); + LibraryImpossibleSuccessorAttr::register(ctx); +} diff --git a/cuda-oxide/crates/dialect-mir/src/ops/call.rs b/cuda-oxide/crates/dialect-mir/src/ops/call.rs index 8fde9fe530..f866ee3331 100644 --- a/cuda-oxide/crates/dialect-mir/src/ops/call.rs +++ b/cuda-oxide/crates/dialect-mir/src/ops/call.rs @@ -9,6 +9,7 @@ use crate::{ attributes::MirPointerKindAuthorityAttr, + library::{LibraryCallSite, LibraryCallTargetAttr, LibrarySourceEffectAttr}, rust_intrinsics, types::{MirTupleType, pointer_kinds_in_type}, }; @@ -47,6 +48,7 @@ use pliron_derive::pliron_op; /// | Name | Type | Description | /// |--------------------------|-------------------------------|-------------| /// | `callee` | StringAttr | Name of the called function. | +/// | `library_source_site` | LibraryCallSite | Original Rust caller and terminator for contextual call remapping. | /// | `external_callee_type` | TypeAttr | Exact Rust-level signature of a foreign item with no `mir.func`. | /// | `call_pointer_kind_authority` | MirPointerKindAuthorityAttr | Must be `AbiBoundary` when `external_callee_type` is present. | /// ``` @@ -73,6 +75,9 @@ use pliron_derive::pliron_op; format, attributes = ( callee: StringAttr, + library_source_site: LibraryCallSite, + library_call_target: LibraryCallTargetAttr, + library_expanded_source: LibrarySourceEffectAttr, external_callee_type: TypeAttr, call_pointer_kind_authority: MirPointerKindAuthorityAttr ) diff --git a/cuda-oxide/crates/dialect-mir/src/ops/function.rs b/cuda-oxide/crates/dialect-mir/src/ops/function.rs index cb1deda4c9..51d1c4c4ff 100644 --- a/cuda-oxide/crates/dialect-mir/src/ops/function.rs +++ b/cuda-oxide/crates/dialect-mir/src/ops/function.rs @@ -14,7 +14,7 @@ use pliron::{ attribute::attr_cast, builtin::{ attr_interfaces::TypedAttrInterface, - attributes::{StringAttr, TypeAttr}, + attributes::{BoolAttr, StringAttr, TypeAttr}, op_interfaces::{ ATTR_KEY_SYM_NAME, IsolatedFromAboveInterface, NOpdsInterface, NRegionsInterface, NResultsInterface, OneRegionInterface, SymbolOpInterface, @@ -44,10 +44,12 @@ use pliron_derive::pliron_op; use crate::{ attributes::{InlineIntentAttr, ReferenceParamValidityAttr}, + library::{LibraryFunctionSourceAttr, LibraryRootArgumentClass}, types::{MirPtrType, MirSliceType}, }; const REFERENCE_PARAM_VALIDITY_ATTR_PREFIX: &str = "reference_param_validity_"; +const LIBRARY_UNCHECKED_INDEXING_ATTR_KEY: &str = "library_unchecked_indexing"; /// Operation attribute key carrying a `mir.func`'s [`InlineIntentAttr`]. pub const INLINE_INTENT_ATTR_KEY: &str = "inline_intent"; @@ -75,6 +77,7 @@ fn inline_intent_key() -> Identifier { /// | `mir_func_type`| TypeAttr | Function type (mir.func_type) | /// | `reference_param_validity_N` | ReferenceParamValidityAttr | Proven reference facts for source argument `N` on a kernel entry | /// | `inline_intent` | InlineIntentAttr | The source function's `#[inline]` request; never on a kernel entry | +/// | `library_unchecked_indexing` | BoolAttr | The effective source bounds-check elision option | /// ``` /// /// # Verification @@ -122,6 +125,77 @@ impl MirFuncOp { TypedHandle::from_handle(ty, ctx).unwrap() } + /// Attach the authenticated original function identity before preparation. + pub fn set_library_function_source( + &self, + ctx: &mut Context, + source: LibraryFunctionSourceAttr, + ) { + self.get_operation().deref_mut(ctx).attributes.set( + Identifier::try_new("library_function_source".to_string()).unwrap(), + source, + ); + } + + /// Original identity, independent from the legalised or contextual symbol. + pub fn library_function_source(&self, ctx: &Context) -> Option { + self.get_operation() + .deref(ctx) + .attributes + .get::( + &Identifier::try_new("library_function_source".to_string()).unwrap(), + ) + .cloned() + } + + /// Retain the effective source option without deciding library admission. + pub fn set_library_unchecked_indexing(&self, ctx: &mut Context, enabled: bool) { + self.get_operation().deref_mut(ctx).attributes.set( + Identifier::try_new(LIBRARY_UNCHECKED_INDEXING_ATTR_KEY.to_string()).unwrap(), + BoolAttr::new(enabled), + ); + } + + /// Whether this function's bounds checks may be elided; absent means false. + pub fn library_unchecked_indexing(&self, ctx: &Context) -> bool { + self.get_operation() + .deref(ctx) + .attributes + .get::( + &Identifier::try_new(LIBRARY_UNCHECKED_INDEXING_ATTR_KEY.to_string()).unwrap(), + ) + .map(|attr| bool::from(attr.clone())) + .unwrap_or(false) + } + + /// Retain the original, zero-based source kernel argument classification. + pub fn set_library_root_argument( + &self, + ctx: &mut Context, + index: usize, + class: LibraryRootArgumentClass, + ) { + self.get_operation().deref_mut(ctx).attributes.set( + Identifier::try_new(format!("library_root_argument_{index}")).unwrap(), + class, + ); + } + + /// Source classification before LLVM argument flattening. + pub fn library_root_argument( + &self, + ctx: &Context, + index: usize, + ) -> Option { + self.get_operation() + .deref(ctx) + .attributes + .get::( + &Identifier::try_new(format!("library_root_argument_{index}")).unwrap(), + ) + .copied() + } + /// Record rustc-proven validity for one source-level kernel argument. /// /// Presence proves `nonnull`; the payload carries the pointee alignment diff --git a/cuda-oxide/crates/dialect-mir/src/ops/library.rs b/cuda-oxide/crates/dialect-mir/src/ops/library.rs new file mode 100644 index 0000000000..a45c9cc580 --- /dev/null +++ b/cuda-oxide/crates/dialect-mir/src/ops/library.rs @@ -0,0 +1,514 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Typed native contracts and source effects retained through MIR preparation. + +use crate::library::{ + LibraryAccess, LibraryCallSite, LibraryNativeMetadata, LibraryRoleAttr, LibraryScalar, + LibraryStorageArg, LibraryStoragePolicy, LibraryWitness, +}; +use crate::types::{ + MirDisjointSliceType, MirFP16Type, MirPointerKind, MirPtrType, MirSliceType, MirStructType, + MirTupleType, +}; +use pliron::{ + builtin::{ + attributes::{StringAttr, TypeAttr}, + type_interfaces::FunctionTypeInterface, + types::{FP32Type, FP64Type, FunctionType, IntegerType, Signedness}, + }, + common_traits::Verify, + context::{Context, Ptr}, + op::Op, + operation::Operation, + result::Error, + r#type::{TypeHandle, Typed, TypedHandle}, + value::Value, + verify_err, +}; +use pliron_derive::pliron_op; + +/// A genuine source-native hook with original runtime operands and declaration. +/// +/// Tuple flattening, scratch insertion and external ABI normalization happen only +/// after the prepared-MIR proof. This operation remains conservatively effectful. +#[pliron_op(name = "mir.library", format, attributes = ( + role: LibraryRoleAttr, site: LibraryCallSite, library_callee: StringAttr, + library_signature: TypeAttr))] +pub struct MirLibraryOp; +impl MirLibraryOp { + pub fn new(op: Ptr) -> Self { + Self { op } + } + + /// Retain the position before evaluating a native marker's operands. + pub fn build( + ctx: &mut Context, + role: LibraryRoleAttr, + site: LibraryCallSite, + callee: String, + ) -> Ptr { + assert_eq!(role, LibraryRoleAttr::NativeOperandBoundary); + let op = Operation::new(ctx, Self::get_concrete_op_info(), vec![], vec![], vec![], 0); + let library = Self::new(op); + library.set_attr_role(ctx, role); + library.set_attr_site(ctx, site); + library.set_attr_library_callee(ctx, StringAttr::new(callee)); + op + } + + pub fn build_call( + ctx: &mut Context, + role: LibraryRoleAttr, + site: LibraryCallSite, + callee: String, + signature: TypedHandle, + operands: Vec, + ) -> Ptr { + let results = signature.deref(ctx).res_types(); + let op = Operation::new( + ctx, + Self::get_concrete_op_info(), + results, + operands, + vec![], + 0, + ); + let library = Self::new(op); + library.set_attr_role(ctx, role); + library.set_attr_site(ctx, site); + library.set_attr_library_callee(ctx, StringAttr::new(callee)); + library.set_attr_library_signature(ctx, TypeAttr::new(signature.into())); + op + } +} +impl Verify for MirLibraryOp { + fn verify(&self, ctx: &Context) -> Result<(), Error> { + let op = &*self.get_operation().deref(ctx); + let Some(role) = self.get_attr_role(ctx) else { + return verify_err!(op.loc(), "mir.library requires a role"); + }; + let Some(site) = self.get_attr_site(ctx) else { + return verify_err!(op.loc(), "mir.library requires a source call site"); + }; + if site.caller.is_empty() { + return verify_err!(op.loc(), "mir.library requires a nonempty caller identity"); + } + let Some(callee) = self.get_attr_library_callee(ctx) else { + return verify_err!(op.loc(), "mir.library requires a callee identity"); + }; + if String::from((*callee).clone()).is_empty() { + return verify_err!(op.loc(), "mir.library requires a nonempty callee identity"); + } + if op.num_regions() != 0 || op.get_num_successors() != 0 { + return verify_err!(op.loc(), "mir.library has no regions or successors"); + } + if *role == LibraryRoleAttr::NativeOperandBoundary { + if op.get_num_operands() != 0 + || op.get_num_results() != 0 + || self.get_attr_library_signature(ctx).is_some() + { + return verify_err!( + op.loc(), + "native operand boundary has no operands, results or signature" + ); + } + return Ok(()); + } + verify_call(ctx, self, *role) + } +} + +fn valid_scalar(scalar: LibraryScalar) -> bool { + match scalar { + LibraryScalar::Signed(bits) | LibraryScalar::Unsigned(bits) => { + matches!(bits, 8 | 16 | 32 | 64) + } + LibraryScalar::Float(bits) => matches!(bits, 16 | 32 | 64), + } +} + +fn scalar_type(ctx: &Context, ty: TypeHandle) -> Option { + let ty = ty.deref(ctx); + let scalar = if let Some(integer) = ty.downcast_ref::() { + let width = u16::try_from(integer.width()).ok()?; + match integer.signedness() { + Signedness::Signed => LibraryScalar::Signed(width), + Signedness::Unsigned => LibraryScalar::Unsigned(width), + Signedness::Signless => return None, + } + } else { + let bits = if ty.is::() { + 16 + } else if ty.is::() { + 32 + } else if ty.is::() { + 64 + } else { + return None; + }; + LibraryScalar::Float(bits) + }; + valid_scalar(scalar).then_some(scalar) +} + +fn pointer_scalar(ctx: &Context, ty: TypeHandle) -> Option { + let ty = ty.deref(ctx); + let ptr = ty.downcast_ref::()?; + if !matches!( + (ptr.kind, ptr.is_mutable), + (MirPointerKind::RawConst, false) | (MirPointerKind::RawMut, true) + ) { + return None; + } + scalar_type(ctx, ptr.pointee) +} + +fn tuple_fields(ctx: &Context, ty: TypeHandle) -> Option> { + ty.deref(ctx) + .downcast_ref::() + .map(|tuple| tuple.types.clone()) +} + +/// Match the retained value without prematurely normalizing an ABI pointer. +/// Only address-space differences are permitted; tuple layout remains exact. +fn retained_type_matches(ctx: &Context, actual: TypeHandle, declared: TypeHandle) -> bool { + if actual == declared { + return true; + } + let actual = actual.deref(ctx); + let declared = declared.deref(ctx); + if let (Some(a), Some(d)) = ( + actual.downcast_ref::(), + declared.downcast_ref::(), + ) { + return a.pointee == d.pointee && a.kind == d.kind && a.is_mutable == d.is_mutable; + } + if let (Some(a), Some(d)) = ( + actual.downcast_ref::(), + declared.downcast_ref::(), + ) { + return a.types.len() == d.types.len() + && a.mem_to_decl == d.mem_to_decl + && a.field_offsets == d.field_offsets + && a.total_size == d.total_size + && a.abi_align == d.abi_align + && a.types + .iter() + .zip(&d.types) + .all(|(&a, &d)| retained_type_matches(ctx, a, d)); + } + false +} + +fn policy_valid(policy: LibraryStoragePolicy) -> bool { + policy.alignment == 0 || policy.alignment.is_power_of_two() +} + +fn descriptor_type(ctx: &Context, ty: TypeHandle) -> bool { + ty.deref(ctx) + .downcast_ref::() + .is_some_and(|ty| { + ty.field_types.len() == 1 + && scalar_type(ctx, ty.field_types[0]) == Some(LibraryScalar::Unsigned(8)) + }) +} + +fn descriptor_borrow(ctx: &Context, ty: TypeHandle) -> bool { + ty.deref(ctx) + .downcast_ref::() + .is_some_and(|ptr| { + ptr.kind == MirPointerKind::UniqueRef + && ptr.is_mutable + && descriptor_type(ctx, ptr.pointee) + }) +} + +fn reservation_scalar(ctx: &Context, ty: TypeHandle) -> Option { + let ty = ty.deref(ctx); + let ty = ty.downcast_ref::()?; + let [field] = ty.field_types.as_slice() else { + return None; + }; + let field = field.deref(ctx); + let pointer = field.downcast_ref::()?; + (pointer.kind == MirPointerKind::RawMut && pointer.is_mutable) + .then(|| scalar_type(ctx, pointer.pointee)) + .flatten() +} + +fn storage_arg_valid(ctx: &Context, storage: LibraryStorageArg, ty: TypeHandle) -> bool { + match storage { + LibraryStorageArg::Implicit => { + tuple_fields(ctx, ty).is_some_and(|fields| fields.is_empty()) + } + LibraryStorageArg::Explicit(policy) => policy_valid(policy) && descriptor_borrow(ctx, ty), + } +} + +fn witness_valid( + ctx: &Context, + witness: LibraryWitness, + access: LibraryAccess, + ty: TypeHandle, + scalar: LibraryScalar, +) -> bool { + let ty = ty.deref(ctx); + match witness { + LibraryWitness::ReadOnlySlice => ty.downcast_ref::().is_some_and(|slice| { + access == LibraryAccess::Read + && slice.kind == MirPointerKind::SharedRef + && !slice.is_mutable + && scalar_type(ctx, slice.element_ty) == Some(scalar) + }), + LibraryWitness::CollectiveCarrier(grain) => { + let Some(ptr) = ty.downcast_ref::() else { + return false; + }; + let reference = matches!( + (ptr.kind, ptr.is_mutable), + (MirPointerKind::SharedRef, false) | (MirPointerKind::UniqueRef, true) + ); + grain > 0 + && reference + && (access == LibraryAccess::Read || ptr.kind == MirPointerKind::UniqueRef) + && ptr + .pointee + .deref(ctx) + .downcast_ref::() + .is_some_and(|carrier| scalar_type(ctx, carrier.element_ty) == Some(scalar)) + } + } +} + +fn native_valid(ctx: &Context, native: LibraryNativeMetadata, args: &[TypeHandle]) -> bool { + use LibraryNativeMetadata as N; + let spec = native.spec(); + if !valid_scalar(spec.scalar) || spec.items == 0 { + return false; + } + let (uniform_index, payload_index) = + match native { + N::Raw { block_threads, .. } | N::Callback { block_threads, .. } => { + let callback = matches!(native, N::Callback { .. }); + let expected_args = if callback { 2 } else { 1 }; + return block_threads > 0 + && args.len() == expected_args + && (!callback + || (spec.scalar == LibraryScalar::Signed(32) + && args[1].deref(ctx).downcast_ref::().is_some_and( + |ptr| ptr.kind == MirPointerKind::SharedRef && !ptr.is_mutable, + ))) + && tuple_fields(ctx, args[0]).is_some_and(|fields| { + fields.into_iter().all(|ty| { + scalar_type(ctx, ty).is_some() || pointer_scalar(ctx, ty).is_some() + }) + }); + } + N::CheckedBlock { .. } => { + if args.len() != 2 { + return false; + } + (0, 1) + } + N::CheckedBlockResource { + access, witness, .. + } => { + if args.len() != 3 || !witness_valid(ctx, witness, access, args[0], spec.scalar) { + return false; + } + (1, 2) + } + N::CheckedBlockStorage { + access, + witness, + storage, + .. + } => { + if args.len() != 4 + || !witness_valid(ctx, witness, access, args[0], spec.scalar) + || !storage_arg_valid(ctx, storage, args[1]) + { + return false; + } + (2, 3) + } + }; + let Some(uniform) = tuple_fields(ctx, args[uniform_index]) else { + return false; + }; + let Some(payload) = tuple_fields(ctx, args[payload_index]) else { + return false; + }; + payload.first().and_then(|&ty| pointer_scalar(ctx, ty)) == Some(spec.scalar) + && payload.iter().all(|&ty| pointer_scalar(ctx, ty).is_some()) + && uniform.first().and_then(|&ty| pointer_scalar(ctx, ty)) == Some(spec.scalar) + && uniform[1..] + .iter() + .all(|&ty| scalar_type(ctx, ty).is_some()) +} + +fn verify_call(ctx: &Context, call: &MirLibraryOp, role: LibraryRoleAttr) -> Result<(), Error> { + let op = call.get_operation().deref(ctx); + let Some(signature) = call.get_attr_library_signature(ctx) else { + return verify_err!( + op.loc(), + "native/storage mir.library requires its runtime signature" + ); + }; + let signature = signature.get_type(ctx); + let signature = signature.deref(ctx); + let Some(signature) = signature.downcast_ref::() else { + return verify_err!(op.loc(), "library_signature must be a FunctionType"); + }; + let args = signature.arg_types(); + let results = signature.res_types(); + if args.len() != op.get_num_operands() + || results.len() != op.get_num_results() + || !args + .iter() + .enumerate() + .all(|(i, &ty)| retained_type_matches(ctx, op.get_operand(i).get_type(ctx), ty)) + || !results + .iter() + .enumerate() + .all(|(i, &ty)| op.get_result(i).get_type(ctx) == ty) + { + return verify_err!( + op.loc(), + "mir.library operands/results do not match the retained runtime signature" + ); + } + let valid = match role { + LibraryRoleAttr::Native(native) => results.is_empty() && native_valid(ctx, native, &args), + LibraryRoleAttr::StorageNew(policy) => { + policy_valid(policy) + && args.is_empty() + && results.len() == 1 + && descriptor_type(ctx, results[0]) + } + LibraryRoleAttr::StorageReconfigure { old, new } => { + policy_valid(old) + && policy_valid(new) + && args.len() == 1 + && descriptor_type(ctx, args[0]) + && results.len() == 1 + && descriptor_type(ctx, results[0]) + } + LibraryRoleAttr::StorageReserve { policy, layout } => { + policy_valid(policy) + && layout.count > 0 + && layout.bytes > 0 + && layout.alignment.is_power_of_two() + && valid_scalar(layout.scalar) + && layout.scalar != LibraryScalar::Float(16) + && args.len() == 1 + && descriptor_borrow(ctx, args[0]) + && results.len() == 1 + && reservation_scalar(ctx, results[0]) == Some(layout.scalar) + } + LibraryRoleAttr::StorageRelease => { + args.len() == 1 && results.is_empty() && reservation_scalar(ctx, args[0]).is_some() + } + _ => false, + }; + if !valid { + return verify_err!( + op.loc(), + "mir.library runtime signature or metadata does not match its native/storage role" + ); + } + Ok(()) +} + +/// Retain an expanded source call's effect and, when present, its actual result. +/// +/// This proof event is effectful until validation removes it. A result operand +/// is observed before destination forwarding, including in Full slot mode. +#[pliron_op(name = "mir.library_source", format, + attributes = (library_source_effect: crate::library::LibrarySourceEffectAttr))] +pub struct MirLibrarySourceOp; + +impl MirLibrarySourceOp { + pub fn new(op: Ptr) -> Self { + Self { op } + } + pub fn build( + ctx: &mut Context, + effect: crate::library::LibrarySourceEffectAttr, + result: Option, + ) -> Ptr { + let op = Operation::new( + ctx, + Self::get_concrete_op_info(), + vec![], + result.into_iter().collect(), + vec![], + 0, + ); + Self::new(op).set_attr_library_source_effect(ctx, effect); + op + } +} +impl Verify for MirLibrarySourceOp { + fn verify(&self, ctx: &Context) -> Result<(), Error> { + let op = self.get_operation().deref(ctx); + if self.get_attr_library_source_effect(ctx).is_none() + || op.get_num_results() != 0 + || op.get_num_operands() > 1 + { + return verify_err!( + op.loc(), + "source event requires identity, zero results and at most its actual result operand" + ); + } + Ok(()) + } +} + +/// Retain a source mutation check on actual evaluated destination addresses. +#[pliron_op(name = "mir.library_write_check", format, + attributes = (library_write_source: crate::library::LibraryWriteSourceAttr))] +pub struct MirLibraryWriteCheckOp; + +impl MirLibraryWriteCheckOp { + pub fn new(op: Ptr) -> Self { + Self { op } + } + pub fn build( + ctx: &mut Context, + source: crate::library::LibraryWriteSourceAttr, + addresses: Vec, + ) -> Ptr { + let op = Operation::new( + ctx, + Self::get_concrete_op_info(), + vec![], + addresses, + vec![], + 0, + ); + Self::new(op).set_attr_library_write_source(ctx, source); + op + } +} +impl Verify for MirLibraryWriteCheckOp { + fn verify(&self, ctx: &Context) -> Result<(), Error> { + let op = self.get_operation().deref(ctx); + if self.get_attr_library_write_source(ctx).is_none() || op.get_num_results() != 0 { + return verify_err!( + op.loc(), + "write check requires original source identity and zero results" + ); + } + Ok(()) + } +} + +pub(crate) fn register(ctx: &mut Context) { + MirLibrarySourceOp::register(ctx); + MirLibraryWriteCheckOp::register(ctx); + MirLibraryOp::register(ctx); +} diff --git a/cuda-oxide/crates/dialect-mir/src/ops/mod.rs b/cuda-oxide/crates/dialect-mir/src/ops/mod.rs index 0e4065cd13..89f0409220 100644 --- a/cuda-oxide/crates/dialect-mir/src/ops/mod.rs +++ b/cuda-oxide/crates/dialect-mir/src/ops/mod.rs @@ -27,6 +27,7 @@ //! │ cast │ Type conversions │ 1 │ //! │ storage │ Variable lifetime markers │ 2 │ //! │ call │ Function calls │ 1 │ +//! │ library │ Native hooks and source proof events │ 3 │ //! └──────────────┴──────────────────────────────────────┴─────┘ //! ``` //! @@ -114,6 +115,7 @@ pub mod control_flow; pub mod debug; pub mod enum_ops; pub mod function; +pub mod library; pub mod memory; pub mod storage; @@ -128,6 +130,7 @@ pub use control_flow::*; pub use debug::*; pub use enum_ops::*; pub use function::*; +pub use library::*; pub use memory::*; pub use storage::*; @@ -145,4 +148,5 @@ pub fn register(ctx: &mut Context) { cast::register(ctx); storage::register(ctx); call::register(ctx); + library::register(ctx); } diff --git a/cuda-oxide/crates/llvm-export/src/export/debug.rs b/cuda-oxide/crates/llvm-export/src/export/debug.rs index ca0bfcfc31..4feb86212a 100644 --- a/cuda-oxide/crates/llvm-export/src/export/debug.rs +++ b/cuda-oxide/crates/llvm-export/src/export/debug.rs @@ -177,9 +177,12 @@ impl<'a> ModuleExportState<'a> { "!{dwarf_version_id} = !{{i32 2, !\"Dwarf Version\", i32 2}}" ) .unwrap(); + // NVVM requires Error (1), including when linking native CUDA LTO-IR. + // Ordinary LLVM debug metadata keeps its Warning (2) merge behavior. + let debug_version_behavior = if self.nvvm_ir_dialect.is_some() { 1 } else { 2 }; writeln!( output, - "!{debug_info_version_id} = !{{i32 2, !\"Debug Info Version\", i32 3}}" + "!{debug_info_version_id} = !{{i32 {debug_version_behavior}, !\"Debug Info Version\", i32 3}}" ) .unwrap(); diff --git a/cuda-oxide/crates/llvm-export/src/export/externs.rs b/cuda-oxide/crates/llvm-export/src/export/externs.rs index ceeab494f0..2cea5a5fc0 100644 --- a/cuda-oxide/crates/llvm-export/src/export/externs.rs +++ b/cuda-oxide/crates/llvm-export/src/export/externs.rs @@ -225,22 +225,18 @@ pub struct DeviceExternDecl { /// NVVM attributes for device extern declarations. /// -/// NOTE: These attributes are currently **not emitted** to the LLVM IR output. -/// When linking LTOIR via nvJitLink, the external library's LTOIR already contains -/// proper attributes (convergent, nounwind, memory, etc.) on the function DEFINITIONS. -/// nvJitLink uses the definition's attributes during LTO, making attributes on -/// declarations redundant. -/// -/// This struct is retained for potential future use or for debugging/inspection. +/// Convergence is emitted on the declaration before libNVVM optimization. +/// Memory-effect annotations remain inactive; they are not inferred from external +/// definitions that become available only during LTO. #[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] pub struct DeviceExternAttrs { - /// Function is convergent (all threads must execute together). + /// Function has convergence requirements for its participating threads. pub is_convergent: bool, - /// Function is pure (no side effects). Maps to LLVM `readnone`. + /// Function is pure (no side effects). Inactive for Rust source annotations. pub is_pure: bool, - /// Function is read-only (only reads memory). Maps to LLVM `readonly`. + /// Function is read-only (only reads memory). Inactive for Rust source annotations. pub is_readonly: bool, } diff --git a/cuda-oxide/crates/llvm-export/src/export/function.rs b/cuda-oxide/crates/llvm-export/src/export/function.rs index db8900f9a2..3ff34785ef 100644 --- a/cuda-oxide/crates/llvm-export/src/export/function.rs +++ b/cuda-oxide/crates/llvm-export/src/export/function.rs @@ -162,7 +162,7 @@ fn export_reference_parameter_attrs( /// Shared-global owner attributes carry the raw symbol, so the module prepass, /// function definition, and global attachment must all normalize it exactly /// once through the same helper. -pub(super) fn exported_function_name(raw_name: &str) -> String { +pub fn exported_function_name(raw_name: &str) -> String { if raw_name.starts_with("llvm_") { decode_intrinsic_identifier(raw_name) } else { diff --git a/cuda-oxide/crates/llvm-export/src/export/mod.rs b/cuda-oxide/crates/llvm-export/src/export/mod.rs index 3c2f4040d4..467e01143f 100644 --- a/cuda-oxide/crates/llvm-export/src/export/mod.rs +++ b/cuda-oxide/crates/llvm-export/src/export/mod.rs @@ -53,6 +53,9 @@ pub use config::{ DebugKind, ExportBackendConfig, FunctionLocalStaticPlacement, NvvmExportConfig, NvvmIrDialect, PtxExportConfig, }; +pub use function::exported_function_name; +pub use module::retain_functions; + pub use externs::{AsDeviceExtern, DeviceExternAttrs, DeviceExternDecl, DeviceExternType}; use pliron::{builtin::ops::ModuleOp, context::Context}; diff --git a/cuda-oxide/crates/llvm-export/src/export/module.rs b/cuda-oxide/crates/llvm-export/src/export/module.rs index 1a9e934851..07f41814a0 100644 --- a/cuda-oxide/crates/llvm-export/src/export/module.rs +++ b/cuda-oxide/crates/llvm-export/src/export/module.rs @@ -47,6 +47,24 @@ fn validate_export_config(config: &dyn ExportBackendConfig) -> Result<(), String Ok(()) } +const RETAINED_FUNCTIONS: &str = "cuda_oxide_retained_functions"; + +/// Retain selected definitions consumed by native code in the same artifact. +/// Names use the collector's identity; export applies its ordinary normalization. +/// The module survives MIR lowering, so no per-function attribute copy is needed. +pub fn retain_functions(ctx: &mut Context, module: &ModuleOp, names: &[String]) { + use pliron::builtin::attributes::{StringAttr, VecAttr}; + module.get_operation().deref_mut(ctx).attributes.set( + RETAINED_FUNCTIONS.try_into().unwrap(), + VecAttr::new( + names + .iter() + .map(|name| StringAttr::new(name.clone()).into()) + .collect(), + ), + ); +} + fn index_module_symbols( state: &mut ModuleExportState<'_>, module: &ModuleOp, @@ -55,7 +73,34 @@ fn index_module_symbols( let Some(block) = region.iter(state.ctx).next() else { return Ok(()); }; + let mut retained = std::collections::HashSet::new(); + { + use pliron::builtin::attributes::{StringAttr, VecAttr}; + let operation = module.get_operation().deref(state.ctx); + if let Some(names) = operation + .attributes + .get::(&RETAINED_FUNCTIONS.try_into().unwrap()) + { + for name in names.0.iter() { + let name = name + .downcast_ref::() + .ok_or("invalid retained function name")?; + retained.insert(String::from(name.clone())); + } + } + } for operation in block.deref(state.ctx).iter(state.ctx) { + if let Some(function) = Operation::get_op::(operation, state.ctx) { + let name = function.get_symbol_name(state.ctx).to_string(); + if retained.remove(&name) { + if function.get_entry_block(state.ctx).is_none() { + return Err(format!("retained callback `{name}` has no definition")); + } + state + .device_functions + .push(super::function::exported_function_name(&name)); + } + } if let Some(global) = Operation::get_op::(operation, state.ctx) { let symbol = global.get_symbol_name(state.ctx).to_string(); let value_type = global.get_type(state.ctx); @@ -125,6 +170,11 @@ fn index_module_symbols( } } } + if !retained.is_empty() { + return Err(format!( + "retained callback definitions are missing: {retained:?}" + )); + } Ok(()) } @@ -487,10 +537,17 @@ fn index_device_externs( validate_device_extern_function_shape(state, decl)?; if let Some(previous) = state.device_externs.get(&decl.export_name) { if previous != decl { - return Err(format!( + let conflict = format!( "conflicting device-extern declarations for `@{}`", decl.export_name - )); + ); + if previous.attrs.is_convergent != decl.attrs.is_convergent { + return Err(format!( + "{conflict}: convergent is {} versus {}", + previous.attrs.is_convergent, decl.attrs.is_convergent + )); + } + return Err(conflict); } continue; } @@ -610,12 +667,9 @@ pub(super) fn export_module_with_externs_impl( // 2. Device extern declarations (before function definitions) // - // NOTE: We intentionally do NOT emit LLVM attributes on these declarations. - // The external LTOIR (from nvcc -dc -dlto) already contains proper attributes - // (convergent, nounwind, memory, etc.) on the function DEFINITIONS. - // When nvJitLink performs LTO linking, it uses the definition's attributes. - // Attributes on declarations are redundant and were causing issues where - // all externs incorrectly got the same attribute group. + // Convergence must be visible before libNVVM optimizes this module, not + // only when external definitions arrive at LTO. Emit it per declaration + // so unrelated externs do not inherit a shared attribute group. if !device_externs.is_empty() { writeln!( &mut output, @@ -643,7 +697,11 @@ pub(super) fn export_module_with_externs_impl( } param.write_llvm_with_attr(&mut output, state.legacy_typed_pointers())?; } - writeln!(&mut output, ")").unwrap(); + write!(&mut output, ")").unwrap(); + if decl.attrs.is_convergent { + write!(&mut output, " convergent").unwrap(); + } + writeln!(&mut output).unwrap(); } writeln!(&mut output).unwrap(); } @@ -711,7 +769,7 @@ pub(super) fn export_module_with_externs_impl( } // 6. Emit attribute groups for convergent intrinsics used by module functions - // Note: Device extern declarations no longer get attribute groups - see section 2 comment. + // Device extern convergence is emitted inline, independently of this group. if state.convergent_used { writeln!(&mut output).unwrap(); writeln!(&mut output, "attributes #0 = {{ convergent }}").unwrap(); diff --git a/cuda-oxide/crates/llvm-export/src/export/names.rs b/cuda-oxide/crates/llvm-export/src/export/names.rs index f3665a4571..5e09a83b12 100644 --- a/cuda-oxide/crates/llvm-export/src/export/names.rs +++ b/cuda-oxide/crates/llvm-export/src/export/names.rs @@ -72,10 +72,13 @@ pub(super) fn has_device_prefix(name: &str) -> bool { /// /// The reserved prefix is needed internally for MIR-level detection but /// should not leak into the final LLVM IR / PTX / LTOIR output. Returns -/// `name` unchanged for non-device symbols and for device-extern declarations -/// (those keep their original-name `link_name` attribute). +/// `name` unchanged for root-specialized helpers, non-device symbols and +/// device-extern declarations (those keep their original-name `link_name` +/// attribute). pub(super) fn strip_device_prefix(name: &str) -> String { - if is_device_extern_symbol(name) { + if name.starts_with(reserved_oxide_symbols::LIBRARY_CONTEXT_PREFIX) + || is_device_extern_symbol(name) + { return name.to_string(); } device_base_name(name) @@ -85,7 +88,25 @@ pub(super) fn strip_device_prefix(name: &str) -> String { #[cfg(test)] mod tests { - use super::{decode_intrinsic_identifier, legacy_nvvm_atomic_add_signature}; + use super::{ + decode_intrinsic_identifier, legacy_nvvm_atomic_add_signature, strip_device_prefix, + }; + + #[test] + fn root_context_keeps_marked_helper_identity() { + for root in ["kernel_a", "kernel_b"] { + let name = format!( + "{}8_{}_cuda_oxide_device_246e25db_helper", + reserved_oxide_symbols::LIBRARY_CONTEXT_PREFIX, + root + ); + assert_eq!(strip_device_prefix(&name), name); + } + assert_eq!( + strip_device_prefix("mycrate__cuda_oxide_device_246e25db_helper"), + "helper" + ); + } #[test] fn intrinsic_identifier_decoding_preserves_dots_and_literal_underscores() { diff --git a/cuda-oxide/crates/llvm-export/tests/export_test/debug_info.rs b/cuda-oxide/crates/llvm-export/tests/export_test/debug_info.rs index abc11b7af6..bbb601856c 100644 --- a/cuda-oxide/crates/llvm-export/tests/export_test/debug_info.rs +++ b/cuda-oxide/crates/llvm-export/tests/export_test/debug_info.rs @@ -90,6 +90,20 @@ fn line_table_debug_metadata_emits_function_scope_and_instruction_locations() { let ir = export_module_to_string_with_config(&ctx, &module, &config).expect("debug export succeeds"); + assert!(ir.contains("!{i32 2, !\"Debug Info Version\", i32 3}")); + for debug_kind in [DebugKind::LineTables, DebugKind::Full] { + let nvvm_config = DebugConfig { + inner: NvvmExportConfig::new(NvvmIrDialect::Modern), + debug_kind, + }; + let nvvm_ir = export_module_to_string_with_config(&ctx, &module, &nvvm_config) + .expect("NVVM debug export succeeds"); + assert!( + nvvm_ir.contains("!{i32 1, !\"Debug Info Version\", i32 3}"), + "NVVM requires Error merge behavior for native LTO linkage:\n{nvvm_ir}" + ); + } + let define_line = ir .lines() .find(|line| line.starts_with("define ")) diff --git a/cuda-oxide/crates/llvm-export/tests/export_test/device_externs.rs b/cuda-oxide/crates/llvm-export/tests/export_test/device_externs.rs index ee928c3819..f68a227611 100644 --- a/cuda-oxide/crates/llvm-export/tests/export_test/device_externs.rs +++ b/cuda-oxide/crates/llvm-export/tests/export_test/device_externs.rs @@ -23,6 +23,113 @@ use pliron::{ use crate::common::module_top_block; +fn mixed_convergence_module(ctx: &mut Context) -> (ModuleOp, Vec) { + let module = ModuleOp::new(ctx, "mixed_convergence".try_into().unwrap()); + let module_block = module_top_block(ctx, &module); + let ptr_ty = PointerType::get(ctx, 0); + let void_ty = VoidType::get(ctx); + let external_ty = FuncType::get(ctx, void_ty.into(), vec![ptr_ty.into()], false); + let caller = FuncOp::new(ctx, "caller".try_into().unwrap(), external_ty); + let entry = caller.get_or_create_entry_block(ctx); + let pointer = entry.deref(ctx).get_argument(0); + let mut externs = Vec::new(); + for (name, is_convergent) in [("collective", true), ("ordinary", false)] { + FuncOp::new(ctx, name.try_into().unwrap(), external_ty) + .get_operation() + .insert_at_back(module_block, ctx); + CallOp::new( + ctx, + CallOpCallable::Direct(name.try_into().unwrap()), + external_ty, + vec![pointer], + ) + .get_operation() + .insert_at_back(entry, ctx); + externs.push(DeviceExternDecl { + export_name: name.to_string(), + param_types: vec![DeviceExternType::pointer_to( + DeviceExternType::Integer(32), + 0, + )], + return_type: DeviceExternType::Void, + attrs: DeviceExternAttrs { + is_convergent, + ..DeviceExternAttrs::default() + }, + }); + } + ReturnOp::new(ctx, None) + .get_operation() + .insert_at_back(entry, ctx); + caller.get_operation().insert_at_back(module_block, ctx); + (module, externs) +} + +#[test] +fn device_extern_convergence_is_per_declaration_in_both_dialects() { + let mut ctx = Context::new(); + let (module, mut externs) = mixed_convergence_module(&mut ctx); + // Repeated identical declarations are still emitted exactly once. + externs.push(externs[0].clone()); + for (dialect, pointer) in [ + (NvvmIrDialect::Modern, "ptr"), + (NvvmIrDialect::LegacyLlvm7, "i32*"), + ] { + let ir = + export_module_with_externs(&ctx, &module, &externs, &NvvmExportConfig::new(dialect)) + .unwrap(); + let declarations: Vec<_> = ir + .lines() + .filter(|line| line.starts_with("declare ")) + .collect(); + assert_eq!( + declarations, + [ + format!("declare void @collective({pointer}) convergent"), + format!("declare void @ordinary({pointer})"), + ], + "{ir}" + ); + assert!( + ir.lines().any(|line| { + line.contains(&format!("call void @collective({pointer} ")) && line.ends_with(" #0") + }), + "{ir}" + ); + if dialect == NvvmIrDialect::LegacyLlvm7 { + assert!(ir.contains("bitcast i8* %v0 to i32*"), "{ir}"); + } + } + + externs[2].attrs.is_convergent = false; + let error = export_module_with_externs(&ctx, &module, &externs, &NvvmExportConfig::default()) + .expect_err("conflicting convergence requirements must fail"); + assert!( + error.contains("conflicting device-extern declarations for `@collective`: convergent is true versus false"), + "{error}" + ); +} + +#[test] +#[ignore = "requires an installed CUDA Toolkit with modern and legacy libNVVM"] +fn device_extern_convergence_compiles_with_libnvvm_in_both_dialects() { + let nvvm = libnvvm_sys::LibNvvm::load().expect("CUDA Toolkit with libNVVM required"); + let mut ctx = Context::new(); + let (module, externs) = mixed_convergence_module(&mut ctx); + for (dialect, arch) in [ + (NvvmIrDialect::Modern, "-arch=compute_120"), + (NvvmIrDialect::LegacyLlvm7, "-arch=compute_80"), + ] { + let ir = + export_module_with_externs(&ctx, &module, &externs, &NvvmExportConfig::new(dialect)) + .unwrap(); + let mut program = libnvvm_sys::Program::new(&nvvm).unwrap(); + program.add_module(ir.as_bytes(), "convergence.ll").unwrap(); + let lto = program.compile(&[arch, "-gen-lto"]).unwrap(); + assert!(!lto.is_empty()); + } +} + #[test] fn legacy_device_extern_adapts_exact_pointer_arguments_and_results() { let mut ctx = Context::new(); diff --git a/cuda-oxide/crates/llvm-export/tests/export_test/kernels_and_roots.rs b/cuda-oxide/crates/llvm-export/tests/export_test/kernels_and_roots.rs index d80832d0a2..95e610ed42 100644 --- a/cuda-oxide/crates/llvm-export/tests/export_test/kernels_and_roots.rs +++ b/cuda-oxide/crates/llvm-export/tests/export_test/kernels_and_roots.rs @@ -504,3 +504,64 @@ fn kernel_keeps_host_addressable_pointer_parameters_and_device_functions_keep_sh "{ir}" ); } + +#[test] +fn native_callback_and_kernel_share_external_roots() { + let mut ctx = Context::new(); + let module = ModuleOp::new(&mut ctx, "native_callback_roots".try_into().unwrap()); + let block = module_top_block(&mut ctx, &module); + let ty = FuncType::get(&ctx, VoidType::get(&ctx).into(), vec![], false); + for name in ["entry", "selected_thunk", "ordinary_helper"] { + let function = FuncOp::new(&mut ctx, name.try_into().unwrap(), ty); + if name == "entry" { + function.get_operation().deref_mut(&ctx).attributes.set( + "gpu_kernel".try_into().unwrap(), + StringAttr::new("true".into()), + ); + } + let entry = function.get_or_create_entry_block(&mut ctx); + ReturnOp::new(&mut ctx, None) + .get_operation() + .insert_at_back(entry, &ctx); + function.get_operation().insert_at_back(block, &ctx); + } + llvm_export::export::retain_functions(&mut ctx, &module, &["selected_thunk".into()]); + let result = export_module_with_externs_and_roots::( + &ctx, + &module, + &[], + &PtxExportConfig, + ) + .unwrap(); + assert_eq!(result.public_symbols, ["entry", "selected_thunk"]); + assert!( + result + .llvm_ir + .contains("@llvm.used = appending global [2 x ptr] [ptr @entry, ptr @selected_thunk]") + ); + llvm_export::export::retain_functions(&mut ctx, &module, &["missing_thunk".into()]); + assert!( + export_module_with_externs_and_roots::( + &ctx, + &module, + &[], + &PtxExportConfig + ) + .unwrap_err() + .contains("definitions are missing") + ); + FuncOp::new(&mut ctx, "declared_thunk".try_into().unwrap(), ty) + .get_operation() + .insert_at_back(block, &ctx); + llvm_export::export::retain_functions(&mut ctx, &module, &["declared_thunk".into()]); + assert!( + export_module_with_externs_and_roots::( + &ctx, + &module, + &[], + &PtxExportConfig + ) + .unwrap_err() + .contains("has no definition") + ); +} diff --git a/cuda-oxide/crates/mir-importer/Cargo.toml b/cuda-oxide/crates/mir-importer/Cargo.toml index ff7cc7d5b1..3daf9ebc70 100644 --- a/cuda-oxide/crates/mir-importer/Cargo.toml +++ b/cuda-oxide/crates/mir-importer/Cargo.toml @@ -19,6 +19,7 @@ dialect-nvvm = { workspace = true } dialect-cute = { workspace = true } cute-layout = { workspace = true } mir-lower = { workspace = true } +mir-transforms = { workspace = true } oxide-artifacts = { workspace = true } cuda-oxide-codegen = { workspace = true } reserved-oxide-symbols = { workspace = true } diff --git a/cuda-oxide/crates/mir-importer/README.md b/cuda-oxide/crates/mir-importer/README.md index 6815539205..8c56d2cc6a 100644 --- a/cuda-oxide/crates/mir-importer/README.md +++ b/cuda-oxide/crates/mir-importer/README.md @@ -147,9 +147,22 @@ The pipeline inspects which intrinsics the code uses and selects a target: | tcgen05 / TMEM | sm_100a | Blackwell datacenter | | WGMMA | sm_90a | Hopper only | | TMA / mbarrier | sm_100 | Hopper+ compatible | +| Cluster compatibility queries | sm_90 by default | Known pre-sm90 targets use one-block clusters | | Basic CUDA | sm_80 | Ampere+ (max compat) | -Override with `CUDA_OXIDE_TARGET=`. +Override with `CUDA_OXIDE_TARGET=`. A device hint used for pre-sm90 +cluster-query fallback pins the artifact to that target; incompatible higher-floor +operations are rejected. Unused hints remain advisory. + +From `cuda-oxide/`, run the source-to-NVVM-IR cluster-query regression with: + +```sh +cargo test -p mir-importer cluster_query_lowering_preserves_target_selection_and_raw_floors +``` + +It checks pre-sm90 compatibility lowering, conditional device-hint pinning, +and the raw intrinsic target floor using the pinned Rust toolchain. It requires +neither a GPU nor CUDA compiler tools. ## Public API diff --git a/cuda-oxide/crates/mir-importer/src/kernel_resources.rs b/cuda-oxide/crates/mir-importer/src/kernel_resources.rs new file mode 100644 index 0000000000..c342afa61f --- /dev/null +++ b/cuda-oxide/crates/mir-importer/src/kernel_resources.rs @@ -0,0 +1,827 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Finalized requirements for fixed views of the dynamic shared-memory base. +//! This is not an allocator: fixed views alias and combine by maximum extent. + +use dialect_mir::{ + library::{LibraryCallPlan, LibraryCallSite, LibraryOperation}, + ops::{MirCallOp, MirExternSharedOp, MirFuncOp}, +}; +use oxide_artifacts::KernelLaunchFacts; +use pliron::builtin::{attributes::IntegerAttr, op_interfaces::SymbolOpInterface}; +use pliron::context::{Context, Ptr}; +use pliron::linked_list::ContainsLinkedList; +use pliron::operation::Operation; +use std::collections::{BTreeMap, BTreeSet}; + +pub(crate) const PLANNED_ROOT: &str = "cuda_oxide_planned_storage_root"; + +pub(crate) const FIXED_SHARED_BYTES: &str = "cuda_oxide_fixed_shared_bytes"; +pub(crate) const INFERRED_SHARED: &str = "cuda_oxide_inferred_shared"; + +fn integer_attr(ctx: &Context, op: Ptr, name: &str) -> Option { + let key: pliron::identifier::Identifier = name.try_into().unwrap(); + op.deref(ctx) + .attributes + .get::(&key) + .map(|value| value.value().to_u64()) +} + +/// Apply the exact geometry required by reachable native library operations. +/// Scratch extents are recorded on their actual shared-memory accesses and +/// collected together with ordinary Rust views below. +pub(crate) fn apply_library_block( + ctx: &mut Context, + entry: Ptr, + plan: &KernelLaunchFacts, +) -> Result<(), String> { + use pliron::builtin::types::{IntegerType, Signedness}; + use pliron::utils::apint::APInt; + + let Some((x, y, z)) = plan.exact_block else { + return Err("native library root is missing its exact block shape".into()); + }; + let required = u64::from(x) * u64::from(y) * u64::from(z); + if let Some(maximum) = integer_attr(ctx, entry, "maxntid") + && required > maximum + { + return Err(format!( + "native library exact block ({x}, {y}, {z}) exceeds launch_bounds maximum {maximum}" + )); + } + let axes = [("reqntid_x", x), ("reqntid_y", y), ("reqntid_z", z)]; + for (name, dimension) in axes { + if let Some(existing) = integer_attr(ctx, entry, name) + && existing != u64::from(dimension) + { + return Err(format!( + "native library exact block ({x}, {y}, {z}) conflicts with kernel launch contract" + )); + } + } + let ty = IntegerType::get(ctx, 64, Signedness::Unsigned); + for (name, dimension) in axes { + entry.deref_mut(ctx).attributes.set( + name.try_into().unwrap(), + IntegerAttr::new( + ty, + APInt::from_u64( + u64::from(dimension), + std::num::NonZeroUsize::new(64).unwrap(), + ), + ), + ); + } + Ok(()) +} + +#[derive(Clone, Default)] +struct FunctionResources { + inferred: bool, + fixed: bool, + unconstrained: bool, + bytes: u64, + alignment: u64, + declared_alignment: Option, + callees: Vec, +} + +#[derive(Clone)] +struct OrdinaryCall { + site: Option, + callee: String, +} + +fn scan(ctx: &Context, op: Ptr, resources: &mut FunctionResources, native_only: bool) { + if let Some(shared) = Operation::get_op::(op, ctx) + && !(native_only && planned_root(ctx, op).is_some()) + { + match integer_attr(ctx, op, FIXED_SHARED_BYTES) { + Some(bytes) => { + resources.fixed = true; + resources.bytes = resources + .bytes + .max(shared.get_byte_offset_value(ctx).saturating_add(bytes)); + resources.alignment = resources.alignment.max(shared.get_alignment_value(ctx)); + } + None => resources.unconstrained = true, + } + } + if let Some(call) = Operation::get_op::(op, ctx) + && let Some(callee) = call.get_attr_callee(ctx) + { + resources.callees.push(OrdinaryCall { + site: call + .get_attr_library_source_site(ctx) + .map(|site| (*site).clone()), + callee: String::from((*callee).clone()), + }); + } + for region in op.deref(ctx).regions() { + for block in region.deref(ctx).iter(ctx) { + for child in block.deref(ctx).iter(ctx) { + scan(ctx, child, resources, native_only); + } + } + } +} + +/// Ordinary requirements captured before preparation can remove views or calls. +/// Consumed by completion so compiler-planned offsets are relocated only once. +pub(crate) struct OrdinaryResources { + kernels: BTreeMap, + alignments: BTreeMap, + functions: BTreeMap, +} + +struct KernelResources { + resources: FunctionResources, + exact_block: Option<(u32, u32, u32)>, + maximum: Option, +} + +fn narrow(kernel: &str, value: u64) -> Result { + u32::try_from(value).map_err(|_| format!("{kernel}: compiled shared-memory layout exceeds u32")) +} + +/// Capture ordinary views, original helper reachability and launch constraints. +/// Unconstrained roots are retained even when they produce no launch fact. +pub(crate) fn capture_ordinary( + ctx: &Context, + module: Ptr, + kernels: impl IntoIterator, +) -> Result { + capture(ctx, module, kernels, true) +} + +/// Replay saved ordinary edges under the finalized helper contexts. Names here +/// are physical MIR symbols; source sites retain their original raw callers. +/// The target resolver is shared with actual context materialization. +pub(crate) fn bind_ordinary_contexts( + ordinary: &mut OrdinaryResources, + contexts: &BTreeMap, + mut resolve: impl FnMut(Option<&LibraryCallSite>, &str, &str) -> Result, +) -> Result<(), String> { + let replaced: BTreeSet<_> = contexts.values().collect(); + let mut targets: BTreeMap<_, _> = ordinary + .functions + .keys() + .filter(|name| !replaced.contains(name)) + .map(|name| (name.clone(), name.clone())) + .collect(); + targets.extend(contexts.clone()); + let mut functions = BTreeMap::new(); + for (emitted, source) in targets { + let mut local = ordinary + .functions + .get(&source) + .ok_or_else(|| format!("ordinary resource context has no source function {source}"))? + .clone(); + for call in &mut local.callees { + call.callee = resolve(call.site.as_ref(), &emitted, &call.callee)?; + } + functions.insert(emitted, local); + } + for kernel in ordinary.kernels.keys() { + if !functions.contains_key(kernel) { + return Err(format!("ordinary resource context removed kernel {kernel}")); + } + } + ordinary.functions = functions; + aggregate_ordinary(ordinary); + Ok(()) +} + +/// Validate saved pre-preparation resources against the actual library plans. +/// Only Native and Reserve emit shared views; zero-byte views still count. +pub(crate) fn validate_ordinary_compatibility( + ordinary: &OrdinaryResources, + calls: &BTreeMap, +) -> Result<(), String> { + let planned: BTreeSet<_> = calls + .values() + .filter(|plan| { + matches!( + plan.operation, + LibraryOperation::Native | LibraryOperation::StorageReserve + ) + }) + .filter_map(|plan| plan.planned_root.as_ref()) + .collect(); + for root in planned { + let saved = ordinary + .kernels + .get(root) + .ok_or("planned storage root has no captured ordinary resources")?; + if saved.resources.unconstrained { + return Err(format!( + "{root}: planned storage cannot include an unconstrained native dynamic shared view" + )); + } + } + for (kernel, saved) in &ordinary.kernels { + if saved.resources.inferred && saved.resources.unconstrained { + return Err(format!( + "{kernel}: inferred dynamic shared memory cannot include unconstrained get/get_raw/offset accesses; describe the complete base region with DynamicSharedArray::get_fixed or declare an explicit launch contract" + )); + } + } + Ok(()) +} + +fn capture( + ctx: &Context, + module: Ptr, + kernels: impl IntoIterator, + ordinary_only: bool, +) -> Result { + let mut functions = BTreeMap::new(); + let mut entries = BTreeMap::new(); + let mut operations = Vec::new(); + walk(ctx, module, &mut operations); + for op in operations { + if let Some(function) = MirFuncOp::wrap(ctx, op) { + let declared_alignment = integer_attr(ctx, op, "dynamic_shared_alignment"); + let mut local = FunctionResources { + inferred: integer_attr(ctx, op, INFERRED_SHARED).is_some(), + alignment: declared_alignment.unwrap_or(1), + declared_alignment, + ..Default::default() + }; + scan(ctx, op, &mut local, ordinary_only); + let name = function.get_symbol_name(ctx).to_string(); + entries.insert(name.clone(), op); + functions.insert(name, local); + } + } + let mut captured = BTreeMap::new(); + for kernel in kernels { + let Some(&entry) = entries.get(&kernel) else { + continue; + }; + let exact_block = match ( + integer_attr(ctx, entry, "reqntid_x"), + integer_attr(ctx, entry, "reqntid_y"), + integer_attr(ctx, entry, "reqntid_z"), + ) { + (Some(x), Some(y), Some(z)) => Some(( + narrow(&kernel, x)?, + narrow(&kernel, y)?, + narrow(&kernel, z)?, + )), + _ => None, + }; + captured.insert( + kernel, + KernelResources { + resources: FunctionResources::default(), + exact_block, + maximum: integer_attr(ctx, entry, "maxntid"), + }, + ); + } + let mut ordinary = OrdinaryResources { + kernels: captured, + alignments: BTreeMap::new(), + functions, + }; + aggregate_ordinary(&mut ordinary); + Ok(ordinary) +} + +/// The existing fixed-view maximum and alignment propagation over saved edges. +/// Replaying after context binding prevents unrelated roots' helper alignments +/// from being merged into every specialized copy. +fn aggregate_ordinary(ordinary: &mut OrdinaryResources) { + ordinary.alignments.clear(); + for (name, local) in &ordinary.functions { + let Some(alignment) = local.declared_alignment else { + continue; + }; + let mut pending = vec![name.clone()]; + let mut visited = BTreeSet::new(); + while let Some(name) = pending.pop() { + if !visited.insert(name.clone()) { + continue; + } + let Some(local) = ordinary.functions.get(&name) else { + continue; + }; + let saved = ordinary.alignments.entry(name).or_default(); + *saved = (*saved).max(alignment); + pending.extend(local.callees.iter().map(|call| call.callee.clone())); + } + } + for (kernel, saved) in &mut ordinary.kernels { + let mut combined = FunctionResources { + alignment: 1, + ..Default::default() + }; + let mut pending = vec![kernel.clone()]; + let mut visited = BTreeSet::new(); + while let Some(name) = pending.pop() { + if !visited.insert(name.clone()) { + continue; + } + let Some(local) = ordinary.functions.get(&name) else { + continue; + }; + merge_resources(&mut combined, local); + combined.alignment = combined + .alignment + .max(ordinary.alignments.get(&name).copied().unwrap_or(1)); + pending.extend(local.callees.iter().map(|call| call.callee.clone())); + } + saved.resources = combined; + } +} + +fn planned_root(ctx: &Context, op: Ptr) -> Option { + let key: pliron::identifier::Identifier = PLANNED_ROOT.try_into().unwrap(); + op.deref(ctx) + .attributes + .get::(&key) + .map(|value| String::from((*value).clone())) +} +fn walk(ctx: &Context, op: Ptr, result: &mut Vec>) { + result.push(op); + for region in op.deref(ctx).regions() { + for block in region.deref(ctx).iter(ctx) { + for child in block.deref(ctx).iter(ctx) { + walk(ctx, child, result); + } + } + } +} +fn merge_resources(saved: &mut FunctionResources, actual: &FunctionResources) { + saved.inferred |= actual.inferred; + saved.fixed |= actual.fixed; + saved.unconstrained |= actual.unconstrained; + saved.bytes = saved.bytes.max(actual.bytes); + saved.alignment = saved.alignment.max(actual.alignment); +} + +/// Relocate late planned views after the saved ordinary prefix and join facts. +/// Preparation is allowed to remove ordinary views/helper edges: their original +/// constraints still apply. The owned snapshot is not reusable for relocation. +pub(crate) fn complete_planned( + ctx: &mut Context, + module: Ptr, + mut ordinary: OrdinaryResources, + roots: &BTreeMap, +) -> Result, String> { + use pliron::builtin::types::{IntegerType, Signedness}; + use pliron::utils::apint::APInt; + + // Raw and legacy calls also emit fixed shared views without a planned + // tag. They contribute to the ordinary prefix, even for zero-byte scratch. + let late = capture_ordinary(ctx, module, ordinary.kernels.keys().cloned())?; + for (name, actual) in late.kernels { + merge_resources( + &mut ordinary.kernels.get_mut(&name).unwrap().resources, + &actual.resources, + ); + } + for (name, alignment) in late.alignments { + let saved = ordinary.alignments.entry(name).or_default(); + *saved = (*saved).max(alignment); + } + let mut operations = Vec::new(); + walk(ctx, module, &mut operations); + let mut relocated = Vec::new(); + for &op in &operations { + let Some(root) = planned_root(ctx, op) else { + continue; + }; + let plan = roots + .get(&root) + .ok_or("planned storage root has no finalized layout")?; + let saved = ordinary + .kernels + .get(&root) + .ok_or("planned storage root has no captured ordinary resources")?; + if saved.resources.unconstrained { + return Err(format!( + "{root}: planned storage cannot include an unconstrained native dynamic shared view" + )); + } + let alignment = u64::from(plan.dynamic_shared_alignment); + if !alignment.is_power_of_two() { + return Err("invalid planned storage alignment".into()); + } + let base = saved + .resources + .bytes + .checked_add(alignment - 1) + .map(|n| n & !(alignment - 1)) + .ok_or("native prefix and storage plan overflow")?; + let end = base + .checked_add(u64::from(plan.dynamic_shared_bytes)) + .ok_or("native prefix and storage plan overflow")?; + u32::try_from(end).map_err(|_| "native prefix and storage plan exceed u32")?; + let shared = Operation::get_op::(op, ctx) + .ok_or("planned storage tag is not on a shared view")?; + let offset = base + .checked_add(shared.get_byte_offset_value(ctx)) + .ok_or("native prefix and storage plan overflow")?; + let bytes = integer_attr(ctx, op, FIXED_SHARED_BYTES) + .ok_or("planned storage view has no fixed extent")?; + let extent = offset + .checked_add(bytes) + .ok_or("native prefix and storage plan overflow")?; + narrow(&root, extent)?; + relocated.push((op, root, offset, extent, shared.get_alignment_value(ctx))); + } + for (root, plan) in roots { + let saved = ordinary + .kernels + .get_mut(root) + .ok_or("native library root has no captured ordinary resources")?; + let shape = plan + .exact_block + .ok_or("native library root is missing its exact block shape")?; + if saved.exact_block.is_some_and(|old| old != shape) { + let (x, y, z) = shape; + return Err(format!( + "native library exact block ({x}, {y}, {z}) conflicts with kernel launch contract" + )); + } + let (x, y, z) = shape; + if let Some(maximum) = saved.maximum + && u64::from(x) * u64::from(y) * u64::from(z) > maximum + { + return Err(format!( + "native library exact block ({x}, {y}, {z}) exceeds launch_bounds maximum {maximum}" + )); + } + saved.exact_block = Some(shape); + } + for (op, root, offset, extent, alignment) in relocated { + MirExternSharedOp::new(op).set_byte_offset_value(ctx, offset); + let resources = &mut ordinary.kernels.get_mut(&root).unwrap().resources; + resources.fixed = true; + resources.bytes = resources.bytes.max(extent); + resources.alignment = resources.alignment.max(alignment); + } + // Restore alignment propagation over the original helper graph before + // propagating across the final graph, which also contains native calls. + let layout_ty = IntegerType::get(ctx, 64, Signedness::Unsigned); + for op in operations { + if let Some(function) = MirFuncOp::wrap(ctx, op) { + let name = function.get_symbol_name(ctx).to_string(); + if let Some(&alignment) = ordinary.alignments.get(&name) { + let alignment = + alignment.max(integer_attr(ctx, op, "dynamic_shared_alignment").unwrap_or(1)); + op.deref_mut(ctx).attributes.set( + "dynamic_shared_alignment".try_into().unwrap(), + IntegerAttr::new( + layout_ty, + APInt::from_u64(alignment, std::num::NonZeroUsize::new(64).unwrap()), + ), + ); + } + if let Some(plan) = roots.get(&name) { + apply_library_block(ctx, op, plan)?; + } + } + } + mir_lower::propagate_kernel_dynamic_shared_alignments(ctx, module); + let actual = capture(ctx, module, ordinary.kernels.keys().cloned(), false)?; + for (name, actual) in actual.kernels { + merge_resources( + &mut ordinary.kernels.get_mut(&name).unwrap().resources, + &actual.resources, + ); + } + let mut facts = BTreeMap::new(); + for (kernel, saved) in ordinary.kernels { + let combined = saved.resources; + if combined.inferred && combined.unconstrained { + return Err(format!( + "{kernel}: inferred dynamic shared memory cannot include unconstrained get/get_raw/offset accesses; describe the complete base region with DynamicSharedArray::get_fixed or declare an explicit launch contract" + )); + } + if combined.unconstrained || !(combined.inferred || combined.fixed) { + continue; + } + facts.insert( + kernel.clone(), + KernelLaunchFacts { + exact_block: saved.exact_block, + dynamic_shared_bytes: narrow(&kernel, combined.bytes)?, + dynamic_shared_alignment: narrow(&kernel, combined.alignment)?, + }, + ); + } + Ok(facts) +} + +#[cfg(test)] +mod tests { + use super::*; + use dialect_mir::{ + ops::MirCallOp, + types::{MirPtrType, MirTupleType}, + }; + use pliron::{ + basic_block::BasicBlock, + builtin::{ + attributes::{StringAttr, TypeAttr}, + op_interfaces::SingleBlockRegionInterface, + ops::ModuleOp, + types::{FunctionType, IntegerType, Signedness}, + }, + irbuild::{ + listener::DummyListener, + rewriter::{IRRewriter, Rewriter}, + }, + op::Op, + utils::apint::APInt, + }; + + fn function( + ctx: &mut Context, + module: Ptr, + name: &str, + ) -> (Ptr, Ptr) { + let signature = FunctionType::get(ctx, vec![], vec![]); + let op = Operation::new( + ctx, + MirFuncOp::get_concrete_op_info(), + vec![], + vec![], + vec![], + 1, + ); + let function = MirFuncOp::new(ctx, op, TypeAttr::new(signature.into())); + function.set_symbol_name(ctx, name.try_into().unwrap()); + Operation::get_op::(module, ctx) + .unwrap() + .append_operation(ctx, op, 0); + let block = BasicBlock::new(ctx, None, vec![]); + block.insert_at_back(op.deref(ctx).get_region(0), ctx); + (op, block) + } + + fn attr(ctx: &mut Context, op: Ptr, name: &str, value: u64) { + let ty = IntegerType::get(ctx, 64, Signedness::Unsigned); + op.deref_mut(ctx).attributes.set( + name.try_into().unwrap(), + IntegerAttr::new( + ty, + APInt::from_u64(value, std::num::NonZeroUsize::new(64).unwrap()), + ), + ); + } + + fn call(ctx: &mut Context, block: Ptr, callee: &str) -> Ptr { + let unit = MirTupleType::get(ctx, vec![]).into(); + let op = Operation::new( + ctx, + MirCallOp::get_concrete_op_info(), + vec![unit], + vec![], + vec![], + 0, + ); + MirCallOp::new(op).set_attr_callee(ctx, StringAttr::new(callee.to_owned())); + op.insert_at_back(block, ctx); + op + } + + fn view( + ctx: &mut Context, + block: Ptr, + offset: u64, + bytes: Option, + planned: Option<&str>, + ) -> Ptr { + let byte = IntegerType::get(ctx, 8, Signedness::Unsigned).into(); + let pointer = MirPtrType::get_shared(ctx, byte, true).into(); + let op = Operation::new( + ctx, + MirExternSharedOp::get_concrete_op_info(), + vec![pointer], + vec![], + vec![], + 0, + ); + let view = MirExternSharedOp::new(op); + view.set_byte_offset_value(ctx, offset); + view.set_alignment_value(ctx, 8); + if let Some(bytes) = bytes { + attr(ctx, op, FIXED_SHARED_BYTES, bytes); + } + if let Some(root) = planned { + op.deref_mut(ctx).attributes.set( + PLANNED_ROOT.try_into().unwrap(), + StringAttr::new(root.to_owned()), + ); + } + op.insert_at_back(block, ctx); + op + } + + #[test] + fn completion_preserves_deleted_helper_views_and_joins_late_native_scratch() { + let ctx = &mut Context::new(); + dialect_mir::register(ctx); + let module = ModuleOp::new(ctx, "resources".try_into().unwrap()).get_operation(); + let (root, body) = function(ctx, module, "kernel"); + let (other, other_body) = function(ctx, module, "other"); + let (_, helper) = function(ctx, module, "helper"); + let (_, raw) = function(ctx, module, "raw"); + attr(ctx, root, "dynamic_shared_alignment", 16); + attr(ctx, other, "dynamic_shared_alignment", 128); + let old_call = call(ctx, body, "helper"); + call(ctx, other_body, "helper"); + let old_view = view(ctx, helper, 5, Some(68), None); + let saved = + capture_ordinary(ctx, module, ["kernel".into(), "other".into(), "raw".into()]).unwrap(); + let mut rewriter = IRRewriter::::default(); + rewriter.erase_operation(ctx, old_call); + rewriter.erase_operation(ctx, old_view); + view(ctx, body, 16, Some(32), None); + view(ctx, raw, 0, Some(0), None); + let planned = view(ctx, body, 4, Some(12), Some("kernel")); + let roots = BTreeMap::from([( + "kernel".into(), + KernelLaunchFacts { + exact_block: Some((64, 1, 1)), + dynamic_shared_bytes: 16, + dynamic_shared_alignment: 16, + }, + )]); + let facts = complete_planned(ctx, module, saved, &roots).unwrap(); + assert_eq!( + MirExternSharedOp::new(planned).get_byte_offset_value(ctx), + 84 + ); + assert_eq!(facts["kernel"].dynamic_shared_bytes, 96); + assert_eq!(facts["kernel"].dynamic_shared_alignment, 128); + assert_eq!(facts["kernel"].exact_block, Some((64, 1, 1))); + assert_eq!(facts["other"].dynamic_shared_bytes, 73); + assert_eq!(facts["raw"].dynamic_shared_bytes, 0); + } + + #[test] + fn context_replay_preserves_distinct_alignments_after_edges_are_deleted() { + let ctx = &mut Context::new(); + dialect_mir::register(ctx); + let module = ModuleOp::new(ctx, "contexts".try_into().unwrap()).get_operation(); + let (a, a_body) = function(ctx, module, "root_a"); + let (b, b_body) = function(ctx, module, "root_b"); + let (_, helper) = function(ctx, module, "helper"); + attr(ctx, a, "dynamic_shared_alignment", 16); + attr(ctx, b, "dynamic_shared_alignment", 128); + let a_call = call(ctx, a_body, "helper"); + let b_call = call(ctx, b_body, "helper"); + for (op, caller) in [(a_call, "raw::a"), (b_call, "raw::b")] { + MirCallOp::new(op).set_attr_library_source_site( + ctx, + LibraryCallSite { + caller: caller.into(), + block: 3, + }, + ); + } + let original_view = view(ctx, helper, 0, Some(13), None); + let mut saved = capture_ordinary(ctx, module, ["root_a".into(), "root_b".into()]).unwrap(); + let mut rewriter = IRRewriter::::default(); + for op in [a_call, b_call, original_view] { + rewriter.erase_operation(ctx, op); + } + let contexts = BTreeMap::from([ + ("helper_a".into(), "helper".into()), + ("helper_b".into(), "helper".into()), + ]); + bind_ordinary_contexts(&mut saved, &contexts, |site, caller, callee| { + assert_eq!(callee, "helper"); + assert_eq!(site.unwrap().block, 3); + Ok(if caller == "root_a" { + "helper_a" + } else { + "helper_b" + } + .into()) + }) + .unwrap(); + let (clone_a, body_a) = function(ctx, module, "helper_a"); + let (clone_b, body_b) = function(ctx, module, "helper_b"); + let view_a = view(ctx, body_a, 0, Some(0), Some("root_a")); + let view_b = view(ctx, body_b, 0, Some(0), Some("root_b")); + let roots = ["root_a", "root_b"] + .into_iter() + .map(|root| { + ( + root.into(), + KernelLaunchFacts { + exact_block: Some((32, 1, 1)), + dynamic_shared_bytes: 0, + dynamic_shared_alignment: 8, + }, + ) + }) + .collect(); + let facts = complete_planned(ctx, module, saved, &roots).unwrap(); + assert_eq!(facts["root_a"].dynamic_shared_alignment, 16); + assert_eq!(facts["root_b"].dynamic_shared_alignment, 128); + for name in ["root_a", "root_b"] { + assert_eq!(facts[name].dynamic_shared_bytes, 16); + } + assert_eq!( + integer_attr(ctx, clone_a, "dynamic_shared_alignment"), + Some(16) + ); + assert_eq!( + integer_attr(ctx, clone_b, "dynamic_shared_alignment"), + Some(128) + ); + for view in [view_a, view_b] { + assert_eq!(MirExternSharedOp::new(view).get_byte_offset_value(ctx), 16); + } + } + + #[test] + fn deleted_unconstrained_helper_still_rejects_planned_or_inferred_roots() { + for planned in [false, true] { + let ctx = &mut Context::new(); + dialect_mir::register(ctx); + let module = ModuleOp::new(ctx, "unconstrained".try_into().unwrap()).get_operation(); + let (root, body) = function(ctx, module, "kernel"); + let (_, helper) = function(ctx, module, "helper"); + if !planned { + attr(ctx, root, INFERRED_SHARED, 1); + } + let old_call = call(ctx, body, "helper"); + let old_view = view(ctx, helper, 0, None, None); + let saved = capture_ordinary(ctx, module, ["kernel".into()]).unwrap(); + let mut rewriter = IRRewriter::::default(); + rewriter.erase_operation(ctx, old_call); + rewriter.erase_operation(ctx, old_view); + let expected = if planned { + "planned storage cannot include an unconstrained" + } else { + "inferred dynamic shared memory cannot include unconstrained" + }; + let site = LibraryCallSite { + caller: "kernel".into(), + block: 0, + }; + if planned { + for operation in [LibraryOperation::Native, LibraryOperation::StorageReserve] { + let mut plan = LibraryCallPlan::internal(operation, "marker".into()); + plan.planned_root = Some("kernel".into()); + // Zero-byte planned views still conflict with the saved raw view. + let calls = BTreeMap::from([(site.clone(), plan)]); + let error = validate_ordinary_compatibility(&saved, &calls).unwrap_err(); + assert!(error.contains(expected), "{error}"); + } + for operation in [ + LibraryOperation::StorageNew, + LibraryOperation::StorageReconfigure, + LibraryOperation::StorageRelease, + LibraryOperation::Redirect, + ] { + let mut plan = LibraryCallPlan::internal(operation, "marker".into()); + plan.planned_root = Some("kernel".into()); + validate_ordinary_compatibility( + &saved, + &BTreeMap::from([(site.clone(), plan)]), + ) + .unwrap(); + } + let untagged = LibraryCallPlan::internal(LibraryOperation::Native, "raw".into()); + validate_ordinary_compatibility(&saved, &BTreeMap::from([(site, untagged)])) + .unwrap(); + } else { + let error = validate_ordinary_compatibility(&saved, &BTreeMap::new()).unwrap_err(); + assert!(error.contains(expected), "{error}"); + } + let mut roots = BTreeMap::new(); + if planned { + view(ctx, body, 0, Some(16), Some("kernel")); + roots.insert( + "kernel".into(), + KernelLaunchFacts { + exact_block: Some((32, 1, 1)), + dynamic_shared_bytes: 16, + dynamic_shared_alignment: 8, + }, + ); + } + let error = complete_planned(ctx, module, saved, &roots).unwrap_err(); + assert!( + error.contains(if planned { + "planned storage cannot include an unconstrained" + } else { + "inferred dynamic shared memory cannot include unconstrained" + }), + "{error}" + ); + } + } +} diff --git a/cuda-oxide/crates/mir-importer/src/lib.rs b/cuda-oxide/crates/mir-importer/src/lib.rs index 796f34e5cc..d4c912516e 100644 --- a/cuda-oxide/crates/mir-importer/src/lib.rs +++ b/cuda-oxide/crates/mir-importer/src/lib.rs @@ -95,6 +95,7 @@ extern crate rustc_target; pub const DEVICE_RUNTIME_CHECKS_VALUE: bool = false; pub mod error; +mod kernel_resources; pub mod pipeline; // Crate-internal: every caller reaches the translator through `pipeline`, and // the handful of predicates outsiders need are re-exported below. Keeping the @@ -114,4 +115,18 @@ pub use pipeline::{ pub use translator::cute::is_cute_compiler_hook_path; pub use translator::facts::KnownDefs; pub use translator::terminator::drop_glue::{drop_glue_is_noop, drop_instance_is_noop}; -pub use translator::terminator::is_panic_entry_path; +pub use translator::terminator::{is_dropped_panic_call, is_panic_entry_path}; + +mod library; +pub use dialect_mir::library::{ + LibraryAccess, LibraryCallKind, LibraryCallTargetAttr, LibraryCallbackCall, + LibraryCallbackDefinition, LibraryCallbackDispatch, LibraryCallbackFunction, + LibraryCallbackSources, LibraryFunctionSourceAttr, LibraryNativeMetadata, LibraryOperation, + LibraryReserveLayout, LibraryRoleAttr, LibraryRootArgumentClass, LibraryScalar, + LibraryStorageArg, LibraryStoragePolicy, LibraryWitness, NativeSpec, +}; +pub use library::{CallTargetRedirects, LibraryCallMetadata, LibraryCallPlan, LibraryCallSite}; +pub use pipeline::{ImportedModule, PreparedImport, import_pipeline_with_libraries}; +pub use translator::body::{checked_collective_successors, kernel_exact_block}; + +pub use mir_transforms::library::RootedNativePlans; diff --git a/cuda-oxide/crates/mir-importer/src/library.rs b/cuda-oxide/crates/mir-importer/src/library.rs new file mode 100644 index 0000000000..32cd215ddc --- /dev/null +++ b/cuda-oxide/crates/mir-importer/src/library.rs @@ -0,0 +1,385 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Retain authenticated native and storage operations in MIR. + +use std::cell::RefCell; +use std::collections::BTreeMap; + +use crate::error::{TranslationErr, TranslationResult}; +use crate::translator::terminator::helpers; +use crate::translator::values::ValueMap; +use crate::translator::{rvalue, types}; +pub use dialect_mir::library::{LibraryCallPlan, LibraryCallSite, LibraryCallTargetAttr}; +use dialect_mir::library::{ + LibraryNativeMetadata, LibraryRoleAttr, LibrarySourceEffectAttr, LibraryWriteSourceAttr, +}; +use dialect_mir::ops::{MirLibraryOp, MirUndefOp}; +use dialect_mir::types::MirTupleType; +use pliron::basic_block::BasicBlock; +use pliron::builtin::types::FunctionType; +use pliron::context::{Context, Ptr}; +use pliron::input_err; +use pliron::location::Location; +use pliron::op::Op; +use pliron::operation::Operation; +use rustc_public::mir; + +/// Internal call targets, keyed by emitted caller and original collected callee +/// export name. Values name the same Rust body in that caller's root context. +pub type CallTargetRedirects = BTreeMap>; + +/// Source semantics authenticated from the genuine definition and concrete types. +/// +/// These inputs are independent from finalized provider plan flags. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LibraryCallMetadata { + pub marker: String, + pub role: LibraryRoleAttr, +} + +#[derive(Default)] +struct SourceCallContext { + effect: Option, + destination_write: Option, +} + +thread_local! { + static SOURCE_CALL: RefCell = const { RefCell::new(SourceCallContext { effect: None, destination_write: None }) }; + static SELECTED_CALL_TARGETS: RefCell> = const { RefCell::new(BTreeMap::new()) }; + static SEMANTICS: RefCell> = const { RefCell::new(BTreeMap::new()) }; +} + +/// Import-only lexical context for the one source call currently expanding. +/// The retained event observes the existing producer before its destination store. +pub(crate) struct SourceEffectGuard(SourceCallContext); +impl SourceEffectGuard { + pub(crate) fn new( + effect: Option, + destination_write: Option, + ) -> Self { + Self(SOURCE_CALL.with(|current| { + current.replace(SourceCallContext { + effect, + destination_write, + }) + })) + } + pub(crate) fn finish(&self, ctx: &mut Context, continuation: Ptr) { + if let Some(effect) = SOURCE_CALL.with(|current| current.borrow_mut().effect.take()) { + let marker = dialect_mir::ops::MirLibrarySourceOp::build(ctx, effect, None); + let event_loc = continuation.deref(ctx).loc().clone(); + marker.deref_mut(ctx).set_loc(event_loc); + marker.insert_before(ctx, continuation); + } + } +} +impl Drop for SourceEffectGuard { + fn drop(&mut self) { + SOURCE_CALL.with(|current| *current.borrow_mut() = std::mem::take(&mut self.0)); + } +} + +/// Capture the current call's source write independently of its result effect. +pub(crate) fn current_destination_write_source() -> Option { + SOURCE_CALL.with(|current| current.borrow().destination_write.clone()) +} + +pub(crate) fn retain_source_result( + ctx: &mut Context, + value: pliron::value::Value, + previous: Ptr, + loc: Location, +) -> Ptr { + let Some(effect) = SOURCE_CALL.with(|current| current.borrow_mut().effect.take()) else { + return previous; + }; + // Checked participation rejects inline assembly independently of its result. + // Do not add a use that blocks forwarding its compiler-owned result bundle. + let result = (!matches!( + effect.kind, + dialect_mir::library::LibrarySourceEffectKind::InlineAssembly + )) + .then_some(value); + let marker = dialect_mir::ops::MirLibrarySourceOp::build(ctx, effect, result); + marker.deref_mut(ctx).set_loc(loc); + marker.insert_after(ctx, previous); + marker +} + +pub(crate) fn set_selected_call_targets(targets: BTreeMap) { + SELECTED_CALL_TARGETS.with(|current| *current.borrow_mut() = targets); +} + +pub(crate) fn get_selected_call_target(site: &LibraryCallSite) -> Option { + SELECTED_CALL_TARGETS.with(|current| current.borrow().get(site).cloned()) +} + +pub(crate) fn attach_call_source(ctx: &mut Context, op: Ptr, site: LibraryCallSite) { + let call = dialect_mir::ops::MirCallOp::new(op); + if let Some(target) = get_selected_call_target(&site) { + call.set_attr_library_call_target(ctx, target); + } + call.set_attr_library_source_site(ctx, site); +} + +pub(crate) fn set_semantics(semantics: BTreeMap) { + SEMANTICS.with(|current| *current.borrow_mut() = semantics); +} + +pub(crate) fn get_semantics(site: &LibraryCallSite) -> Option { + SEMANTICS.with(|calls| calls.borrow().get(site).cloned()) +} + +/// Runtime argument positions in the genuine source marker, before ABI lowering. +fn runtime_indices(role: LibraryRoleAttr) -> &'static [usize] { + use LibraryNativeMetadata as Native; + match role { + LibraryRoleAttr::Native(Native::Raw { .. }) => &[1], + LibraryRoleAttr::Native(Native::Callback { .. }) => &[1, 3], + LibraryRoleAttr::Native(Native::CheckedBlock { .. }) => &[1, 2], + LibraryRoleAttr::Native(Native::CheckedBlockResource { .. }) => &[1, 2, 3], + LibraryRoleAttr::Native(Native::CheckedBlockStorage { .. }) => &[1, 2, 3, 4], + LibraryRoleAttr::StorageNew(_) => &[], + LibraryRoleAttr::StorageReconfigure { .. } + | LibraryRoleAttr::StorageReserve { .. } + | LibraryRoleAttr::StorageRelease => &[0], + LibraryRoleAttr::NativeOperandBoundary => unreachable!("boundary is importer-owned"), + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn translate_call( + ctx: &mut Context, + body: &mir::Body, + metadata: &LibraryCallMetadata, + site: LibraryCallSite, + instance: rustc_public::mir::mono::Instance, + args: &[mir::Operand], + destination: &mir::Place, + target: &Option, + block: Ptr, + previous: Option>, + values: &mut ValueMap, + blocks: &[Ptr], + loc: Location, +) -> TranslationResult> { + use rustc_public::ty::{Abi, RigidTy, TyKind}; + let signature = instance + .ty() + .kind() + .fn_sig() + .ok_or_else(|| { + pliron::input_error!( + loc.clone(), + TranslationErr::invalid_op("library marker has no resolved signature") + ) + })? + .skip_binder(); + if instance.mangled_name() != metadata.marker + || signature.abi != Abi::Rust + || signature.c_variadic + || signature.inputs().len() != args.len() + { + return input_err!( + loc, + TranslationErr::invalid_op("library marker signature changed after authentication") + ); + } + let mut previous = previous; + if matches!(metadata.role, LibraryRoleAttr::Native(_)) { + // The finalized before-reuse barrier must precede source operand loads, + // not merely the eventual native call. This position has no completion + // authority of its own; the late emitter consumes it with the call plan. + let boundary = MirLibraryOp::build( + ctx, + LibraryRoleAttr::NativeOperandBoundary, + site.clone(), + metadata.marker.clone(), + ); + boundary.deref_mut(ctx).set_loc(loc.clone()); + helpers::insert_op(ctx, boundary, block, previous); + previous = Some(boundary); + } + let indices = runtime_indices(metadata.role); + let mut operands = Vec::with_capacity(indices.len()); + let mut declared = Vec::with_capacity(indices.len()); + for &index in indices { + let (Some(operand), Some(ty)) = (args.get(index), signature.inputs().get(index)) else { + return input_err!( + loc, + TranslationErr::invalid_op( + "library marker operand count changed after authentication" + ) + ); + }; + declared.push(types::translate_type(ctx, ty)?); + let (value, last) = + rvalue::translate_operand(ctx, body, operand, values, block, previous, loc.clone())?; + operands.push(value); + previous = last; + } + let unit = matches!(signature.output().kind(), TyKind::RigidTy(RigidTy::Tuple(fields)) if fields.is_empty()); + let results = if unit { + vec![] + } else { + vec![types::translate_type(ctx, &signature.output())?] + }; + let signature = FunctionType::get(ctx, declared, results); + let (prepared, previous) = + if target.is_none() && matches!(metadata.role, LibraryRoleAttr::Native(_)) { + (None, previous) + } else { + let (prepared, previous) = helpers::prepare_destination_write( + ctx, + body, + destination, + values, + block, + previous, + loc.clone(), + )?; + (Some(prepared), previous) + }; + let retained = MirLibraryOp::build_call( + ctx, + metadata.role, + site, + metadata.marker.clone(), + signature, + operands, + ); + retained.deref_mut(ctx).set_loc(loc.clone()); + helpers::insert_op(ctx, retained, block, previous); + let Some(prepared) = prepared else { + return Ok(crate::translator::terminator::emit_unreachable_after( + ctx, + block, + Some(retained), + loc, + )); + }; + let (result, previous) = if unit { + let ty = MirTupleType::get(ctx, vec![]).into(); + let op = MirUndefOp::new(ctx, ty).get_operation(); + helpers::insert_op(ctx, op, block, Some(retained)); + (op.deref(ctx).get_result(0), op) + } else { + (retained.deref(ctx).get_result(0), retained) + }; + helpers::emit_prepared_result_and_goto( + ctx, + prepared, + result, + target, + block, + previous, + values, + blocks, + loc, + "library marker requires a continuation", + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use dialect_mir::library::{LibrarySourceEffectKind, LibraryWriteKind}; + use dialect_mir::ops::{MirLibrarySourceOp, MirReturnOp}; + use pliron::builtin::types::{IntegerType, Signedness}; + use pliron::linked_list::ContainsLinkedList; + + #[test] + fn source_call_guard_restores_independent_effect_and_destination() { + let site = LibraryCallSite { + caller: "source".to_string(), + block: 3, + }; + let outer_effect = LibrarySourceEffectAttr { + site: site.clone(), + kind: LibrarySourceEffectKind::UnitHint, + }; + let inner_effect = LibrarySourceEffectAttr { + site: site.clone(), + kind: LibrarySourceEffectKind::BlockQuery, + }; + let write = LibraryWriteSourceAttr { + site, + statement: 2, + kind: LibraryWriteKind::Indirect { + nominal_protected: false, + }, + }; + let mut ctx = Context::new(); + crate::translator::register_dialects(&mut ctx); + let ty = IntegerType::get(&ctx, 32, Signedness::Unsigned).into(); + let block = BasicBlock::new(&mut ctx, None, vec![]); + let producer = Operation::new( + &mut ctx, + MirUndefOp::get_concrete_op_info(), + vec![ty], + vec![], + vec![], + 0, + ); + producer.insert_at_back(block, &ctx); + let value = producer.deref(&ctx).get_result(0); + let continuation = Operation::new( + &mut ctx, + MirReturnOp::get_concrete_op_info(), + vec![], + vec![value], + vec![], + 0, + ); + continuation.insert_after(&ctx, producer); + assert!(current_destination_write_source().is_none()); + { + let outer = SourceEffectGuard::new(Some(outer_effect.clone()), Some(write.clone())); + { + let inner = SourceEffectGuard::new(Some(inner_effect.clone()), Some(write.clone())); + let marker = retain_source_result(&mut ctx, value, producer, Location::Unknown); + assert_eq!(marker.deref(&ctx).get_operand(0), value); + assert_eq!( + *MirLibrarySourceOp::new(marker) + .get_attr_library_source_effect(&ctx) + .unwrap(), + inner_effect + ); + assert_eq!(current_destination_write_source(), Some(write.clone())); + assert!(SOURCE_CALL.with(|current| current.borrow().effect.is_none())); + inner.finish(&mut ctx, continuation); + assert_eq!(current_destination_write_source(), Some(write.clone())); + } + assert_eq!( + SOURCE_CALL.with(|current| current.borrow().effect.clone()), + Some(outer_effect.clone()) + ); + let early_return = || -> Result<(), ()> { + let _inner = SourceEffectGuard::new(None, None); + Err(())?; + Ok(()) + }; + assert!(early_return().is_err()); + assert_eq!(current_destination_write_source(), Some(write)); + assert_eq!( + SOURCE_CALL.with(|current| current.borrow().effect.clone()), + Some(outer_effect) + ); + outer.finish(&mut ctx, continuation); + assert!(SOURCE_CALL.with(|current| current.borrow().effect.is_none())); + assert!(current_destination_write_source().is_some()); + } + assert!(current_destination_write_source().is_none()); + assert!(SOURCE_CALL.with(|current| current.borrow().effect.is_none())); + assert_eq!( + block + .deref(&ctx) + .iter(&ctx) + .filter(|&op| Operation::get_op::(op, &ctx).is_some()) + .count(), + 2, + "result and unit effects are each retained once" + ); + } +} diff --git a/cuda-oxide/crates/mir-importer/src/pipeline.rs b/cuda-oxide/crates/mir-importer/src/pipeline.rs index 9b587d233e..b9bdead845 100644 --- a/cuda-oxide/crates/mir-importer/src/pipeline.rs +++ b/cuda-oxide/crates/mir-importer/src/pipeline.rs @@ -11,12 +11,13 @@ //! # Pipeline Steps //! //! ```text -//! MIR -> dialect-mir -> verify -> mem2reg -> annotated loop unroll -//! -> LLVM dialect -> LLVM IR -> PTX +//! MIR -> dialect-mir -> verify -> prepare -> library proof/provider binding +//! -> optional transforms -> LLVM dialect -> LLVM IR -> PTX //! ``` //! -//! Builds with variable debug information skip `mem2reg` and loop unrolling so -//! source variables remain in stable stack slots. +//! Normal preparation promotes locals with `mem2reg`. Full variable debug keeps +//! stack slots and skips optional transforms; the required library proof uses +//! those slots at the same preparation boundary. //! //! # GPU Target Selection //! @@ -39,7 +40,8 @@ use cuda_oxide_codegen::__private::{ BackendOptions, DeviceBackend, ModuleArtifactKind, ModulePipelineRequest, OutputFiles, - PipelineTrace, append_to_module, compile_translated_module, verify_operation, + PipelineTrace, PreparedModuleStage, append_to_module, prepare_translated_module, + verify_operation, }; pub use cuda_oxide_codegen::__private::{ DeviceArchHint, DeviceExternAttrs, DeviceExternDecl, PipelineError, @@ -57,8 +59,8 @@ use pliron::printable::Printable; use pliron::r#type::Typed; use rustc_public::mir::mono::Instance; use rustc_public::ty::Ty; -use std::collections::BTreeMap; -use std::path::Path; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; fn stderr_pipeline_trace(message: &str) { eprintln!("{message}"); @@ -83,6 +85,10 @@ pub struct CollectedFunction { pub is_kernel: bool, /// The name to export in PTX. For kernels, this is the user-visible name. pub export_name: String, + /// Original source identity used by the retained library analyses. + pub library_source: Option, + /// Authenticated classes in original kernel argument order. + pub library_root_arguments: Vec, /// rustc MIR source-scope data used to build inlined debug scopes. pub debug_source_scopes: Option, /// rustc's statement-level debug assignments, aligned with stable MIR. @@ -181,9 +187,12 @@ pub struct CompilationResult { pub allow_fma_contraction: bool, /// Per-kernel source launch bounds preserved for post-link diagnostics. pub kernel_launch_bounds: BTreeMap, + /// Finalized resource requirements, keyed by concrete exported kernel name. + pub kernel_launch_facts: BTreeMap, } /// Configuration for the compilation pipeline. +#[derive(Clone)] pub struct PipelineConfig { /// Directory for output files (`.ll`, `.ptx`). pub output_dir: std::path::PathBuf, @@ -459,6 +468,21 @@ fn backend_options_for(config: &PipelineConfig) -> BackendOptions { backend_options } +/// Select single-block-cluster semantics only for a known pre-sm90 target. +/// Invalid inputs are still diagnosed by the shared backend validator. +fn cluster_query_fallback_target(options: &BackendOptions) -> Option { + if let Some(target) = &options.target_arch { + let target = cuda_oxide_codegen::experimental::Target::parse(target).ok()?; + return (target.capability() < 90).then(|| target.sm()); + } + match &options.device_arch_hint { + Some(DeviceArchHint::Valid(arch)) if arch.as_ref().capability() < 90 => { + Some(arch.as_ref().sm()) + } + _ => None, + } +} + /// Runs the full compilation pipeline on collected functions. /// /// # Pipeline Steps @@ -501,6 +525,506 @@ pub fn run_pipeline( config: &PipelineConfig, known_defs: crate::translator::facts::KnownDefs, ) -> Result { + import_pipeline_with_libraries( + functions, + device_externs, + config, + known_defs, + BTreeMap::new(), + BTreeMap::new(), + BTreeMap::new(), + Default::default(), + )? + .finish() +} + +/// An imported module and the configuration captured for its compilation. +/// +/// The returned state retains no rustc-facing identities. Preparation borrows +/// this owner and takes its completion data once; the paused stage never owns +/// references into itself. +#[doc(hidden)] +#[must_use = "finish the imported module to emit its artifact"] +pub struct ImportedModule { + ctx: Context, + module_op_ptr: pliron::context::Ptr, + config: PipelineConfig, + backend_options: BackendOptions, + device_externs: Vec, + expected_kernels: Vec, + legaliser: Legaliser, + symbols: BTreeMap, + marker_exports: BTreeMap, + source_targets: BTreeMap>, + callback_sources: crate::LibraryCallbackSources, + outputs: ImportOutputs, + completion: Option, +} + +struct ImportOutputs { + llvm_ir: PathBuf, + ptx: PathBuf, + cubin: PathBuf, + stale: Vec, +} + +struct ImportCompletion { + kernel_names: Vec, + kernel_launch_bounds: BTreeMap, + library_roots: BTreeMap, +} + +/// An imported module paused at the mandatory library-analysis boundary. +/// +/// Normal builds have promoted locals; Full debug keeps the original slots. +/// The importer owner must outlive this consuming stage. Its saved ordinary +/// resource constraints survive preparation even when operations are removed. +#[doc(hidden)] +#[must_use = "finish the prepared import to emit its artifact"] +pub struct PreparedImport<'a> { + stage: PreparedModuleStage<'a, 'a>, + config: &'a PipelineConfig, + outputs: &'a ImportOutputs, + completion: ImportCompletion, + ordinary: crate::kernel_resources::OrdinaryResources, + legaliser: &'a mut Legaliser, + symbols: BTreeMap, + marker_exports: BTreeMap, + source_targets: BTreeMap>, + callback_sources: crate::LibraryCallbackSources, + original_roles: BTreeMap, + library_rooted_functions: BTreeMap>, + library_validation: Option, + callback_symbols: BTreeMap, + libraries_bound: bool, + library_calls: BTreeMap, + library_externs: Vec, +} + +impl ImportedModule { + /// Prepare this module once, preserving the ordinary resource constraints. + pub fn prepare(&mut self) -> Result, PipelineError> { + let completion = self.completion.take().ok_or_else(|| { + PipelineError::Translation("this imported module was already prepared".into()) + })?; + let ordinary = crate::kernel_resources::capture_ordinary( + &self.ctx, + self.module_op_ptr, + completion.kernel_names.clone(), + ) + .map_err(PipelineError::Translation)?; + let original_roles = mir_transforms::library::capture_library_roles( + self.module_op_ptr, + &self.ctx, + &self.symbols, + ) + .map_err(|error| PipelineError::Translation(error.disp(&self.ctx).to_string()))?; + let request = ModulePipelineRequest::for_rust_pipeline( + &self.device_externs, + self.config.emit_nvvm_ir, + &self.backend_options, + self.config.debug_kind, + match &self.config.device_backend { + DeviceBackend::Native => None, + DeviceBackend::CutlassMlir(config) => Some(config), + }, + &self.expected_kernels, + OutputFiles { + llvm_ir: &self.outputs.llvm_ir, + ptx: &self.outputs.ptx, + cubin: &self.outputs.cubin, + mlir: self.config.mlir_output.as_deref(), + stale_before_export: &self.outputs.stale, + }, + PipelineTrace { + verbose: self.config.verbose, + dump_mir: self.config.show_mir_dialect, + dump_llvm: self.config.show_llvm_dialect, + sink: Some(stderr_pipeline_trace), + }, + ); + let stage = prepare_translated_module(&mut self.ctx, self.module_op_ptr, &request)?; + Ok(PreparedImport { + stage, + config: &self.config, + outputs: &self.outputs, + completion, + ordinary, + legaliser: &mut self.legaliser, + symbols: std::mem::take(&mut self.symbols), + marker_exports: std::mem::take(&mut self.marker_exports), + source_targets: std::mem::take(&mut self.source_targets), + callback_sources: std::mem::take(&mut self.callback_sources), + callback_symbols: BTreeMap::new(), + original_roles, + library_rooted_functions: BTreeMap::new(), + library_validation: None, + libraries_bound: false, + library_calls: BTreeMap::new(), + library_externs: Vec::new(), + }) + } + + /// Prepare, lower library operations and emit the captured compilation. + pub fn finish(mut self) -> Result { + self.prepare()?.finish() + } +} + +impl PreparedImport<'_> { + /// Admit source-root geometry before native provider or layout work. + pub fn admit_library_geometry( + &mut self, + rooted_functions: &BTreeMap>, + ) -> Result, PipelineError> { + let (ctx, module, _) = self.stage.ir(); + mir_transforms::library::admit_library_roots( + module, + ctx, + rooted_functions, + &self.symbols, + &self.original_roles, + ) + .map_err(|error| PipelineError::Translation(error.disp(ctx).to_string())) + } + + /// Prove the prepared library program before executing its native provider. + pub fn validate_libraries( + &mut self, + rooted_functions: &BTreeMap>, + ) -> Result, PipelineError> { + if self.library_validation.is_some() || self.libraries_bound { + return Err(PipelineError::Translation( + "this prepared import already has its library proof".into(), + )); + } + let shapes = self.admit_library_geometry(rooted_functions)?; + let (ctx, module, analyses) = self.stage.ir(); + let callback_symbols = mir_transforms::library::admit_library_callbacks( + module, + ctx, + &self.callback_sources, + &self.symbols, + ) + .map_err(PipelineError::Translation)?; + let retained = callback_symbols.values().cloned().collect::>(); + let module_op = Operation::get_op::(module, ctx) + .expect("imported module"); + llvm_export::export::retain_functions(ctx, &module_op, &retained); + self.callback_symbols = callback_symbols + .into_iter() + .map(|(identity, symbol)| { + ( + identity, + llvm_export::export::exported_function_name(&symbol), + ) + }) + .collect(); + let validation = mir_transforms::library::analyze_libraries( + module, + ctx, + analyses, + rooted_functions, + &self.symbols, + &self.original_roles, + &shapes, + None, + ) + .map_err(|error| PipelineError::Translation(error.disp(ctx).to_string()))?; + self.library_rooted_functions = rooted_functions.clone(); + self.library_validation = Some(validation); + Ok(shapes) + } + + /// Actual emitted thunk names available only after callback admission. + pub fn callback_symbols(&self) -> &BTreeMap { + &self.callback_symbols + } + + /// Join provider layouts with the proof, then bind the final shared plan. + pub fn finish_library_planning( + &mut self, + native_plans: &mir_transforms::library::RootedNativePlans, + externs: Vec, + roots: BTreeMap, + ) -> Result<(), PipelineError> { + let validation = self.library_validation.take().ok_or_else(|| { + PipelineError::Translation("native library planning requires its source proof".into()) + })?; + let mut plans = + mir_transforms::library::allocate_library_storage(&validation, native_plans) + .map_err(PipelineError::Translation)?; + let (ctx, module, analyses) = self.stage.ir(); + mir_transforms::library::analyze_libraries( + module, + ctx, + analyses, + &self.library_rooted_functions, + &self.symbols, + &self.original_roles, + &validation.shapes, + Some(&mut plans), + ) + .map_err(|error| PipelineError::Translation(error.disp(ctx).to_string()))?; + let assembly = mir_transforms::library::assemble_library_plans( + module, + ctx, + &self.library_rooted_functions, + &self.source_targets, + &validation, + &plans, + native_plans, + roots, + ) + .map_err(PipelineError::Translation)?; + self.bind_libraries( + assembly.contexts, + assembly.calls, + assembly.redirects, + externs, + assembly.roots, + ) + } + + /// Bind the actual provider/layout result and materialize its helper contexts. + /// Source names and physical symbols remain distinct throughout this step. + fn bind_libraries( + &mut self, + contexts: BTreeMap, + mut calls: BTreeMap, + redirects: crate::CallTargetRedirects, + externs: Vec, + roots: BTreeMap, + ) -> Result<(), PipelineError> { + if self.libraries_bound { + return Err(PipelineError::Translation( + "this prepared import already has its library plan".into(), + )); + } + for name in contexts.keys() { + self.symbols + .entry(name.clone()) + .or_insert_with(|| self.legaliser.legalise(name).to_string()); + } + // Resource tags use the physical module names. Source sites and + // authenticated redirect identities keep their raw-name conventions. + for call in calls.values_mut() { + if let Some(root) = &mut call.planned_root { + *root = self.symbols.get(root).cloned().ok_or_else(|| { + PipelineError::Translation("planned root has no imported symbol".into()) + })?; + } + } + let physical_contexts = contexts + .iter() + .map(|(emitted, source)| { + let emitted = self.symbols.get(emitted).cloned(); + let source = self.symbols.get(source).cloned(); + match (emitted, source) { + (Some(emitted), Some(source)) => Ok((emitted, source)), + _ => Err(PipelineError::Translation( + "library context has no imported source symbol".into(), + )), + } + }) + .collect::, _>>()?; + let materialize = !contexts.is_empty() + || !redirects.is_empty() + || calls + .values() + .any(|plan| plan.operation == crate::LibraryOperation::Redirect); + if materialize { + let raw_names: BTreeMap<_, _> = self + .symbols + .iter() + .map(|(raw, symbol)| (symbol.clone(), raw.clone())) + .collect(); + crate::kernel_resources::bind_ordinary_contexts( + &mut self.ordinary, + &physical_contexts, + |site, caller, callee| { + let raw_caller = raw_names + .get(caller) + .ok_or("saved resource caller has no original name")?; + mir_transforms::library::resolve_library_call_target( + site, + raw_caller, + callee, + &self.symbols, + &self.source_targets, + &self.marker_exports, + &calls, + &redirects, + ) + }, + ) + .map_err(PipelineError::Translation)?; + } + crate::kernel_resources::validate_ordinary_compatibility(&self.ordinary, &calls) + .map_err(PipelineError::Translation)?; + let (ctx, module, analyses) = self.stage.ir(); + if materialize { + mir_transforms::library::materialize_library_contexts( + module, + ctx, + analyses, + &contexts, + &self.symbols, + &self.source_targets, + &self.marker_exports, + &calls, + &redirects, + ) + .map_err(|error| PipelineError::Translation(error.disp(ctx).to_string()))?; + } + let physical_roots = roots + .into_iter() + .map(|(root, facts)| { + self.symbols + .get(&root) + .cloned() + .map(|symbol| (symbol, facts)) + .ok_or_else(|| { + PipelineError::Translation("library root has no imported symbol".into()) + }) + }) + .collect::, _>>()?; + let functions: Vec<_> = module + .deref(ctx) + .regions() + .flat_map(|region| { + region + .deref(ctx) + .iter(ctx) + .flat_map(|block| block.deref(ctx).iter(ctx)) + }) + .filter_map(|op| dialect_mir::ops::MirFuncOp::wrap(ctx, op)) + .collect(); + for function in functions { + let op = function.get_operation(); + let symbol = function.get_symbol_name(ctx).to_string(); + if let Some(plan) = physical_roots.get(&symbol) { + crate::kernel_resources::apply_library_block(ctx, op, plan) + .map_err(PipelineError::Translation)?; + } + if self.config.debug_kind.variables_enabled() { + // The clone keeps static identities and source scopes. Only + // the containing emitted function's linkage identity changes. + attach_debug_global_variables(ctx, op, &self.config.debug_global_variables); + } + } + self.library_calls = calls; + self.library_externs = externs; + self.completion.library_roots = physical_roots; + self.libraries_bound = true; + Ok(()) + } + + /// Lower the validated library plan and finalize resources exactly once. + pub fn finish(self) -> Result { + if !self.libraries_bound && !self.original_roles.is_empty() { + return Err(PipelineError::Translation( + "native library operations require a finalized library plan".into(), + )); + } + if !self.libraries_bound { + // Direct callers have no native plan to bind. Library callers + // validate the same saved ordinary facts during binding. + crate::kernel_resources::validate_ordinary_compatibility( + &self.ordinary, + &self.library_calls, + ) + .map_err(PipelineError::Translation)?; + } + let Self { + stage, + config, + outputs, + completion, + ordinary, + library_calls, + library_externs, + .. + } = self; + let mut stage = stage.bind_device_externs(library_externs)?; + let (ctx, module, analyses) = stage.ir(); + mir_transforms::library::lower_planned_library_operations( + module, + ctx, + analyses, + &library_calls, + ) + .map_err(|error| PipelineError::Translation(error.disp(ctx).to_string()))?; + let kernel_launch_facts = crate::kernel_resources::complete_planned( + ctx, + module, + ordinary, + &completion.library_roots, + ) + .map_err(PipelineError::Translation)?; + // Resource relocation changes MIR addresses and root attributes. + let mut invalidated = pliron::pass::PassResult::default(); + invalidated.ir_changed = pliron::irbuild::IRStatus::Changed; + analyses.retain_preserved(&invalidated); + let generated = stage.finish()?; + let artifact_kind = match generated.artifact_kind { + ModuleArtifactKind::NvvmIr => { + write_nvvm_compile_options_sidecar( + &config.output_dir, + &config.output_name, + config.allow_fma_contraction, + config.debug_kind, + )?; + // The target is the completion record for the options sidecar. + write_nvvm_target_sidecar( + &config.output_dir, + &config.output_name, + &generated.target, + )?; + CompilationArtifactKind::NvvmIr + } + ModuleArtifactKind::Ptx => CompilationArtifactKind::Ptx, + ModuleArtifactKind::Cubin => CompilationArtifactKind::Cubin, + }; + Ok(CompilationResult { + artifact_path: match generated.artifact_kind { + ModuleArtifactKind::NvvmIr => outputs.llvm_ir.clone(), + ModuleArtifactKind::Ptx => outputs.ptx.clone(), + ModuleArtifactKind::Cubin => outputs.cubin.clone(), + }, + artifact_kind, + ll_path: outputs.llvm_ir.clone(), + ptx_path: outputs.ptx.clone(), + target: generated.target, + allow_fma_contraction: config.allow_fma_contraction, + kernel_launch_bounds: completion.kernel_launch_bounds, + kernel_launch_facts, + }) + } +} + +/// Import and verify functions without starting resource finalization or codegen. +/// +/// Rustc identities are used only during import. The returned module owns its +/// context and snapshots configuration, authenticated typed operations and +/// original call identities. Provider plans are bound after the prepared-MIR proof. +#[doc(hidden)] +#[allow(clippy::too_many_arguments)] +pub fn import_pipeline_with_libraries( + functions: &[CollectedFunction], + device_externs: &[DeviceExternDecl], + config: &PipelineConfig, + known_defs: crate::translator::facts::KnownDefs, + library_semantics: BTreeMap, + source_targets: BTreeMap>, + selected_call_targets: BTreeMap< + crate::LibraryCallSite, + dialect_mir::library::LibraryCallTargetAttr, + >, + callback_sources: crate::LibraryCallbackSources, +) -> Result { + crate::library::set_semantics(library_semantics); + crate::library::set_selected_call_targets(selected_call_targets); // Install the driver-resolved lang-item ids and emitted-function identities // for this run. Replace (rather than merge) both thread-local sets so // stale state from a previous run on this thread cannot survive. @@ -514,6 +1038,10 @@ pub fn run_pipeline( prepare_output_dir(&config.output_dir)?; + let mut backend_options = backend_options_for(config); + let cluster_query_target = cluster_query_fallback_target(&backend_options); + let mut used_cluster_query_fallback = false; + let mut ctx = Context::new(); // Step 1: Register dialects @@ -529,6 +1057,8 @@ pub fn run_pipeline( let module_op_ptr = module.get_operation(); let mut legaliser = Legaliser::default(); + let mut symbols = BTreeMap::new(); + let mut marker_exports = BTreeMap::new(); let mut kernel_launch_bounds = BTreeMap::new(); let mut expected_kernels = Vec::new(); @@ -551,7 +1081,32 @@ pub fn run_pipeline( .body() .ok_or_else(|| PipelineError::NoBody(func.export_name.clone()))?; - let func_op_ptr = crate::translator::body::translate_body( + if let Some(source) = &func.library_source { + if source.export != func.export_name || source.mangled != func.instance.mangled_name() { + return Err(PipelineError::Translation(format!( + "{}: library source identity does not match the imported function", + func.export_name + ))); + } + let expected_arguments = if func.is_kernel { + body.arg_locals().len() + } else { + 0 + }; + if func.library_root_arguments.len() != expected_arguments { + return Err(PipelineError::Translation(format!( + "{}: library argument classes do not match the imported root arguments", + func.export_name + ))); + } + } else if !func.library_root_arguments.is_empty() { + return Err(PipelineError::Translation(format!( + "{}: library argument classes require their source identity", + func.export_name + ))); + } + + let (func_op_ptr, used_fallback) = crate::translator::body::translate_body( &mut ctx, &body, &func.instance, @@ -564,11 +1119,28 @@ pub fn run_pipeline( config.debug_kind, func.debug_source_scopes.as_ref(), func.statement_debug_info.as_ref(), + cluster_query_target.is_some(), ) .map_err(|e| { // Use .disp(&ctx) for rich error formatting with location and backtrace PipelineError::Translation(format!("{}: {}", func.export_name, e.disp(&ctx))) })?; + used_cluster_query_fallback |= used_fallback; + + let imported_function = Operation::get_op::(func_op_ptr, &ctx) + .expect("body translation produces a MIR function"); + if let Some(source) = &func.library_source { + imported_function.set_library_function_source(&mut ctx, source.clone()); + } + for (index, &class) in func.library_root_arguments.iter().enumerate() { + imported_function.set_library_root_argument(&mut ctx, index, class); + } + + symbols.insert( + func.export_name.clone(), + legaliser.legalise(&func.export_name).to_string(), + ); + marker_exports.insert(func.instance.mangled_name(), func.export_name.clone()); if config.debug_kind.variables_enabled() { attach_debug_global_variables(&mut ctx, func_op_ptr, &config.debug_global_variables); @@ -601,83 +1173,61 @@ pub fn run_pipeline( append_to_module(&ctx, module_op_ptr, func_op_ptr); } - let ll_path = config.output_dir.join(format!("{}.ll", config.output_name)); - let ptx_path = config - .output_dir - .join(format!("{}.ptx", config.output_name)); - let cubin_path = config - .output_dir - .join(format!("{}.cubin", config.output_name)); - let stale_artifacts = stale_compilation_artifact_paths(&config.output_dir, &config.output_name); - - let backend_options = backend_options_for(config); - - let request = ModulePipelineRequest::for_rust_pipeline( - device_externs, - config.emit_nvvm_ir, - &backend_options, - config.debug_kind, - match &config.device_backend { - DeviceBackend::Native => None, - DeviceBackend::CutlassMlir(config) => Some(config), - }, - &expected_kernels, - OutputFiles { - llvm_ir: &ll_path, - ptx: &ptx_path, - cubin: &cubin_path, - mlir: config.mlir_output.as_deref(), - stale_before_export: &stale_artifacts, - }, - PipelineTrace { - verbose: config.verbose, - dump_mir: config.show_mir_dialect, - dump_llvm: config.show_llvm_dialect, - sink: Some(stderr_pipeline_trace), - }, + // Ordinary Rust calls cross from shared-address-space values to generic + // pointer parameters. Use the imported callee's real signature, retaining + // all unsupported mismatches for the normal module verifier. + crate::translator::terminator::helpers::normalize_internal_shared_pointer_arguments( + &mut ctx, + module_op_ptr, ); - let generated = compile_translated_module(&mut ctx, module_op_ptr, &request)?; - - match generated.artifact_kind { - ModuleArtifactKind::NvvmIr => { - write_nvvm_compile_options_sidecar( - &config.output_dir, - &config.output_name, - config.allow_fma_contraction, - config.debug_kind, - )?; - // Publish the target last: its version marker is the completion record - // that says the sibling options file is required. - write_nvvm_target_sidecar(&config.output_dir, &config.output_name, &generated.target)?; - Ok(CompilationResult { - artifact_path: ll_path.clone(), - artifact_kind: CompilationArtifactKind::NvvmIr, - ll_path, - ptx_path, - target: generated.target, - allow_fma_contraction: config.allow_fma_contraction, - kernel_launch_bounds, - }) - } - ModuleArtifactKind::Ptx => Ok(CompilationResult { - artifact_path: ptx_path.clone(), - artifact_kind: CompilationArtifactKind::Ptx, - ll_path, - ptx_path, - target: generated.target, - allow_fma_contraction: config.allow_fma_contraction, - kernel_launch_bounds, - }), - ModuleArtifactKind::Cubin => Ok(CompilationResult { - artifact_path: cubin_path, - artifact_kind: CompilationArtifactKind::Cubin, - ll_path, - ptx_path, - target: generated.target, - allow_fma_contraction: config.allow_fma_contraction, + for declaration in device_externs { + symbols.insert( + declaration.export_name.clone(), + legaliser.legalise(&declaration.export_name).to_string(), + ); + } + + // Cluster fallback has already selected a target-dependent implementation. + // Pin its advisory target before preparing the module. + if used_cluster_query_fallback && backend_options.target_arch.is_none() { + backend_options.target_arch = cluster_query_target; + backend_options.target_arch_source = + "the device hint, pinned by pre-sm90 cluster query fallback"; + } + let outputs = ImportOutputs { + llvm_ir: config.output_dir.join(format!("{}.ll", config.output_name)), + ptx: config + .output_dir + .join(format!("{}.ptx", config.output_name)), + cubin: config + .output_dir + .join(format!("{}.cubin", config.output_name)), + stale: stale_compilation_artifact_paths(&config.output_dir, &config.output_name), + }; + let kernel_names = functions + .iter() + .filter(|function| function.is_kernel) + .map(|function| symbols[&function.export_name].clone()) + .collect(); + Ok(ImportedModule { + ctx, + module_op_ptr, + config: config.clone(), + backend_options, + device_externs: device_externs.to_vec(), + expected_kernels, + legaliser, + symbols, + marker_exports, + source_targets, + callback_sources, + outputs, + completion: Some(ImportCompletion { + kernel_names, kernel_launch_bounds, + library_roots: BTreeMap::new(), }), - } + }) } fn launch_bounds_from_mir_func( @@ -770,7 +1320,9 @@ fn write_nvvm_compile_options_sidecar( }) } -fn stale_compilation_artifact_paths( +/// Paths the pipeline may remove or replace when compiling one module. +/// Callers can preflight these before compilation to protect external inputs. +pub fn stale_compilation_artifact_paths( output_dir: &Path, output_name: &str, ) -> Vec { @@ -796,6 +1348,249 @@ mod tests { use llvm_export::ops::DebugLocalTypeKind; use std::fs; + #[test] + fn cluster_query_fallback_requires_a_known_pre90_target() { + for (explicit, hint, expected) in [ + (None, None, None), + (None, Some("sm_80"), Some("sm_80")), + (None, Some("sm_120"), None), + (None, Some("invalid"), None), + (Some("sm_80"), Some("sm_120"), Some("sm_80")), + (Some("sm_120"), Some("sm_80"), None), + (Some("invalid"), Some("sm_80"), None), + ] { + let mut options = BackendOptions::default(); + options.target_arch = explicit.map(str::to_owned); + options.device_arch_hint = hint.map(|hint| DeviceArchHint::parse(hint.to_owned())); + assert_eq!( + cluster_query_fallback_target(&options).as_deref(), + expected, + "explicit={explicit:?}, hint={hint:?}" + ); + } + } + + #[test] + fn cluster_query_lowering_preserves_target_selection_and_raw_floors() { + const CHILD: &str = "CUDA_OXIDE_TEST_CLUSTER_QUERY_PIPELINE"; + if std::env::var_os(CHILD).is_none() { + // Hint-only cases must not inherit an explicit target. Keep + // environment changes out of the parallel test process. + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "pipeline::tests::cluster_query_lowering_preserves_target_selection_and_raw_floors", + "--nocapture", + ]) + .env(CHILD, "1") + .env_remove("CUDA_OXIDE_TARGET") + .env_remove("CUDA_OXIDE_DEVICE_ARCH") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "cuda_oxide_cluster_queries_{}_{}", + std::process::id(), + unique + )); + fs::create_dir_all(&root).unwrap(); + let fixture = root.join("queries.rs"); + // These non-inlined placeholders reproduce the defining paths at the + // importer boundary. Their bodies are not collected: actual dispatch + // must replace each call, never execute the sentinel value. + fs::write( + &fixture, + r#" +#![allow(dead_code, non_snake_case)] +macro_rules! queries { + ($($name:ident),*) => {$( + #[inline(never)] pub fn $name() -> u32 { 99 } + )*}; +} +mod cluster { + queries!(cluster_ctaidX, cluster_ctaidY, cluster_ctaidZ, + cluster_nctaidX, cluster_nctaidY, cluster_nctaidZ, + __cluster_idxX, __cluster_idxY, __cluster_idxZ, + __cluster_grid_dimX, __cluster_grid_dimY, __cluster_grid_dimZ); +} +mod warp { + #[inline(never)] pub fn elect_sync(_: u32) -> (u32, bool) { (99, false) } +} +mod __cuda_oxide_intrinsic_abi_v1 { queries!(i0266); } +fn compatibility() -> (u32, u32, u32, u32, u32, u32, u32, u32, u32, u32, u32, u32) { + use cluster::*; + (cluster_ctaidX(), cluster_ctaidY(), cluster_ctaidZ(), + cluster_nctaidX(), cluster_nctaidY(), cluster_nctaidZ(), + __cluster_idxX(), __cluster_idxY(), __cluster_idxZ(), + __cluster_grid_dimX(), __cluster_grid_dimY(), __cluster_grid_dimZ()) +} +fn compatibility_and_elect() -> (u32, u32) { + (cluster::cluster_nctaidX(), warp::elect_sync(u32::MAX).0) +} +fn elect_only() -> u32 { warp::elect_sync(u32::MAX).0 } +fn raw() -> u32 { __cuda_oxide_intrinsic_abi_v1::i0266() } +"#, + ) + .unwrap(); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let sysroot = std::process::Command::new(rustc) + .args(["--print", "sysroot"]) + .output() + .unwrap(); + assert!(sysroot.status.success()); + let sysroot = String::from_utf8(sysroot.stdout).unwrap(); + + for crate_name in ["cuda_device", "cuda_intrinsics"] { + let args = vec![ + "rustc".to_string(), + "--edition=2024".to_string(), + "--crate-type=rlib".to_string(), + format!("--crate-name={crate_name}"), + "--emit=metadata".to_string(), + "-Zmir-opt-level=0".to_string(), + format!("--out-dir={}", root.display()), + format!("--sysroot={}", sysroot.trim()), + fixture.display().to_string(), + ]; + let output_dir = root.join(crate_name); + std::thread::Builder::new() + .stack_size(64 * 1024 * 1024) + .spawn(move || { + rustc_public::run!(&args, || { + use rustc_public::CrateDef as _; + + let compile = |name: &str, target: Option<&str>, hint: Option<&str>| { + let item = rustc_public::all_local_items() + .into_iter() + .find(|item| item.name() == format!("{crate_name}::{name}")) + .expect("fixture entry"); + let instance = Instance::try_from(item).unwrap(); + let body = instance.body().unwrap(); + let function = CollectedFunction { + instance, + rustc_mir_block_count: body.blocks.len(), + // These monomorphic, branch-free fixtures have + // no specialization-dependent dead successors. + rustc_mono_successors: body + .blocks + .iter() + .map(|block| block.terminator.successors()) + .collect(), + is_kernel: false, + export_name: name.to_string(), + library_source: None, + library_root_arguments: Vec::new(), + debug_source_scopes: None, + statement_debug_info: None, + inline: None, + }; + let config = PipelineConfig { + output_dir: output_dir.clone(), + output_name: name.to_string(), + verbose: false, + emit_nvvm_ir: true, + target_arch: target.map(str::to_string), + device_arch_hint: hint + .map(|hint| DeviceArchHint::parse(hint.to_string())), + ..PipelineConfig::default() + }; + run_pipeline(&[function], &[], &config, Default::default()) + .map(|result| { + (result.target, fs::read_to_string(result.ll_path).unwrap()) + }) + .map_err(|error| error.to_string()) + }; + if crate_name == "cuda_device" { + for (target, hint) in + [(Some("sm_80"), Some("sm_120")), (None, Some("sm_80"))] + { + let (target, ir) = compile("compatibility", target, hint).unwrap(); + assert_eq!(target, "sm_80"); + for (register_index, register) in + ["ctaid", "nctaid"].iter().enumerate() + { + for (axis_index, axis) in ["x", "y", "z"].iter().enumerate() { + let instruction = format!( + "call i32 @llvm.nvvm.read.ptx.sreg.{register}.{axis}()" + ); + let value = ir + .lines() + .find_map(|line| { + line.contains(&instruction).then(|| { + line.trim().split_once(" = ").unwrap().0 + }) + }) + .expect("compatibility register read"); + let slot = 6 + register_index * 3 + axis_index; + assert!( + ir.lines().any(|line| { + line.contains("insertvalue") + && line + .contains(&format!("i32 {value}, {slot}")) + }), + "{ir}" + ); + } + } + assert!(!ir.contains("sreg.cluster")); + assert!(!ir.contains("sreg.ncluster")); + for index in 0..6 { + let value = u32::from(index >= 3); + assert!( + ir.lines().any(|line| { + line.contains("insertvalue") + && line.contains(&format!("i32 {value}, {index}")) + }), + "{ir}" + ); + } + } + let error = compile("compatibility_and_elect", None, Some("sm_80")) + .expect_err("used fallback must pin the advisory hint"); + assert!(error.contains("sm_80") && error.contains("Sm90"), "{error}"); + let (target, _) = compile("elect_only", None, Some("sm_80")).unwrap(); + assert_eq!(target, "sm_90", "unused fallback must not pin the hint"); + let (target, ir) = compile("compatibility", None, None).unwrap(); + assert_eq!(target, "sm_90"); + assert!( + ir.contains("call i32 @llvm.nvvm.read.ptx.sreg.cluster.nctaid.x()") + ); + } else { + let error = compile("raw", Some("sm_80"), None) + .expect_err("raw intrinsics retain their native target floor"); + assert!( + error.contains("v1:i0266") && error.contains("sm_90"), + "{error}" + ); + let (target, ir) = compile("raw", Some("sm_120"), None).unwrap(); + assert_eq!(target, "sm_120"); + assert!( + ir.contains("call i32 @llvm.nvvm.read.ptx.sreg.cluster.nctaid.x()") + ); + } + std::ops::ControlFlow::<(), ()>::Continue(()) + }) + }) + .unwrap() + .join() + .unwrap() + .expect("in-process fixture compilation succeeds"); + } + fs::remove_dir_all(root).unwrap(); + } + #[test] fn test_pipeline_config_default_values() { let config = PipelineConfig::default(); @@ -1428,6 +2223,81 @@ fn adversarial_export_name() -> u64 { fs::remove_dir_all(&root).expect("clean up temp output dir"); } + #[test] + fn paused_import_owns_emission_settings_and_externs() { + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "cuda_oxide_paused_import_{}_{}", + std::process::id(), + unique + )); + let mut config = PipelineConfig { + output_dir: root.clone(), + output_name: "captured".to_string(), + verbose: false, + emit_nvvm_ir: true, + target_arch: Some("sm_86".to_string()), + allow_fma_contraction: false, + ..PipelineConfig::default() + }; + let mut externs = vec![DeviceExternDecl { + export_name: "captured_extern".to_string(), + param_types: vec![], + return_type: DeviceExternType::Void, + attrs: DeviceExternAttrs::default(), + }]; + let first = import_pipeline_with_libraries( + &[], + &externs, + &config, + Default::default(), + BTreeMap::new(), + BTreeMap::new(), + BTreeMap::new(), + Default::default(), + ) + .unwrap(); + verify_operation(&first.ctx, first.module_op_ptr, "paused import").unwrap(); + assert!(!root.join("captured.ll").exists()); + + // A second import replaces the importer thread-local state. Neither it + // nor later caller changes may alter the first module's emission. + config.output_name = "later".to_string(); + config.target_arch = Some("sm_80".to_string()); + config.allow_fma_contraction = true; + externs.clear(); + let later = run_pipeline(&[], &externs, &config, Default::default()).unwrap(); + drop(config); + drop(externs); + let captured = first.finish().unwrap(); + + assert_eq!(later.target, "sm_80"); + assert_eq!(captured.target, "sm_86"); + assert_eq!(captured.ll_path, root.join("captured.ll")); + assert_eq!(captured.artifact_kind, CompilationArtifactKind::NvvmIr); + assert!(!captured.allow_fma_contraction); + assert!( + fs::read_to_string(&captured.ll_path) + .unwrap() + .contains("declare void @captured_extern()") + ); + assert!( + !fs::read_to_string(&later.ll_path) + .unwrap() + .contains("captured_extern") + ); + let options = fs::read_to_string(root.join("captured.options")).unwrap(); + assert!( + !oxide_artifacts::ArtifactCompileOptions::from_sidecar_text(&options) + .unwrap() + .fma_contraction_enabled() + ); + fs::remove_dir_all(root).unwrap(); + } + #[test] fn nvvm_sidecar_preserves_deferred_debug_policy() { let unique = std::time::SystemTime::now() diff --git a/cuda-oxide/crates/mir-importer/src/translator/block.rs b/cuda-oxide/crates/mir-importer/src/translator/block.rs index 4ea656a61a..3d9d7489c8 100644 --- a/cuda-oxide/crates/mir-importer/src/translator/block.rs +++ b/cuda-oxide/crates/mir-importer/src/translator/block.rs @@ -40,7 +40,7 @@ use rustc_public::mir; /// * `ctx` - Pliron IR context /// * `body` - The full MIR body (needed for local declarations) /// * `mir_block` - The MIR block to translate -/// * `_idx` - Block index (unused, kept for debugging) +/// * `library_site` - Original MIR block in the concrete emitted caller /// * `block_ptr` - Target Pliron IR block (already created) /// * `value_map` - MIR local → alloca slot mapping /// * `block_map` - Block index → Pliron IR block mapping @@ -58,7 +58,7 @@ pub fn translate_block( ctx: &mut Context, body: &mir::Body, mir_block: &mir::BasicBlock, - _idx: usize, + library_site: crate::LibraryCallSite, block_ptr: Ptr, value_map: &mut ValueMap, block_map: &[Ptr], @@ -84,7 +84,18 @@ pub fn translate_block( // the trap aborts the kernel, and the memory state of an aborted kernel is // unspecified, so the store has no defined observer either way. if terminator::is_dropped_panic_call(&mir_block.terminator) { - terminator::emit_dropped_panic_trap(ctx, &mir_block.terminator, block_ptr, prev_op); + let end = + terminator::emit_dropped_panic_trap(ctx, &mir_block.terminator, block_ptr, prev_op); + let marker = dialect_mir::ops::MirLibrarySourceOp::build( + ctx, + dialect_mir::library::LibrarySourceEffectAttr { + site: library_site.clone(), + kind: dialect_mir::library::LibrarySourceEffectKind::Abort, + }, + None, + ); + marker.insert_before(ctx, end); + dialect_mir::library::set_library_source_site(ctx, end, library_site); return Ok(()); } @@ -99,8 +110,16 @@ pub fn translate_block( prev_op, )?; } - let op_ptr = - statement::translate_statement(ctx, body, stmt, value_map, block_ptr, prev_op)?; + let op_ptr = statement::translate_statement( + ctx, + body, + stmt, + value_map, + block_ptr, + prev_op, + &library_site, + statement_index, + )?; prev_op = op_ptr; } @@ -125,6 +144,7 @@ pub fn translate_block( block_map, rustc_mono_successors, legaliser, + library_site, )?; Ok(()) diff --git a/cuda-oxide/crates/mir-importer/src/translator/body.rs b/cuda-oxide/crates/mir-importer/src/translator/body.rs index 2af4b2c883..374a53e346 100644 --- a/cuda-oxide/crates/mir-importer/src/translator/body.rs +++ b/cuda-oxide/crates/mir-importer/src/translator/body.rs @@ -392,6 +392,46 @@ fn detect_contract_block_config( Ok(detected) } +/// Read the same exact root shape used by kernel translation, before library +/// specialization. Validate the internal/public MIR bridge before scanning. +pub fn kernel_exact_block( + instance: &mono::Instance, + block_count: usize, + successors: &[Vec], +) -> Result, String> { + let body = instance.body().ok_or("kernel has no MIR body")?; + validate_monomorphized_successors(&body, block_count, successors)?; + let reachable = compute_reachable_blocks(&body, successors); + Ok(detect_contract_block_config(&body, &reachable)?.map(|shape| [shape.x, shape.y, shape.z])) +} + +/// Device CFG for checked collectives, using exactly the importer's unwind and +/// bounds-check policies. Checked calls may not rely on elided bounds checks. +pub fn checked_collective_successors( + body: &mir::Body, + block_count: usize, + successors: &[Vec], +) -> Result>, String> { + validate_monomorphized_successors(body, block_count, successors)?; + let reachable = compute_reachable_blocks(body, successors); + if detect_unchecked_indexing_config(body, &reachable)? || unchecked_indexing_env_enabled() { + return Err("checked block collectives do not support unchecked indexing; retain runtime bounds checks".into()); + } + Ok(body + .blocks + .iter() + .zip(successors) + .map(|(block, exact)| { + let normal = non_unwind_successors(block); + exact + .iter() + .copied() + .filter(|target| normal.contains(target)) + .collect() + }) + .collect()) +} + /// Rejects an exact block shape that needs more threads than /// `#[launch_bounds]` allows. /// @@ -527,6 +567,27 @@ fn detect_dynamic_shared_alignment( None } +fn has_inferred_dynamic_shared( + body: &mir::Body, + reachable: &std::collections::BTreeSet, +) -> bool { + reachable.iter().any(|&index| { + let mir::TerminatorKind::Call { + func: mir::Operand::Constant(constant), + .. + } = &body.blocks[index].terminator.kind + else { + return false; + }; + let TyKind::RigidTy(RigidTy::FnDef(def_id, _)) = constant.const_.ty().kind() else { + return false; + }; + def_id.krate().name.as_str() == "cuda_device" + && (def_id.name() == "__dynamic_shared_inferred" + || def_id.name().ends_with("::__dynamic_shared_inferred")) + }) +} + /// Return the non-unwind successors of a block's terminator. /// /// [`mir::Terminator::successors`] includes unwind cleanup blocks alongside @@ -1883,6 +1944,10 @@ fn emit_entry_allocas( /// * `inline` - The `#[inline]` request, recorded as [`InlineIntentAttr`] on /// non-kernel functions (`set_inline_intent_attr`) /// * `override_name` - Custom export name (defaults to instance name) +/// * `pre90_cluster_queries` - Use single-block-cluster query semantics +/// +/// Returns the function operation and whether it used a pre-sm90 cluster +/// query fallback. The caller must preserve that target choice. pub fn translate_body( ctx: &mut Context, body: &mir::Body, @@ -1896,7 +1961,8 @@ pub fn translate_body( debug_kind: DebugKind, debug_source_scopes: Option<&DebugSourceScopeMap>, statement_debug_info: Option<&StatementDebugInfoMap>, -) -> TranslationResult> { + pre90_cluster_queries: bool, +) -> TranslationResult<(Ptr, bool)> { if let Some(statement_debug_info) = statement_debug_info { if statement_debug_info.blocks.len() != body.blocks.len() { return input_err_noloc!(TranslationErr::invalid_op(format!( @@ -1935,6 +2001,7 @@ pub fn translate_body( let num_locals = body.locals().len(); let mut value_map = ValueMap::new(num_locals); value_map.set_debug_variables(debug_kind.variables_enabled()); + value_map.pre90_cluster_queries = pre90_cluster_queries; // Resolve the per-body unchecked-indexing policy. Like the dynamic-shared // marker, the `#[kernel(unchecked_indexing)]` marker is scanned on any @@ -2114,6 +2181,7 @@ pub fn translate_body( // Create MirFuncOp and set the function type attribute and symbol name let mir_func_op = MirFuncOp::new(ctx, op_ptr, func_type_attr); + mir_func_op.set_library_unchecked_indexing(ctx, unchecked_indexing); let name_str = if let Some(name) = override_name { name.to_string() @@ -2348,6 +2416,22 @@ pub fn translate_body( } } + if has_inferred_dynamic_shared(body, &reachable) { + use pliron::builtin::{ + attributes::IntegerAttr, + types::{IntegerType, Signedness}, + }; + use pliron::utils::apint::APInt; + let ty = IntegerType::get(ctx, 32, Signedness::Unsigned); + mir_func_op.get_operation().deref_mut(ctx).attributes.set( + crate::kernel_resources::INFERRED_SHARED.try_into().unwrap(), + IntegerAttr::new( + ty, + APInt::from_u32(1, std::num::NonZeroUsize::new(32).unwrap()), + ), + ); + } + // Attribute macros may run before `#[kernel]`. Generic expansion forwards // that marker to the entry but also keeps the original in its helper, so // record markers on any function. mir-lower treats every marked local @@ -2468,7 +2552,10 @@ pub fn translate_body( ctx, body, mir_block, - idx, + crate::LibraryCallSite { + caller: name_str.clone(), + block: idx, + }, block_ptr, &mut value_map, &block_map, @@ -2498,7 +2585,7 @@ pub fn translate_body( } } - Ok(op_ptr) + Ok((op_ptr, value_map.used_cluster_query_fallback)) } /// Choose the debugger-visible spelling independently of a function's symbol. diff --git a/cuda-oxide/crates/mir-importer/src/translator/facts.rs b/cuda-oxide/crates/mir-importer/src/translator/facts.rs index a6b24ac367..32df25d15e 100644 --- a/cuda-oxide/crates/mir-importer/src/translator/facts.rs +++ b/cuda-oxide/crates/mir-importer/src/translator/facts.rs @@ -219,6 +219,246 @@ pub(crate) fn validate_grid_constant_marker_owner( }) } +// Source facts consumed by the prepared library checker. These are nominal +// identity/signature questions only; no participation or alias proof runs here. +use rustc_middle::ty::{Instance, Ty, TyCtxt, TyKind, TypingEnv}; +use rustc_span::def_id::{CRATE_DEF_INDEX, DefId}; + +fn library_device_type(tcx: TyCtxt<'_>, module_name: &str, name: &str) -> Option { + for &krate in tcx.crates(()) { + if tcx.crate_name(krate).as_str() != "cuda_device" { + continue; + } + let root = DefId { + krate, + index: CRATE_DEF_INDEX, + }; + let module = tcx + .module_children(root) + .iter() + .find(|item| item.ident.name.as_str() == module_name)? + .res + .opt_def_id()?; + return tcx + .module_children(module) + .iter() + .find(|item| item.ident.name.as_str() == name) + .and_then(|item| item.res.opt_def_id()); + } + None +} + +fn library_dynamic_shared_getter<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> bool { + let Some(name) = tcx.opt_item_name(instance.def_id()) else { + return false; + }; + if !matches!(name.as_str(), "get" | "get_raw" | "get_fixed" | "offset") { + return false; + } + let Some(implementation) = tcx.impl_of_assoc(instance.def_id()) else { + return false; + }; + if tcx.impl_is_of_trait(implementation) { + return false; + } + let owner = tcx.normalize_erasing_regions( + TypingEnv::fully_monomorphized(), + tcx.type_of(implementation).instantiate(tcx, instance.args), + ); + let TyKind::Adt(def, args) = owner.kind() else { + return false; + }; + if Some(def.did()) != library_device_type(tcx, "shared", "DynamicSharedArray") { + return false; + } + let signature = tcx + .fn_sig(instance.def_id()) + .instantiate(tcx, instance.args) + .skip_binder(); + let inputs = signature.inputs(); + let expected = if name.as_str() == "get_raw" { + tcx.types.u8 + } else { + args.type_at(0) + }; + !signature.c_variadic() + && (if name.as_str() == "offset" { + inputs == [tcx.types.usize] + } else { + inputs.is_empty() + }) + && matches!(signature.output().kind(), TyKind::RawPtr(ty, rustc_middle::ty::Mutability::Mut) if *ty == expected) +} + +fn library_collective_carrier<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<(Ty<'tcx>, u64)> { + let TyKind::Adt(def, args) = ty.kind() else { + return None; + }; + if Some(def.did()) != library_device_type(tcx, "disjoint", "DisjointSlice") { + return None; + } + let mut types = args.types(); + let element = types.next()?; + let TyKind::Adt(space, args) = types.next()?.kind() else { + return None; + }; + if Some(space.did()) != library_device_type(tcx, "disjoint", "CollectiveTiles") { + return None; + } + Some((element, args.const_at(0).try_to_target_usize(tcx)?)) +} + +fn library_resource_parameter<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool { + library_collective_carrier(tcx, ty).is_some() + || matches!(ty.kind(), TyKind::Ref(_, inner, rustc_middle::ty::Mutability::Not) + if matches!(inner.kind(), TyKind::Slice(_))) +} + +/// Nominal source restriction for one indirect assignment prefix. +pub(crate) fn library_contains_resource(ty: rustc_public::ty::Ty) -> bool { + use rustc_public::rustc_internal; + rustc_middle::ty::tls::with(|tcx| { + let mut pending = vec![rustc_internal::internal(tcx, ty)]; + let mut visited = std::collections::HashSet::new(); + while let Some(ty) = pending.pop() { + if !visited.insert(ty) { + continue; + } + if library_resource_parameter(tcx, ty) + || matches!(ty.kind(), TyKind::Adt(def, _) if ["StorageDescriptor", "StorageReservation"].iter().any(|name| Some(def.did()) == library_device_type(tcx, "library", name))) + { + return true; + } + match ty.kind() { + TyKind::Ref(_, inner, _) + | TyKind::RawPtr(inner, _) + | TyKind::Array(inner, _) + | TyKind::Slice(inner) => pending.push(*inner), + TyKind::Tuple(fields) => pending.extend(fields.iter()), + TyKind::Adt(def, args) => pending.extend(def.all_fields().map(|field| { + tcx.normalize_erasing_regions( + TypingEnv::fully_monomorphized(), + field.ty(tcx, args), + ) + })), + TyKind::Closure(_, args) => pending.push(args.as_closure().tupled_upvars_ty()), + _ => {} + } + } + false + }) +} + +/// Classify only genuine source calls whose expansion would erase checker facts. +pub(crate) fn library_source_effect( + func: &mir::Operand, + args: &[mir::Operand], + destination: &mir::Place, + body: &mir::Body, +) -> Option { + use dialect_mir::library::LibrarySourceEffectKind as Effect; + use rustc_public::{ + rustc_internal, + ty::{RigidTy, TyKind as PublicTyKind}, + }; + let mir::Operand::Constant(constant) = func else { + return None; + }; + let PublicTyKind::RigidTy(RigidTy::FnDef(def, substs)) = constant.const_.ty().kind() else { + return None; + }; + let public = mir::mono::Instance::resolve(def, &substs).ok()?; + rustc_middle::ty::tls::with(|tcx| { + let instance = rustc_internal::internal(tcx, public); + let path = tcx.def_path_str(instance.def_id()); + let krate = tcx.crate_name(instance.def_id().krate); + let name = path.rsplit("::").next().unwrap_or(""); + // Stable libcore implements these integer operations through borrowed + // comparisons, unlike the equivalent integer_min/max intrinsics. Only + // the genuine default methods on by-value integers have this summary: + // a user Ord implementation may run arbitrary comparison/drop effects. + if krate.as_str() == "core" + && [rustc_span::sym::cmp_ord_min, rustc_span::sym::cmp_ord_max] + .into_iter() + .any(|item| tcx.get_diagnostic_item(item) == Some(instance.def_id())) + && args.len() == 2 + { + let signature = tcx + .fn_sig(instance.def_id()) + .instantiate(tcx, instance.args) + .skip_binder(); + let output = signature.output(); + if matches!(output.kind(), TyKind::Int(_) | TyKind::Uint(_)) + && signature.inputs() == [output, output] + && !signature.c_variadic() + && args.iter().all(|arg| { + arg.ty(body.locals()) + .is_ok_and(|ty| rustc_internal::internal(tcx, ty) == output) + }) + && destination + .ty(body.locals()) + .is_ok_and(|ty| rustc_internal::internal(tcx, ty) == output) + { + return Some(Effect::ScalarIntrinsic); + } + } + if krate.as_str() == "cuda_device" + && super::terminator::intrinsics::asm::InlinePtxCallKind::from_path(&path).is_some() + { + return Some(Effect::InlineAssembly); + } + if library_dynamic_shared_getter(tcx, instance) { + return Some(Effect::DynamicShared); + } + if (krate.as_str() == "cuda_device" && path.ends_with("::sync_threads")) + || (krate.as_str() == "cuda_intrinsics" && path.ends_with("::i0034")) + { + return Some(Effect::BlockSync); + } + let generated = &super::terminator::intrinsics::generated::is_generated_intrinsic_path; + let raw = &super::terminator::intrinsics::generated::is_raw_generated_intrinsic_path; + if (matches!(krate.as_str(), "cuda_device" | "cuda-device") && generated(&path)) + || (krate.as_str() == "cuda_intrinsics" && raw(&path)) + { + return Some(match name { + "blockDim_x" | "blockDim_y" | "blockDim_z" | "blockIdx_x" | "blockIdx_y" + | "blockIdx_z" | "gridDim_x" | "gridDim_y" | "gridDim_z" | "i0003" | "i0007" + | "i0011" | "i0002" | "i0006" | "i0010" | "i0004" | "i0008" | "i0012" => { + Effect::BlockQuery + } + "threadIdx_x" | "threadIdx_y" | "threadIdx_z" | "i0001" | "i0005" | "i0009" => { + Effect::LaneQuery + } + _ => Effect::UnsupportedIntrinsic, + }); + } + if matches!(instance.def, rustc_middle::ty::InstanceKind::Intrinsic(_)) { + let unit = destination + .ty(body.locals()) + .is_ok_and(|ty| rustc_internal::internal(tcx, ty).is_unit()); + if name == "assert_inhabited" + && matches!(krate.as_str(), "core" | "std") + && args.is_empty() + && unit + && instance.args.len() == 1 + && instance.args.types().next().is_some_and(|ty| { + tcx.layout_of(TypingEnv::fully_monomorphized().as_query_input(ty)) + .is_ok_and(|layout| !layout.is_uninhabited()) + }) + { + return Some(Effect::UnitHint); + } + return Some(match name { + "integer_min" | "integer_max" | "unchecked_add" | "unchecked_sub" + | "unchecked_mul" => Effect::ScalarIntrinsic, + "arith_offset" => Effect::PointerOffset, + _ => Effect::UnsupportedIntrinsic, + }); + } + None + }) +} + // ============================================================================ // Identity facts (crate-anchored; a miss falls through, never errors) // ============================================================================ diff --git a/cuda-oxide/crates/mir-importer/src/translator/statement.rs b/cuda-oxide/crates/mir-importer/src/translator/statement.rs index 74588c3098..d8c0bab20f 100644 --- a/cuda-oxide/crates/mir-importer/src/translator/statement.rs +++ b/cuda-oxide/crates/mir-importer/src/translator/statement.rs @@ -66,6 +66,7 @@ use std::num::NonZeroUsize; /// /// The last inserted operation (for chaining), or `prev_op` if no ops were created. /// For `Rvalue::Use`, no operation is created - just updates `value_map`. +#[allow(clippy::too_many_arguments)] pub fn translate_statement( ctx: &mut Context, body: &mir::Body, @@ -73,9 +74,101 @@ pub fn translate_statement( value_map: &mut ValueMap, block_ptr: Ptr, prev_op: Option>, + site: &crate::LibraryCallSite, + statement: usize, ) -> TranslationResult>> { + use dialect_mir::library::{LibraryWriteKind, LibraryWriteSourceAttr}; let loc = span_to_location(ctx, stmt.source_info.span); + let kind = match &stmt.kind { + mir::StatementKind::Intrinsic(mir::NonDivergingIntrinsic::CopyNonOverlapping(_)) => { + Some(LibraryWriteKind::Bulk) + } + mir::StatementKind::Assign(place, _) + | mir::StatementKind::SetDiscriminant { place, .. } => { + classify_indirect_write(body, place, &loc)? + } + _ => None, + }; + let source = kind.map(|kind| LibraryWriteSourceAttr { + site: site.clone(), + statement, + kind, + }); + translate_statement_inner( + ctx, + body, + stmt, + value_map, + block_ptr, + prev_op, + source.as_ref(), + loc, + ) +} + +/// Classify a surviving indirect destination from its original place types. +/// Dynamic address provenance remains the stateful consumer's responsibility. +pub(crate) fn classify_indirect_write( + body: &mir::Body, + place: &mir::Place, + loc: &Location, +) -> TranslationResult> { + if !place + .projection + .iter() + .any(|p| matches!(p, mir::ProjectionElem::Deref)) + { + return Ok(None); + } + let mut nominal_protected = false; + for length in 0..=place.projection.len() { + let prefix = mir::Place { + local: place.local, + projection: place.projection[..length].to_vec(), + }; + let ty = prefix.ty(body.locals()).map_err(|error| { + input_error!( + loc.clone(), + TranslationErr::invalid_op(format!( + "cannot resolve a resource mutation target type: {error:?}" + )) + ) + })?; + nominal_protected |= facts::library_contains_resource(ty); + } + Ok(Some(dialect_mir::library::LibraryWriteKind::Indirect { + nominal_protected, + })) +} +/// Attach the source check to the actual destination mutation, after its +/// address was evaluated and before any destination bytes are overwritten. +pub(crate) fn retain_source_write( + ctx: &mut Context, + mutation: Ptr, + source: Option<&dialect_mir::library::LibraryWriteSourceAttr>, +) { + if let Some(source) = source { + let address = mutation.deref(ctx).get_operand(0); + let check = + dialect_mir::ops::MirLibraryWriteCheckOp::build(ctx, source.clone(), vec![address]); + let event_loc = mutation.deref(ctx).loc().clone(); + check.deref_mut(ctx).set_loc(event_loc); + check.insert_before(ctx, mutation); + } +} + +#[allow(clippy::too_many_arguments)] +fn translate_statement_inner( + ctx: &mut Context, + body: &mir::Body, + stmt: &mir::Statement, + value_map: &mut ValueMap, + block_ptr: Ptr, + prev_op: Option>, + write_source: Option<&dialect_mir::library::LibraryWriteSourceAttr>, + loc: Location, +) -> TranslationResult>> { match &stmt.kind { mir::StatementKind::Assign(place, rvalue) => { // Fast paths: array initializers assigned to an addressable local. @@ -85,7 +178,15 @@ pub fn translate_statement( match rvalue { mir::Rvalue::Aggregate(mir::AggregateKind::Array(_), operands) => { return translate_array_agg_into_alloca( - ctx, body, place, operands, value_map, block_ptr, prev_op, loc, + ctx, + body, + place, + operands, + value_map, + block_ptr, + prev_op, + loc, + write_source, ); } // A fully-constant array: copy it in from an immutable @@ -103,6 +204,7 @@ pub fn translate_statement( prev_op, loc.clone(), )? { + retain_source_write(ctx, last, write_source); return Ok(Some(last)); } } @@ -117,7 +219,16 @@ pub fn translate_statement( ) })? as usize; return translate_repeat_into_alloca( - ctx, body, place, operand, n, value_map, block_ptr, prev_op, loc, + ctx, + body, + place, + operand, + n, + value_map, + block_ptr, + prev_op, + loc, + write_source, ); } _ => {} @@ -220,7 +331,10 @@ pub fn translate_statement( store_op.insert_at_front(block_ptr, ctx); } - Ok(Some(store_op)) + { + retain_source_write(ctx, store_op, write_source); + Ok(Some(store_op)) + } } mir::ProjectionElem::Field(field_idx, field_ty) => { // struct.field = value (field assignment) @@ -290,7 +404,10 @@ pub fn translate_statement( ); store_op.deref_mut(ctx).set_loc(loc); store_op.insert_after(ctx, field_addr_op); - Ok(Some(store_op)) + { + retain_source_write(ctx, store_op, write_source); + Ok(Some(store_op)) + } } mir::ProjectionElem::ConstantIndex { offset, @@ -388,7 +505,10 @@ pub fn translate_statement( current_prev, loc, ); - Ok(Some(store_op)) + { + retain_source_write(ctx, store_op, write_source); + Ok(Some(store_op)) + } } mir::ProjectionElem::Index(index_local) => { // arr[i] = value with runtime index. @@ -453,7 +573,10 @@ pub fn translate_statement( current_prev, loc, ); - Ok(Some(store_op)) + { + retain_source_write(ctx, store_op, write_source); + Ok(Some(store_op)) + } } _ => input_err!( loc, @@ -546,7 +669,10 @@ pub fn translate_statement( store_op.deref_mut(ctx).set_loc(loc); store_op.insert_after(ctx, addr_op); - Ok(Some(store_op)) + { + retain_source_write(ctx, store_op, write_source); + Ok(Some(store_op)) + } } ( mir::ProjectionElem::Field(outer_field_idx, outer_field_ty), @@ -652,7 +778,10 @@ pub fn translate_statement( store_op.deref_mut(ctx).set_loc(loc); store_op.insert_after(ctx, inner_addr_op); - Ok(Some(store_op)) + { + retain_source_write(ctx, store_op, write_source); + Ok(Some(store_op)) + } } ( mir::ProjectionElem::Deref, @@ -676,6 +805,7 @@ pub fn translate_statement( prev_op, block_ptr, loc, + write_source, ) } ( @@ -709,6 +839,7 @@ pub fn translate_statement( prev_op, block_ptr, loc, + write_source, ) } ( @@ -729,6 +860,7 @@ pub fn translate_statement( prev_op, block_ptr, loc, + write_source, ) } ( @@ -753,6 +885,7 @@ pub fn translate_statement( prev_op, block_ptr, loc, + write_source, ) } (mir::ProjectionElem::Downcast(_), mir::ProjectionElem::Field(_, _)) => { @@ -773,6 +906,7 @@ pub fn translate_statement( prev_op, block_ptr, loc, + write_source, ) } _ => input_err!( @@ -802,6 +936,7 @@ pub fn translate_statement( prev_op, block_ptr, loc, + write_source, ) } } @@ -905,7 +1040,10 @@ pub fn translate_statement( } else { memcpy_op.insert_at_front(block_ptr, ctx); } - Ok(Some(memcpy_op)) + { + retain_source_write(ctx, memcpy_op, write_source); + Ok(Some(memcpy_op)) + } } // Statements with observable runtime effect that are not yet lowered. @@ -1029,7 +1167,10 @@ pub fn translate_statement( set_op.insert_at_front(block_ptr, ctx); } - Ok(Some(set_op)) + { + retain_source_write(ctx, set_op, write_source); + Ok(Some(set_op)) + } } } } @@ -1060,6 +1201,7 @@ fn store_through_place_address( prev_op: Option>, block_ptr: Ptr, loc: Location, + write_source: Option<&dialect_mir::library::LibraryWriteSourceAttr>, ) -> TranslationResult>> { let mut current_prev = prev_op; if let Some(rvalue_op) = rvalue_op_opt { @@ -1092,6 +1234,9 @@ fn store_through_place_address( loc.clone(), )? { + if let Some(last) = result { + retain_source_write(ctx, last, write_source); + } return Ok(result); } @@ -1133,7 +1278,10 @@ fn store_through_place_address( Some(prev) => store_op.insert_after(ctx, prev), None => store_op.insert_at_front(block_ptr, ctx), } - Ok(Some(store_op)) + { + retain_source_write(ctx, store_op, write_source); + Ok(Some(store_op)) + } } /// Extract the element type and address space from a pointer that points @@ -1316,6 +1464,7 @@ fn translate_repeat_into_alloca( block_ptr: Ptr, prev_op: Option>, loc: Location, + write_source: Option<&dialect_mir::library::LibraryWriteSourceAttr>, ) -> TranslationResult>> { let (elem_val, after_operand) = rvalue::translate_operand( ctx, @@ -1354,6 +1503,7 @@ fn translate_repeat_into_alloca( current_prev, block_ptr, loc.clone(), + write_source, )?; current_prev = after_store.or(current_prev); } @@ -1375,6 +1525,7 @@ fn translate_array_agg_into_alloca( block_ptr: Ptr, prev_op: Option>, loc: Location, + write_source: Option<&dialect_mir::library::LibraryWriteSourceAttr>, ) -> TranslationResult>> { let mut current_prev = prev_op; @@ -1416,6 +1567,7 @@ fn translate_array_agg_into_alloca( current_prev, block_ptr, loc.clone(), + write_source, )?; current_prev = after_store.or(current_prev); } diff --git a/cuda-oxide/crates/mir-importer/src/translator/terminator/drop_glue.rs b/cuda-oxide/crates/mir-importer/src/translator/terminator/drop_glue.rs index 3d95fc512e..bec9740c64 100644 --- a/cuda-oxide/crates/mir-importer/src/translator/terminator/drop_glue.rs +++ b/cuda-oxide/crates/mir-importer/src/translator/terminator/drop_glue.rs @@ -164,6 +164,7 @@ pub(super) fn emit_drop_glue( block_map: &[Ptr], legaliser: &mut Legaliser, loc: Location, + library_site: crate::LibraryCallSite, ) -> TranslationResult> { // Resolve the monomorphized drop_in_place:: instance. let drop_instance = Instance::resolve_drop_in_place(dropped_ty); @@ -249,6 +250,7 @@ pub(super) fn emit_drop_glue( 0, ); call_op.deref_mut(ctx).set_loc(loc.clone()); + crate::library::attach_call_source(ctx, call_op, library_site); let callee_attr = pliron::builtin::attributes::StringAttr::new(callee_name.to_string()); call_op.deref_mut(ctx).attributes.set( diff --git a/cuda-oxide/crates/mir-importer/src/translator/terminator/helpers.rs b/cuda-oxide/crates/mir-importer/src/translator/terminator/helpers.rs index 78012fbfe0..65917f4026 100644 --- a/cuda-oxide/crates/mir-importer/src/translator/terminator/helpers.rs +++ b/cuda-oxide/crates/mir-importer/src/translator/terminator/helpers.rs @@ -20,7 +20,8 @@ use crate::error::{TranslationErr, TranslationResult}; use crate::translator::values::{ValueMap, establish_declared_pointer_type}; -use crate::translator::{payload_store, rvalue}; +use crate::translator::{payload_store, rvalue, statement::retain_source_write}; +use dialect_mir::library::LibraryWriteSourceAttr; use dialect_mir::{ attributes::MirPointerKindAuthorityAttr, ops::{MirCallOp, MirConstructArrayOp, MirGotoOp}, @@ -74,10 +75,13 @@ pub(crate) enum PreparedDestinationWrite { /// representation coercions and ZST handling. Local(mir::Local), /// An ordinary projected place whose writable address is fixed already. - Address(Value), + Address(Value, Option), /// An enum payload that must be rebuilt rather than written through a raw /// payload pointer. - CanonicalPayload(payload_store::PreparedCanonicalPayloadStore), + CanonicalPayload( + payload_store::PreparedCanonicalPayloadStore, + Option, + ), } fn require_writable_destination_address( @@ -136,7 +140,10 @@ pub(crate) fn prepare_destination_write( ); }; return Ok(( - PreparedDestinationWrite::CanonicalPayload(prepared), + PreparedDestinationWrite::CanonicalPayload( + prepared, + crate::library::current_destination_write_source(), + ), prepared_prev.or(prev_op), )); } @@ -155,7 +162,10 @@ pub(crate) fn prepare_destination_write( require_writable_destination_address(translated_address, &destination.projection, loc)?; Ok(( - PreparedDestinationWrite::Address(address), + PreparedDestinationWrite::Address( + address, + crate::library::current_destination_write_source(), + ), address_prev.or(prev_op), )) } @@ -172,11 +182,12 @@ pub(crate) fn finish_destination_write( ) -> TranslationResult> { use dialect_mir::ops::MirStoreOp; + let prev_op = crate::library::retain_source_result(ctx, value, prev_op, loc.clone()); match destination { PreparedDestinationWrite::Local(local) => Ok(value_map .store_local(ctx, local, value, block_ptr, Some(prev_op)) .unwrap_or(prev_op)), - PreparedDestinationWrite::Address(address) => { + PreparedDestinationWrite::Address(address, source) => { let store_op = Operation::new( ctx, MirStoreOp::get_concrete_op_info(), @@ -187,17 +198,20 @@ pub(crate) fn finish_destination_write( ); store_op.deref_mut(ctx).set_loc(loc); store_op.insert_after(ctx, prev_op); + retain_source_write(ctx, store_op, source.as_ref()); Ok(store_op) } - PreparedDestinationWrite::CanonicalPayload(payload) => { - payload_store::rebuild_and_store_prepared( + PreparedDestinationWrite::CanonicalPayload(payload, source) => { + let store_op = payload_store::rebuild_and_store_prepared( ctx, &payload, value, block_ptr, Some(prev_op), loc, - ) + )?; + retain_source_write(ctx, store_op, source.as_ref()); + Ok(store_op) } } } @@ -339,6 +353,42 @@ pub fn emit_function_call( value_map: &mut ValueMap, block_map: &[Ptr], loc: Location, +) -> TranslationResult> { + emit_function_call_at_site( + ctx, + body, + callee_name, + args, + destination, + return_type, + external_callee_type, + target, + block_ptr, + prev_op, + value_map, + block_map, + loc, + None, + ) +} + +/// Emit an original source call with its collected caller/terminator identity. +#[allow(clippy::too_many_arguments)] +pub fn emit_function_call_at_site( + ctx: &mut Context, + body: &mir::Body, + callee_name: &str, + args: &[mir::Operand], + destination: &mir::Place, + return_type: TypeHandle, + external_callee_type: Option, + target: &Option, + block_ptr: Ptr, + prev_op: Option>, + value_map: &mut ValueMap, + block_map: &[Ptr], + loc: Location, + source_site: Option, ) -> TranslationResult> { let mut arg_values = Vec::new(); let mut last_op = prev_op; @@ -350,39 +400,6 @@ pub fn emit_function_call( last_op = arg_last_op; } - // A Rust foreign declaration uses generic pointers at its ABI surface, - // while an argument may retain a concrete GPU address space internally - // (for example, `*mut T` in shared memory). Adapt that representation at - // the frozen declaration boundary, but only when pointee, mutability, and - // source pointer kind already agree exactly. This is not authority to turn - // a raw pointer into a reference or otherwise manufacture provenance. - if let Some(signature) = external_callee_type { - let expected_arguments = { - let signature_ref = signature.deref(ctx); - let signature = signature_ref - .downcast_ref::() - .expect("foreign callee type must be a builtin FunctionType"); - signature.arg_types() - }; - - // Variadic declarations are rejected before this point, so unequal - // arity is invalid and must remain visible to MirCallOp verification. - if arg_values.len() == expected_arguments.len() { - for (argument, expected) in arg_values.iter_mut().zip(expected_arguments) { - let (normalized, normalized_last_op) = normalize_call_pointer_argument( - ctx, - *argument, - expected, - block_ptr, - last_op, - loc.clone(), - ); - *argument = normalized; - last_op = normalized_last_op; - } - } - } - // Ordinary Rust helper signatures also expose generic pointers. Resolve // the monomorphized operand type (not an unresolved trait signature), and // adapt only address space. All other type/provenance mismatches still @@ -422,6 +439,75 @@ pub fn emit_function_call( } } + emit_function_call_values( + ctx, + body, + callee_name, + arg_values, + destination, + return_type, + external_callee_type, + target, + block_ptr, + last_op, + value_map, + block_map, + loc, + source_site, + ) +} + +/// Emit already translated arguments at an explicit foreign ABI boundary. +#[allow(clippy::too_many_arguments)] +pub(crate) fn emit_function_call_values( + ctx: &mut Context, + body: &mir::Body, + callee_name: &str, + mut arg_values: Vec, + destination: &mir::Place, + return_type: TypeHandle, + external_callee_type: Option, + target: &Option, + block_ptr: Ptr, + mut last_op: Option>, + value_map: &mut ValueMap, + block_map: &[Ptr], + loc: Location, + source_site: Option, +) -> TranslationResult> { + // A Rust foreign declaration uses generic pointers at its ABI surface, + // while an argument may retain a concrete GPU address space internally + // (for example, `*mut T` in shared memory). Adapt that representation at + // the frozen declaration boundary, but only when pointee, mutability, and + // source pointer kind already agree exactly. This is not authority to turn + // a raw pointer into a reference or otherwise manufacture provenance. + if let Some(signature) = external_callee_type { + let expected_arguments = { + let signature_ref = signature.deref(ctx); + let signature = signature_ref + .downcast_ref::() + .expect("foreign callee type must be a builtin FunctionType"); + signature.arg_types() + }; + + // Variadic declarations are rejected before this point, so unequal + // arity is invalid and must remain visible to MirCallOp verification. + if arg_values.len() == expected_arguments.len() { + for (argument, expected) in arg_values.iter_mut().zip(expected_arguments) { + let (normalized, normalized_last_op) = normalize_call_pointer_argument( + ctx, + *argument, + expected, + block_ptr, + last_op, + loc.clone(), + ); + *argument = normalized; + last_op = normalized_last_op; + } + } + } + let prepared_destination = if target.is_some() { let (prepared, prepared_last_op) = prepare_destination_write( ctx, @@ -455,6 +541,9 @@ pub fn emit_function_call( pliron::identifier::Identifier::try_from("callee").unwrap(), callee_attr, ); + if let Some(site) = source_site { + crate::library::attach_call_source(ctx, call_op, site); + } if let Some(signature) = external_callee_type { MirCallOp::new(call_op).set_external_callee_signature(ctx, signature); } @@ -497,6 +586,111 @@ pub fn emit_function_call( )) } +/// Adapt raw shared pointers at an imported Rust function's generic ABI. +/// Parameter types come from the actual `mir.func`, not the call operands. +/// Every mismatch except otherwise-identical AS3 -> AS0 raw pointers remains +/// visible to ordinary call verification. +pub(crate) fn normalize_internal_shared_pointer_arguments( + ctx: &mut Context, + module: Ptr, +) { + use dialect_mir::{ + ops::MirFuncOp, + types::{MirPointerKind, address_space}, + }; + use pliron::{ + builtin::op_interfaces::SymbolOpInterface, + linked_list::{ContainsLinkedList, LinkedList}, + }; + use std::collections::BTreeMap; + + fn collect(ctx: &Context, op: Ptr, operations: &mut Vec>) { + operations.push(op); + for region in op.deref(ctx).regions() { + for block in region.deref(ctx).iter(ctx) { + for child in block.deref(ctx).iter(ctx) { + collect(ctx, child, operations); + } + } + } + } + let mut operations = Vec::new(); + collect(ctx, module, &mut operations); + let signatures: BTreeMap<_, _> = operations + .iter() + .filter_map(|&op| { + let function = MirFuncOp::wrap(ctx, op)?; + Some(( + function.get_symbol_name(ctx).to_string(), + function.get_type(ctx).deref(ctx).arg_types(), + )) + }) + .collect(); + for op in operations { + let Some(call) = Operation::get_op::(op, ctx) else { + continue; + }; + if call.get_attr_external_callee_type(ctx).is_some() { + continue; + } + let Some(callee) = call + .get_attr_callee(ctx) + .map(|callee| String::from((*callee).clone())) + else { + continue; + }; + let Some(parameters) = signatures.get(&callee) else { + continue; + }; + if parameters.len() != op.deref(ctx).get_num_operands() { + continue; + } + let block = op + .deref(ctx) + .get_parent_block() + .expect("imported call has a block"); + let loc = op.deref(ctx).loc().clone(); + let mut previous = op.deref(ctx).get_prev(); + for (index, &expected) in parameters.iter().enumerate() { + let value = op.deref(ctx).get_operand(index); + let actual = value.get_type(ctx); + let matching_shared_raw = { + let actual = actual.deref(ctx); + let expected = expected.deref(ctx); + match ( + actual.downcast_ref::(), + expected.downcast_ref::(), + ) { + (Some(actual), Some(expected)) => { + actual.address_space == address_space::SHARED + && expected.address_space == address_space::GENERIC + && actual.pointee == expected.pointee + && actual.is_mutable == expected.is_mutable + && actual.kind == expected.kind + && matches!( + actual.kind, + MirPointerKind::RawMut | MirPointerKind::RawConst + ) + } + _ => false, + } + }; + if matching_shared_raw { + let (value, last) = normalize_call_pointer_argument( + ctx, + value, + expected, + block, + previous, + loc.clone(), + ); + Operation::replace_operand(op, ctx, index, value); + previous = last; + } + } + } +} + /// Adapt a call pointer argument only when the address space is the /// sole difference from its declared ABI parameter type. fn normalize_call_pointer_argument( @@ -891,6 +1085,204 @@ mod tests { ); } + #[test] + fn internal_shared_pointer_boundary_preserves_other_mismatches() { + use dialect_mir::{library::LibraryCallSite, types::MirPointerKind}; + use pliron::{common_traits::Verify, linked_list::ContainsLinkedList}; + + for ( + actual_space, + actual_kind, + expected_space, + expected_kind, + expected_bits, + expected_mutable, + adapts, + verifies, + ) in [ + ( + 3, + MirPointerKind::RawMut, + 0, + MirPointerKind::RawMut, + 32, + true, + true, + true, + ), + ( + 3, + MirPointerKind::RawConst, + 0, + MirPointerKind::RawConst, + 32, + false, + true, + true, + ), + ( + 3, + MirPointerKind::RawMut, + 0, + MirPointerKind::UniqueRef, + 32, + true, + false, + false, + ), + ( + 3, + MirPointerKind::SharedRef, + 0, + MirPointerKind::SharedRef, + 32, + false, + false, + false, + ), + ( + 3, + MirPointerKind::RawMut, + 0, + MirPointerKind::RawMut, + 8, + true, + false, + false, + ), + ( + 3, + MirPointerKind::RawMut, + 0, + MirPointerKind::RawMut, + 32, + false, + false, + false, + ), + ( + 1, + MirPointerKind::RawMut, + 0, + MirPointerKind::RawMut, + 32, + true, + false, + false, + ), + ( + 0, + MirPointerKind::RawMut, + 3, + MirPointerKind::RawMut, + 32, + true, + false, + false, + ), + ( + 0, + MirPointerKind::RawMut, + 0, + MirPointerKind::RawMut, + 32, + true, + false, + true, + ), + ] { + let ctx = &mut Context::new(); + dialect_mir::register(ctx); + let actual_element = IntegerType::get(ctx, 32, Signedness::Unsigned).into(); + let expected_element = + IntegerType::get(ctx, expected_bits, Signedness::Unsigned).into(); + let actual_mutable = matches!( + actual_kind, + MirPointerKind::RawMut | MirPointerKind::UniqueRef + ); + let actual = MirPtrType::get_with_kind( + ctx, + actual_element, + actual_mutable, + actual_space, + actual_kind, + ) + .into(); + let expected = MirPtrType::get_with_kind( + ctx, + expected_element, + expected_mutable, + expected_space, + expected_kind, + ) + .into(); + let module = ModuleOp::new(ctx, "call_boundary".try_into().unwrap()); + let make_function = |ctx: &mut Context, name: &str, parameter: TypeHandle| { + let ty = FunctionType::get(ctx, vec![parameter], vec![]); + let op = Operation::new( + ctx, + MirFuncOp::get_concrete_op_info(), + vec![], + vec![], + vec![], + 1, + ); + let function = MirFuncOp::new(ctx, op, TypeAttr::new(ty.into())); + function.set_symbol_name(ctx, name.try_into().unwrap()); + module.append_operation(ctx, op, 0); + let block = BasicBlock::new(ctx, None, vec![parameter]); + block.insert_at_back(op.deref(ctx).get_region(0), ctx); + block + }; + make_function(ctx, "callee", expected); + let block = make_function(ctx, "caller", actual); + let argument = block.deref(ctx).get_argument(0); + let call = Operation::new( + ctx, + MirCallOp::get_concrete_op_info(), + vec![], + vec![argument], + vec![], + 0, + ); + let typed_call = MirCallOp::new(call); + typed_call.set_attr_callee(ctx, StringAttr::new("callee".into())); + let site = LibraryCallSite { + caller: "original::caller".into(), + block: 7, + }; + typed_call.set_attr_library_source_site(ctx, site.clone()); + call.insert_at_back(block, ctx); + let location = call.deref(ctx).loc().clone(); + + normalize_internal_shared_pointer_arguments(ctx, module.get_operation()); + + assert_eq!(call.deref(ctx).get_operand(0) != argument, adapts); + assert_eq!(typed_call.verify(ctx).is_ok(), verifies); + assert_eq!( + typed_call.get_attr_library_source_site(ctx).as_deref(), + Some(&site) + ); + assert_eq!(*call.deref(ctx).loc(), location); + assert!(typed_call.get_attr_external_callee_type(ctx).is_none()); + assert!( + typed_call + .get_attr_call_pointer_kind_authority(ctx) + .is_none() + ); + if adapts { + let cast = + Operation::get_op::(block.deref(ctx).iter(ctx).next().unwrap(), ctx) + .unwrap(); + assert_eq!( + cast.get_attr_pointer_kind_authority(ctx).as_deref(), + Some(&MirPointerKindAuthorityAttr::AbiBoundary) + ); + cast.verify(ctx).unwrap(); + } + } + } + #[test] fn foreign_pointer_argument_adapts_only_its_address_space() { let mut ctx = Context::new(); diff --git a/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/cluster_query.rs b/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/cluster_query.rs new file mode 100644 index 0000000000..1c605c982f --- /dev/null +++ b/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/cluster_query.rs @@ -0,0 +1,187 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Single-block-cluster queries for known targets below sm90. +//! +//! Intercept compatibility paths before generated dispatch erases their +//! identity. Raw `cuda_intrinsics` calls keep their native target requirements. + +use crate::error::{TranslationErr, TranslationResult}; +use crate::translator::{terminator::helpers, values::ValueMap}; +use dialect_mir::ops::MirConstantOp; +use dialect_nvvm::ops::{ + ReadPtxSregCtaidXOp, ReadPtxSregCtaidYOp, ReadPtxSregCtaidZOp, ReadPtxSregNctaidXOp, + ReadPtxSregNctaidYOp, ReadPtxSregNctaidZOp, +}; +use pliron::basic_block::BasicBlock; +use pliron::builtin::attributes::IntegerAttr; +use pliron::builtin::types::{IntegerType, Signedness}; +use pliron::context::{Context, Ptr}; +use pliron::input_err; +use pliron::location::Location; +use pliron::op::Op; +use pliron::operation::Operation; +use pliron::utils::apint::APInt; +use rustc_public::mir; +use std::num::NonZeroUsize; + +#[derive(Debug, PartialEq, Eq)] +enum Fallback { + Constant(u32), + BlockX, + BlockY, + BlockZ, + GridX, + GridY, + GridZ, +} + +fn fallback(name: &str) -> Option { + Some(match name { + "cuda_device::cluster::cluster_ctaidX" + | "cuda_device::cluster::cluster_ctaidY" + | "cuda_device::cluster::cluster_ctaidZ" => Fallback::Constant(0), + "cuda_device::cluster::cluster_nctaidX" + | "cuda_device::cluster::cluster_nctaidY" + | "cuda_device::cluster::cluster_nctaidZ" => Fallback::Constant(1), + "cuda_device::cluster::__cluster_idxX" => Fallback::BlockX, + "cuda_device::cluster::__cluster_idxY" => Fallback::BlockY, + "cuda_device::cluster::__cluster_idxZ" => Fallback::BlockZ, + "cuda_device::cluster::__cluster_grid_dimX" => Fallback::GridX, + "cuda_device::cluster::__cluster_grid_dimY" => Fallback::GridY, + "cuda_device::cluster::__cluster_grid_dimZ" => Fallback::GridZ, + _ => return None, + }) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn try_dispatch( + ctx: &mut Context, + body: &mir::Body, + name: &str, + args: &[mir::Operand], + destination: &mir::Place, + target: &Option, + block: Ptr, + prev_op: Option>, + values: &mut ValueMap, + blocks: &[Ptr], + loc: Location, +) -> TranslationResult>> { + if !values.pre90_cluster_queries { + return Ok(None); + } + let Some(fallback) = fallback(name) else { + return Ok(None); + }; + if !args.is_empty() { + return input_err!( + loc, + TranslationErr::unsupported(format!( + "cluster query `{name}` expects zero arguments, got {}", + args.len() + )) + ); + } + + macro_rules! register { + ($op:ty, $marker:literal) => { + helpers::emit_generated_nvvm_intrinsic( + ctx, + body, + <$op>::get_concrete_op_info(), + $marker, + destination, + target, + block, + prev_op, + values, + blocks, + loc, + )? + }; + } + let operation = match fallback { + Fallback::Constant(value) => { + let (destination, prev_op) = helpers::prepare_destination_write( + ctx, + body, + destination, + values, + block, + prev_op, + loc.clone(), + )?; + let ty = IntegerType::get(ctx, 32, Signedness::Unsigned); + let operation = Operation::new( + ctx, + MirConstantOp::get_concrete_op_info(), + vec![ty.into()], + vec![], + vec![], + 0, + ); + let value = APInt::from_u64(u64::from(value), NonZeroUsize::new(32).unwrap()); + MirConstantOp::new(operation).set_attr_value(ctx, IntegerAttr::new(ty, value)); + operation.deref_mut(ctx).set_loc(loc.clone()); + helpers::insert_op(ctx, operation, block, prev_op); + let result = operation.deref(ctx).get_result(0); + helpers::emit_prepared_result_and_goto( + ctx, + destination, + result, + target, + block, + operation, + values, + blocks, + loc, + "cluster query has no successor", + )? + } + Fallback::BlockX => register!(ReadPtxSregCtaidXOp, "v1:i0002"), + Fallback::BlockY => register!(ReadPtxSregCtaidYOp, "v1:i0006"), + Fallback::BlockZ => register!(ReadPtxSregCtaidZOp, "v1:i0010"), + Fallback::GridX => register!(ReadPtxSregNctaidXOp, "v1:i0004"), + Fallback::GridY => register!(ReadPtxSregNctaidYOp, "v1:i0008"), + Fallback::GridZ => register!(ReadPtxSregNctaidZOp, "v1:i0012"), + }; + values.used_cluster_query_fallback = true; + Ok(Some(operation)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_compatibility_queries_receive_fallbacks() { + assert_eq!( + fallback("cuda_device::cluster::cluster_ctaidX"), + Some(Fallback::Constant(0)) + ); + assert_eq!( + fallback("cuda_device::cluster::cluster_nctaidZ"), + Some(Fallback::Constant(1)) + ); + assert_eq!( + fallback("cuda_device::cluster::__cluster_idxY"), + Some(Fallback::BlockY) + ); + assert_eq!( + fallback("cuda_device::cluster::__cluster_grid_dimZ"), + Some(Fallback::GridZ) + ); + for name in [ + "cuda_intrinsics::__cuda_oxide_intrinsic_abi_v1::i0266", + "cuda_intrinsics::sreg::cluster_dim_x", + "cuda_device::cluster::cluster_sync", + "cuda_device::cluster::map_shared_rank", + "other::cluster::cluster_ctaidX", + ] { + assert_eq!(fallback(name), None, "{name}"); + } + } +} diff --git a/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/memory.rs b/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/memory.rs index 9de8b59a6f..aeb119c206 100644 --- a/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/memory.rs +++ b/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/memory.rs @@ -1010,6 +1010,7 @@ pub fn emit_dynamic_shared_get( loc: Location, byte_offset: u64, alignment: u64, + fixed_extent: bool, ) -> TranslationResult> { use dialect_mir::ops::MirExternSharedOp; // Get the destination type to determine the pointer element type @@ -1041,6 +1042,32 @@ pub fn emit_dynamic_shared_get( } }; + let layout = if fixed_extent { + Some( + pointee_ty + .layout() + .map_err(|error| { + pliron::input_error!( + loc.clone(), + TranslationErr::unsupported(format!( + "cannot determine fixed shared layout: {error:?}" + )) + ) + })? + .shape(), + ) + } else { + None + }; + let alignment = layout + .as_ref() + .map_or(alignment, |layout| alignment.max(layout.abi_align)); + if fixed_extent && !alignment.is_power_of_two() { + return input_err!( + loc.clone(), + TranslationErr::invalid_op("fixed shared alignment must be a nonzero power of two") + ); + } let elem_ty = crate::translator::types::translate_type(ctx, &pointee_ty)?; // Create a shared memory pointer type (addrspace 3) @@ -1067,6 +1094,27 @@ pub fn emit_dynamic_shared_get( 0, ); op.deref_mut(ctx).set_loc(loc.clone()); + if let Some(layout) = layout { + use pliron::builtin::attributes::IntegerAttr; + use pliron::utils::apint::APInt; + let ty = pliron::builtin::types::IntegerType::get( + ctx, + 64, + pliron::builtin::types::Signedness::Unsigned, + ); + op.deref_mut(ctx).attributes.set( + crate::kernel_resources::FIXED_SHARED_BYTES + .try_into() + .unwrap(), + IntegerAttr::new( + ty, + APInt::from_u64( + layout.size.bytes() as u64, + std::num::NonZeroUsize::new(64).unwrap(), + ), + ), + ); + } let extern_shared = MirExternSharedOp::new(op); diff --git a/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/mod.rs b/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/mod.rs index 5c50b07cde..175d25669d 100644 --- a/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/mod.rs +++ b/cuda-oxide/crates/mir-importer/src/translator/terminator/intrinsics/mod.rs @@ -46,6 +46,7 @@ pub mod asm; pub mod atomic; pub mod bigint; pub mod bitops; +pub(crate) mod cluster_query; pub mod debug; pub mod exact_div; pub mod float_math; diff --git a/cuda-oxide/crates/mir-importer/src/translator/terminator/mod.rs b/cuda-oxide/crates/mir-importer/src/translator/terminator/mod.rs index 524771dea8..1d32e874fe 100644 --- a/cuda-oxide/crates/mir-importer/src/translator/terminator/mod.rs +++ b/cuda-oxide/crates/mir-importer/src/translator/terminator/mod.rs @@ -7,7 +7,8 @@ //! //! This module translates MIR terminators (return, goto, call, switch, etc.) //! into `dialect-mir` operations. GPU intrinsics from `cuda_device` are -//! expanded inline to `dialect-nvvm` operations. +//! expanded inline to `dialect-nvvm` operations. Authenticated native library +//! calls remain typed until the mandatory prepared-MIR proof and lowering. //! //! All non-entry blocks are argument-less: cross-block data flow travels //! through the per-local alloca slots owned by [`ValueMap`], so every branch @@ -100,7 +101,7 @@ use rustc_public::ty::ConstantKind; /// /// Calls to `cuda_device` functions are expanded inline to `dialect-nvvm` operations. /// This includes thread indexing, synchronization, warp primitives, and -/// tensor core operations. +/// tensor core operations. Native library calls remain typed until mandatory lowering. #[allow(clippy::too_many_arguments)] pub fn translate_terminator( ctx: &mut Context, @@ -112,6 +113,52 @@ pub fn translate_terminator( block_map: &[Ptr], rustc_mono_successors: &[usize], legaliser: &mut Legaliser, + library_site: crate::LibraryCallSite, +) -> TranslationResult> { + let site = library_site.clone(); + let translated = translate_terminator_inner( + ctx, + body, + term, + value_map, + block_ptr, + prev_op, + block_map, + rustc_mono_successors, + legaliser, + library_site, + )?; + if matches!(term.kind, mir::TerminatorKind::Drop { .. }) + && crate::library::get_selected_call_target(&site).is_some() + { + let marker = dialect_mir::ops::MirLibrarySourceOp::build( + ctx, + dialect_mir::library::LibrarySourceEffectAttr { + site: site.clone(), + kind: dialect_mir::library::LibrarySourceEffectKind::NontrivialDrop, + }, + None, + ); + let event_loc = translated.deref(ctx).loc().clone(); + marker.deref_mut(ctx).set_loc(event_loc); + marker.insert_before(ctx, translated); + } + dialect_mir::library::set_library_source_site(ctx, translated, site); + Ok(translated) +} + +#[allow(clippy::too_many_arguments)] +fn translate_terminator_inner( + ctx: &mut Context, + body: &mir::Body, + term: &mir::Terminator, + value_map: &mut ValueMap, + block_ptr: Ptr, + prev_op: Option>, + block_map: &[Ptr], + rustc_mono_successors: &[usize], + legaliser: &mut Legaliser, + library_site: crate::LibraryCallSite, ) -> TranslationResult> { let loc = span_to_location(ctx, term.source_info.span); @@ -155,6 +202,7 @@ pub fn translate_terminator( block_map, loc, legaliser, + library_site, ), mir::TerminatorKind::SwitchInt { discr, targets } => match rustc_mono_successors { @@ -169,7 +217,16 @@ pub fn translate_terminator( ) ), _ => translate_switch( - ctx, body, discr, targets, block_ptr, prev_op, value_map, block_map, loc, + ctx, + body, + discr, + targets, + block_ptr, + prev_op, + value_map, + block_map, + loc, + library_site, ), }, @@ -178,8 +235,18 @@ pub fn translate_terminator( target, unwind, } => translate_drop( - ctx, body, place, *target, unwind, block_ptr, prev_op, value_map, block_map, legaliser, + ctx, + body, + place, + *target, + unwind, + block_ptr, + prev_op, + value_map, + block_map, + legaliser, loc, + library_site, ), mir::TerminatorKind::Unreachable => { @@ -525,6 +592,7 @@ fn translate_switch( value_map: &mut ValueMap, block_map: &[Ptr], loc: Location, + library_site: crate::LibraryCallSite, ) -> TranslationResult> { // Translate discriminant let (discr_value, last_op) = match discr { @@ -538,6 +606,11 @@ fn translate_switch( let branches: Vec<_> = targets.branches().collect(); let otherwise_idx: usize = targets.otherwise(); + let impossible = body.blocks[otherwise_idx].statements.is_empty() + && matches!( + body.blocks[otherwise_idx].terminator.kind, + mir::TerminatorKind::Unreachable + ); // For bool switches (2 branches), use MirCondBranchOp if branches.len() == 1 { @@ -653,6 +726,10 @@ fn translate_switch( Operation::get_op::(op, ctx) .expect("MirCondBranchOp") .set_operand_segment_sizes(ctx, segment_sizes); + dialect_mir::library::set_library_source_site(ctx, op, library_site.clone()); + if impossible { + dialect_mir::library::set_library_impossible_successor(ctx, op, 1); + } op.deref_mut(ctx).set_loc(loc); // Use last_inserted_op (which accounts for NOT/EQ ops created above) @@ -766,6 +843,10 @@ fn translate_switch( Operation::get_op::(branch_op, ctx) .expect("MirCondBranchOp") .set_operand_segment_sizes(ctx, segment_sizes); + dialect_mir::library::set_library_source_site(ctx, branch_op, library_site.clone()); + if impossible && i == n - 1 { + dialect_mir::library::set_library_impossible_successor(ctx, branch_op, 1); + } branch_op.deref_mut(ctx).set_loc(loc.clone()); branch_op.insert_after(ctx, cmp_op); @@ -825,6 +906,7 @@ fn translate_drop( block_map: &[Ptr], legaliser: &mut Legaliser, loc: Location, + library_site: crate::LibraryCallSite, ) -> TranslationResult> { let dropped_ty = place.ty(body.locals()).map_err(|e| { input_error!( @@ -842,8 +924,18 @@ fn translate_drop( // Fallback: emit a device-side call to drop_in_place::(ptr). drop_glue::emit_drop_glue( - ctx, body, place, dropped_ty, target, block_ptr, prev_op, value_map, block_map, legaliser, + ctx, + body, + place, + dropped_ty, + target, + block_ptr, + prev_op, + value_map, + block_map, + legaliser, loc, + library_site, ) } @@ -957,6 +1049,88 @@ fn translate_call( block_map: &[Ptr], loc: Location, legaliser: &mut Legaliser, + library_site: crate::LibraryCallSite, +) -> TranslationResult> { + let effect = super::facts::library_source_effect(func, args, destination, body).map(|kind| { + dialect_mir::library::LibrarySourceEffectAttr { + site: library_site.clone(), + kind, + } + }); + let destination_write = super::statement::classify_indirect_write(body, destination, &loc)? + .map(|kind| dialect_mir::library::LibraryWriteSourceAttr { + site: library_site.clone(), + statement: body.blocks[library_site.block].statements.len(), + kind, + }); + let guard = crate::library::SourceEffectGuard::new(effect.clone(), destination_write); + let translated = translate_call_inner( + ctx, + body, + func, + args, + destination, + target, + unwind, + block_ptr, + prev_op, + value_map, + block_map, + loc, + legaliser, + library_site, + )?; + if let Some(effect) = effect { + // Mark the original direct scalar source call for the checker's + // precision summary. Placeholders keep their existing annotation; + // call type verification is independent of this source metadata. + let operations: Vec<_> = block_ptr.deref(ctx).iter(ctx).collect(); + let start = prev_op + .and_then(|p| operations.iter().position(|op| *op == p)) + .map_or(0, |n| n + 1); + for &op in &operations[start..] { + if let Some(call) = Operation::get_op::(op, ctx) { + let placeholder = call.get_attr_callee(ctx).is_some_and(|callee| { + dialect_mir::rust_intrinsics::is_known_placeholder(&String::from( + (*callee).clone(), + )) + }); + let scalar_source_call = effect.kind + == dialect_mir::library::LibrarySourceEffectKind::ScalarIntrinsic + && call + .get_attr_library_source_site(ctx) + .is_some_and(|site| *site == effect.site) + && call + .get_attr_library_call_target(ctx) + .is_some_and(|target| { + target.kind == dialect_mir::library::LibraryCallKind::Direct + }); + if placeholder || scalar_source_call { + call.set_attr_library_expanded_source(ctx, effect.clone()); + } + } + } + } + guard.finish(ctx, translated); + Ok(translated) +} + +#[allow(clippy::too_many_arguments)] +fn translate_call_inner( + ctx: &mut Context, + body: &mir::Body, + func: &mir::Operand, + args: &[mir::Operand], + destination: &mir::Place, + target: &Option, + unwind: &mir::UnwindAction, + block_ptr: Ptr, + prev_op: Option>, + value_map: &mut ValueMap, + block_map: &[Ptr], + loc: Location, + legaliser: &mut Legaliser, + library_site: crate::LibraryCallSite, ) -> TranslationResult> { // See comment in translate_assert for rationale. let _ = unwind; @@ -968,6 +1142,67 @@ fn translate_call( let (pattern_name, call_name, substs_args, type_substs) = extract_func_info(func, &loc)?; let loc = call_debug_location(pattern_name.as_deref(), loc); + if let mir::Operand::Constant(constant) = func + && let rustc_public::ty::TyKind::RigidTy(rustc_public::ty::RigidTy::FnDef(def, substs)) = + constant.const_.ty().kind() + && let Ok(instance) = rustc_public::mir::mono::Instance::resolve(def, &substs) + { + // Authentication precedes provider/layout finalization. Retaining the + // original call needs its genuine role and signature, never a plan. + if let Some(metadata) = crate::library::get_semantics(&library_site) { + if metadata.marker != instance.mangled_name() { + return input_err!( + loc, + TranslationErr::invalid_op( + "library semantic site changed after authentication" + ) + ); + } + return crate::library::translate_call( + ctx, + body, + &metadata, + library_site, + instance, + args, + destination, + &target_usize, + block_ptr, + prev_op, + value_map, + block_map, + loc, + ); + } + } + + if let mir::Operand::Constant(constant) = func + && let rustc_public::ty::TyKind::RigidTy(rustc_public::ty::RigidTy::FnDef(def, _)) = + constant.const_.ty().kind() + && def.krate().name == "cuda_device" + && matches!( + def.def_id().name().as_str().rsplit("::").next(), + Some( + reserved_oxide_symbols::LIBRARY_CALL_V1 + | reserved_oxide_symbols::LIBRARY_CALLBACK_CALL_V1 + | reserved_oxide_symbols::LIBRARY_CHECKED_BLOCK_CALL_V1 + | reserved_oxide_symbols::LIBRARY_CHECKED_BLOCK_RESOURCE_CALL_V1 + | reserved_oxide_symbols::LIBRARY_CHECKED_BLOCK_STORAGE_CALL_V1 + | reserved_oxide_symbols::LIBRARY_STORAGE_NEW_V1 + | reserved_oxide_symbols::LIBRARY_STORAGE_RECONFIGURE_V1 + | reserved_oxide_symbols::LIBRARY_STORAGE_RESERVE_V1 + | reserved_oxide_symbols::LIBRARY_STORAGE_RELEASE_V1 + ) + ) + { + return input_err!( + loc, + TranslationErr::invalid_op( + "native library marker has no authenticated call-site semantics" + ) + ); + } + // Keep CuTe calls as CuTe operations while their tensor and layout // meaning is still visible. Ordinary source helpers continue through the // normal Rust-body path. @@ -987,14 +1222,9 @@ fn translate_call( return result; } - // Is the trait-method Self type SharedArray? Shared with - // `values::classify_call` so intrinsic dispatch and destination-slot - // classification can't drift. + // Share intrinsic dispatch classification with the destination-slot model. let on_shared_array = substs_args.as_ref().is_some_and(self_ty_is_shared_array); - // Skip precondition_check calls - these are UB check assertions that are - // dead code because we return false for RuntimeChecks(UbChecks). - // The MIR still contains these calls, but they're in dead branches. if let Some(target_idx) = target_usize && let mir::Operand::Constant(const_op) = func && let rustc_public::ty::TyKind::RigidTy(rustc_public::ty::RigidTy::FnDef(fn_def, _)) = @@ -1056,6 +1286,7 @@ fn translate_call( block_map, loc, legaliser, + library_site, ); } if let Some(function_item) = extract_function_item_target(&args[0], body, &loc)? { @@ -1072,6 +1303,7 @@ fn translate_call( block_map, loc, legaliser, + library_site, ); } } @@ -1169,6 +1401,7 @@ fn translate_call( loc, 0, // byte_offset = 0 for get() and get_raw() alignment, // User-specified or default alignment + name.contains("::get_fixed"), ); } else if name.contains("::offset") { // DynamicSharedArray::offset(byte_offset) - get pointer at byte offset @@ -1378,7 +1611,7 @@ fn translate_call( // normalized the same way, so caller and callee stay consistent. let return_type = types::translate_destination_type(ctx, body, destination, &loc)?; - helpers::emit_function_call( + helpers::emit_function_call_at_site( ctx, body, legal_name.as_ref(), @@ -1392,6 +1625,7 @@ fn translate_call( value_map, block_map, loc, + Some(library_site), ) } @@ -1496,6 +1730,7 @@ const KERNEL_METADATA_MARKERS: &[&str] = &[ "cuda_device::thread::__unchecked_indexing_config", "cuda_device::cluster::__cluster_config", "cuda_device::shared::__dynamic_shared_alignment", + "cuda_device::shared::__dynamic_shared_inferred", ]; /// Keep the macro-injected launch-context constructor out of user line tables. @@ -1538,6 +1773,7 @@ fn translate_function_item_call( block_map: &[Ptr], loc: Location, legaliser: &mut Legaliser, + library_site: crate::LibraryCallSite, ) -> TranslationResult> { use dialect_mir::ops::{MirCallOp, MirExtractFieldOp}; use pliron::builtin::attributes::StringAttr; @@ -1632,6 +1868,7 @@ fn translate_function_item_call( 0, ); call_op.deref_mut(ctx).set_loc(loc.clone()); + crate::library::attach_call_source(ctx, call_op, library_site); call_op.deref_mut(ctx).attributes.set( Identifier::try_from("callee").unwrap(), @@ -1705,6 +1942,7 @@ fn translate_closure_call( block_map: &[Ptr], loc: Location, legaliser: &mut Legaliser, + library_site: crate::LibraryCallSite, ) -> TranslationResult> { use dialect_mir::ops::{MirCallOp, MirExtractFieldOp}; use pliron::builtin::attributes::StringAttr; @@ -1923,6 +2161,7 @@ fn translate_closure_call( 0, ); call_op.deref_mut(ctx).set_loc(loc.clone()); + crate::library::attach_call_source(ctx, call_op, library_site); // Set callee attribute let callee_attr = StringAttr::new(callee); @@ -1977,7 +2216,7 @@ fn translate_closure_call( /// /// Runtime-reachable panic paths whose diverging call is dropped must use /// [`emit_trap_unreachable_after`] instead. -fn emit_unreachable_after( +pub(crate) fn emit_unreachable_after( ctx: &mut Context, block_ptr: Ptr, prev_op: Option>, @@ -2721,6 +2960,22 @@ fn try_dispatch_intrinsic( return Ok(Some(operation)); } + if let Some(operation) = intrinsics::cluster_query::try_dispatch( + ctx, + body, + name, + args, + destination, + target, + block_ptr, + prev_op, + value_map, + block_map, + loc.clone(), + )? { + return Ok(Some(operation)); + } + if let Some(operation) = intrinsics::generated::try_dispatch_generated_intrinsic( ctx, body, @@ -3197,7 +3452,8 @@ fn try_dispatch_intrinsic( loc, ))) } - "cuda_device::shared::__dynamic_shared_alignment" => { + "cuda_device::shared::__dynamic_shared_alignment" + | "cuda_device::shared::__dynamic_shared_inferred" => { // Zero-cost marker injected by #[launch_contract]. body.rs records // the const alignment on the kernel; no runtime call survives. let actual_prev_op = match prev_op { @@ -3446,6 +3702,652 @@ fn try_dispatch_intrinsic( mod tests { use super::*; + #[test] + fn primitive_ord_summary_authenticates_and_retains_direct_calls() { + use crate::translator::body::translate_body; + use dialect_mir::library::{LibraryCallKind, LibrarySourceEffectKind}; + use dialect_mir::ops::MirCallOp; + use llvm_export::export::DebugKind; + use pliron::common_traits::Verify; + use rustc_public::mir::mono::Instance; + use rustc_public::ty::{RigidTy, TyKind}; + + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "cuda_oxide_ord_summary_{}_{}", + std::process::id(), + unique + )); + std::fs::create_dir_all(&root).unwrap(); + let fixture = root.join("calls.rs"); + std::fs::write( + &fixture, + r#" +#![allow(dead_code)] +pub fn integer(value: i32, bound: i32) -> i32 { value.max(0).min(bound) } +pub fn unsigned(value: u32, bound: u32) -> u32 { value.min(bound).max(0) } +mod spoof { #[inline(never)] pub fn max(a: i32, _: i32) -> i32 { a } } +pub fn named(a: i32, b: i32) -> i32 { spoof::max(a, b) } +mod spoof_trait { + pub trait Ord { fn min(self, other: Self) -> Self; } + impl Ord for i32 { #[inline(never)] fn min(self, _: Self) -> Self { self } } +} +pub fn local_trait(a: i32, b: i32) -> i32 { ::min(a, b) } +#[derive(PartialEq, Eq)] pub struct Effect(i32); +static mut OBSERVED: i32 = 0; +impl core::cmp::Ord for Effect { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + unsafe { OBSERVED = self.0; } + self.0.cmp(&other.0) + } +} +impl PartialOrd for Effect { + fn partial_cmp(&self, other: &Self) -> Option { + Some(core::cmp::Ord::cmp(self, other)) + } +} +pub fn custom(a: Effect, b: Effect) -> Effect { core::cmp::Ord::max(a, b) } +"#, + ) + .unwrap(); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let sysroot = std::process::Command::new(rustc) + .args(["--print", "sysroot"]) + .output() + .unwrap(); + assert!(sysroot.status.success()); + let args = vec![ + "rustc".into(), + "--edition=2024".into(), + "--crate-type=rlib".into(), + "--crate-name=ord_summary".into(), + "--emit=metadata".into(), + "-Zmir-opt-level=0".into(), + format!("--out-dir={}", root.display()), + format!( + "--sysroot={}", + String::from_utf8(sysroot.stdout).unwrap().trim() + ), + fixture.display().to_string(), + ]; + std::thread::Builder::new() + .stack_size(64 * 1024 * 1024) + .spawn(move || { + rustc_public::run!(&args, || { + for (name, scalar_calls) in [ + ("integer", 2), + ("unsigned", 2), + ("named", 0), + ("local_trait", 0), + ("custom", 0), + ] { + let item = rustc_public::all_local_items() + .into_iter() + .find(|item| item.name() == format!("ord_summary::{name}")) + .unwrap(); + let instance = Instance::try_from(item).unwrap(); + let body = instance.body().unwrap(); + let calls: Vec<_> = body + .blocks + .iter() + .enumerate() + .filter_map(|(block, bb)| { + let mir::TerminatorKind::Call { + func, + args, + destination, + .. + } = &bb.terminator.kind + else { + return None; + }; + let effect = crate::translator::facts::library_source_effect( + func, + args, + destination, + &body, + ); + let mir::Operand::Constant(c) = func else { + panic!("direct fixture call") + }; + let TyKind::RigidTy(RigidTy::FnDef(def, args)) = + c.const_.ty().kind() + else { + panic!("function item") + }; + Some(( + block, + Instance::resolve(def, &args).unwrap().mangled_name(), + effect, + )) + }) + .collect(); + assert_eq!(calls.len(), if scalar_calls == 0 { 1 } else { 2 }, "{name}"); + assert_eq!( + calls + .iter() + .filter(|(_, _, effect)| *effect + == Some(LibrarySourceEffectKind::ScalarIntrinsic)) + .count(), + scalar_calls, + "{name}" + ); + if scalar_calls == 0 { + continue; + } + for (debug, direct) in [ + (DebugKind::Off, true), + (DebugKind::Full, true), + (DebugKind::Off, false), + ] { + crate::library::set_selected_call_targets( + calls + .iter() + .filter(|_| direct) + .map(|(block, marker, _)| { + ( + crate::LibraryCallSite { + caller: name.into(), + block: *block, + }, + crate::LibraryCallTargetAttr { + export: marker.clone(), + marker: marker.clone(), + kind: LibraryCallKind::Direct, + }, + ) + }) + .collect(), + ); + let mut ctx = Context::new(); + crate::translator::register_dialects(&mut ctx); + let successors = body + .blocks + .iter() + .map(|b| b.terminator.successors()) + .collect::>(); + let (function, _) = translate_body( + &mut ctx, + &body, + &instance, + body.blocks.len(), + &successors, + false, + None, + Some(name), + &mut Legaliser::default(), + debug, + None, + None, + false, + ) + .unwrap(); + function.deref(&ctx).verify(&ctx).unwrap(); + let scalar = crate::translator::types::translate_type( + &mut ctx, + &body.locals()[1].ty, + ) + .unwrap(); + let mut retained = Vec::new(); + for block in function.deref(&ctx).get_region(0).deref(&ctx).iter(&ctx) { + for op in block.deref(&ctx).iter(&ctx) { + if let Some(call) = Operation::get_op::(op, &ctx) { + let site = call.get_attr_library_source_site(&ctx).unwrap(); + let effect = call.get_attr_library_expanded_source(&ctx); + assert_eq!( + call.get_attr_library_call_target(&ctx).is_some(), + direct + ); + if direct { + let effect = + effect.expect("genuine scalar call retained"); + assert_eq!( + effect.kind, + LibrarySourceEffectKind::ScalarIntrinsic + ); + assert_eq!(*site, effect.site); + } else { + assert!( + effect.is_none(), + "call without a selected target must stay ordinary" + ); + } + assert_eq!( + op.deref(&ctx) + .operands() + .map(|value| value.get_type(&ctx)) + .collect::>(), + vec![scalar, scalar] + ); + assert_eq!(op.deref(&ctx).get_num_results(), 1); + assert_eq!( + op.deref(&ctx).get_result(0).get_type(&ctx), + scalar + ); + retained.push(( + site.block, + String::from( + (*call.get_attr_callee(&ctx).unwrap()).clone(), + ), + )); + } + } + } + retained.sort(); + assert_eq!( + retained, + calls + .iter() + .map(|(bb, marker, _)| (*bb, marker.clone())) + .collect::>() + ); + } + } + crate::library::set_selected_call_targets(Default::default()); + std::ops::ControlFlow::<(), _>::Continue(()) + }) + }) + .unwrap() + .join() + .unwrap() + .expect("fixture compilation succeeds"); + std::fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn native_storage_retention_preserves_original_sites_and_runtime_values() { + use crate::translator::body::translate_body; + use dialect_mir::library::{ + LibraryAccess, LibraryNativeMetadata, LibraryOperation, LibraryReserveLayout, + LibraryRoleAttr, LibraryScalar, LibraryStorageArg, LibraryStoragePolicy, + LibraryWitness, NativeSpec, + }; + use dialect_mir::ops::{MirCallOp, MirExternSharedOp, MirLibraryOp}; + use llvm_export::export::DebugKind; + use pliron::common_traits::Verify; + use rustc_public::mir::mono::Instance; + use rustc_public::ty::{RigidTy, TyKind}; + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "cuda_oxide_block_retention_{}_{}", + std::process::id(), + unique + )); + std::fs::create_dir_all(&root).unwrap(); + let fixture = root.join("calls.rs"); + // Nominal import fixture; the backend authenticates these declarations + // independently. No finalized provider plan is supplied to the importer. + std::fs::write(&fixture,r#" +mod library { + pub struct StorageDescriptor { token: u8 } + pub struct StorageReservation { pointer: *mut T } + #[inline(never)] pub fn cuda_oxide_library_v1_246e25db_storage_new() -> StorageDescriptor<0,0,false,false> { StorageDescriptor { token: 0 } } + #[inline(never)] pub fn cuda_oxide_library_v1_246e25db_storage_reconfigure(_: StorageDescriptor<0,0,false,false>) -> StorageDescriptor<128,16,false,true> { StorageDescriptor { token: 0 } } + #[inline(never)] pub fn cuda_oxide_library_v1_246e25db_storage_reserve(_: &mut StorageDescriptor<128,16,false,true>) -> StorageReservation { StorageReservation { pointer: core::ptr::null_mut() } } + #[inline(never)] pub fn cuda_oxide_library_v1_246e25db_storage_release(_: StorageReservation) {} + #[inline(never)] pub fn cuda_oxide_library_v1_246e25db_checked_block_storage_call(_: R, _: &[i32], _: &mut StorageDescriptor<128,16,false,true>, _: (*const i32,usize), _: (*mut i32,)) {} + #[inline(never)] pub fn cuda_oxide_library_v1_246e25db_call(_: R, _: (*const i32,*mut i32)) {} + #[inline(never)] pub fn cuda_oxide_library_v1_246e25db_call_with_callback(_: R, _: (*const i32,*mut i32), _: F, _: &i32) {} + #[inline(never)] pub fn cuda_oxide_library_v1_246e25db_checked_block_call(_: R, _: (*const i32,usize), _: (*mut i32,)) {} + #[inline(never)] pub fn cuda_oxide_library_v1_246e25db_checked_block_resource_call(_: R, _: &[i32], _: (*const i32,usize), _: (*mut i32,)) {} +} +fn registration() -> &'static str { "{}" } +unsafe extern "C" fn thunk(_: *const i32, a: i32, b: i32) -> i32 { a + b } +pub fn entry(input: *const i32, output: *mut i32, slice: &[i32]) { + library::cuda_oxide_library_v1_246e25db_call(registration,(input,output)); + library::cuda_oxide_library_v1_246e25db_checked_block_call(registration,(input,7),(output,)); + library::cuda_oxide_library_v1_246e25db_checked_block_resource_call(registration,slice,(input,7),(output,)); + let old = library::cuda_oxide_library_v1_246e25db_storage_new(); + let mut storage = library::cuda_oxide_library_v1_246e25db_storage_reconfigure(old); + let reservation = library::cuda_oxide_library_v1_246e25db_storage_reserve(&mut storage); + library::cuda_oxide_library_v1_246e25db_checked_block_storage_call(registration,slice,&mut storage,(input,7),(output,)); + library::cuda_oxide_library_v1_246e25db_storage_release(reservation); + library::cuda_oxide_library_v1_246e25db_call_with_callback(registration,(input,output),thunk,&7); +} +"#).unwrap(); + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let sysroot = std::process::Command::new(rustc) + .args(["--print", "sysroot"]) + .output() + .unwrap(); + assert!(sysroot.status.success()); + let args = vec![ + "rustc".to_string(), + "--edition=2024".into(), + "--crate-type=rlib".into(), + "--crate-name=cuda_device".into(), + "--emit=metadata".into(), + "-Zmir-opt-level=0".into(), + format!("--out-dir={}", root.display()), + format!( + "--sysroot={}", + String::from_utf8(sysroot.stdout).unwrap().trim() + ), + fixture.display().to_string(), + ]; + std::thread::Builder::new() + .stack_size(64 * 1024 * 1024) + .spawn(move || { + rustc_public::run!(&args, || { + let item = rustc_public::all_local_items() + .into_iter() + .find(|item| item.name() == "cuda_device::entry") + .unwrap(); + let instance = Instance::try_from(item).unwrap(); + let body = instance.body().unwrap(); + let successors: Vec<_> = body + .blocks + .iter() + .map(|b| b.terminator.successors()) + .collect(); + let source: Vec<_> = body + .blocks + .iter() + .enumerate() + .filter_map(|(bb, block)| { + let mir::TerminatorKind::Call { + func: mir::Operand::Constant(c), + .. + } = &block.terminator.kind + else { + return None; + }; + let TyKind::RigidTy(RigidTy::FnDef(def, args)) = c.const_.ty().kind() + else { + return None; + }; + Some((bb, Instance::resolve(def, &args).unwrap().mangled_name())) + }) + .collect(); + assert_eq!(source.len(), 9); + let spec = NativeSpec { + scalar: LibraryScalar::Signed(32), + items: 4, + }; + let roles = [ + LibraryNativeMetadata::Raw { + spec, + block_threads: 64, + }, + LibraryNativeMetadata::CheckedBlock { spec }, + LibraryNativeMetadata::CheckedBlockResource { + spec, + access: LibraryAccess::Read, + witness: LibraryWitness::ReadOnlySlice, + }, + ] + .map(LibraryRoleAttr::Native); + let policy = LibraryStoragePolicy { + capacity: 128, + alignment: 16, + auto_sync: false, + exclusive: true, + }; + let roles = [ + roles[0], + roles[1], + roles[2], + LibraryRoleAttr::StorageNew(LibraryStoragePolicy::default()), + LibraryRoleAttr::StorageReconfigure { + old: LibraryStoragePolicy::default(), + new: policy, + }, + LibraryRoleAttr::StorageReserve { + policy, + layout: LibraryReserveLayout { + scalar: spec.scalar, + count: 4, + bytes: 16, + alignment: 16, + }, + }, + LibraryRoleAttr::Native(LibraryNativeMetadata::CheckedBlockStorage { + spec, + access: LibraryAccess::Read, + witness: LibraryWitness::ReadOnlySlice, + storage: LibraryStorageArg::Explicit(policy), + }), + LibraryRoleAttr::StorageRelease, + LibraryRoleAttr::Native(LibraryNativeMetadata::Callback { + spec, + block_threads: 64, + }), + ]; + for (caller, debug) in [ + ("original_entry", DebugKind::Off), + ("alternate_entry", DebugKind::Full), + ] { + crate::library::set_semantics(Default::default()); + let mut ctx = Context::new(); + crate::translator::register_dialects(&mut ctx); + let error = translate_body( + &mut ctx, + &body, + &instance, + body.blocks.len(), + &successors, + false, + None, + Some(caller), + &mut Legaliser::default(), + debug, + None, + None, + false, + ) + .unwrap_err(); + assert!( + error + .to_string() + .contains("no authenticated call-site semantics"), + "{error}" + ); + let semantics = source + .iter() + .zip(roles) + .map(|((block, marker), role)| { + ( + crate::LibraryCallSite { + caller: caller.into(), + block: *block, + }, + crate::LibraryCallMetadata { + marker: marker.clone(), + role, + }, + ) + }) + .collect(); + crate::library::set_semantics(semantics); + let mut ctx = Context::new(); + crate::translator::register_dialects(&mut ctx); + let (function, _) = translate_body( + &mut ctx, + &body, + &instance, + body.blocks.len(), + &successors, + false, + None, + Some(caller), + &mut Legaliser::default(), + debug, + None, + None, + false, + ) + .unwrap(); + let mut retained = Vec::new(); + let mut boundaries = Vec::new(); + for block in function.deref(&ctx).get_region(0).deref(&ctx).iter(&ctx) { + for op in block.deref(&ctx).iter(&ctx) { + assert!(Operation::get_op::(op, &ctx).is_none()); + assert!(Operation::get_op::(op, &ctx).is_none()); + if let Some(library) = Operation::get_op::(op, &ctx) { + library.verify(&ctx).unwrap(); + let site = library.get_attr_site(&ctx).unwrap(); + assert_eq!(site.caller, caller); + let marker = String::from( + (*library.get_attr_library_callee(&ctx).unwrap()).clone(), + ); + assert_eq!( + marker, + source.iter().find(|(bb, _)| *bb == site.block).unwrap().1 + ); + if *library.get_attr_role(&ctx).unwrap() + == LibraryRoleAttr::NativeOperandBoundary + { + assert_eq!(op.deref(&ctx).get_num_operands(), 0); + assert_eq!(op.deref(&ctx).get_num_results(), 0); + boundaries.push(site.block); + continue; + } + retained.push(( + site.block, + *library.get_attr_role(&ctx).unwrap(), + op.deref(&ctx).get_num_operands(), + op.deref(&ctx).get_num_results(), + )); + } + } + } + retained.sort_by_key(|row| row.0); + assert_eq!( + retained, + source + .iter() + .zip(roles) + .zip([ + (1, 0), + (2, 0), + (3, 0), + (0, 1), + (1, 1), + (1, 1), + (4, 0), + (1, 0), + (2, 0) + ]) + .map(|(((bb, _), role), (operands, results))| ( + *bb, role, operands, results + )) + .collect::>() + ); + assert_eq!( + boundaries, + [ + source[0].0, + source[1].0, + source[2].0, + source[6].0, + source[8].0 + ] + ); + let plans = source + .iter() + .enumerate() + .map(|(index, (block, marker))| { + ( + crate::LibraryCallSite { + caller: caller.into(), + block: *block, + }, + { + let operation = match roles[index] { + LibraryRoleAttr::Native(_) => LibraryOperation::Native, + LibraryRoleAttr::StorageNew(_) => { + LibraryOperation::StorageNew + } + LibraryRoleAttr::StorageReconfigure { .. } => { + LibraryOperation::StorageReconfigure + } + LibraryRoleAttr::StorageReserve { .. } => { + LibraryOperation::StorageReserve + } + LibraryRoleAttr::StorageRelease => { + LibraryOperation::StorageRelease + } + LibraryRoleAttr::NativeOperandBoundary => { + unreachable!() + } + }; + let mut plan = crate::LibraryCallPlan::internal( + operation, + marker.clone(), + ); + if matches!(roles[index], LibraryRoleAttr::Native(_)) { + plan.symbol = format!("native_{index}"); + plan.checked_block = index != 0 && index != 8; + plan.callback = index == 8; + plan.resource_witness = index == 2 || index == 6; + plan.storage_witness = index == 6; + plan.scratch_bytes = if index == 2 { 0 } else { 32 }; + plan.scratch_alignment = 8; + } else if index == 5 { + plan.scratch_bytes = 16; + plan.scratch_alignment = 16; + } + plan + }, + ) + }) + .collect(); + mir_transforms::library::lower_planned_library_operations( + function, + &mut ctx, + &mut pliron::pass::AnalysisManager::default(), + &plans, + ) + .unwrap(); + let mut native_arguments = Vec::new(); + let mut scratch_views = 0; + for block in function.deref(&ctx).get_region(0).deref(&ctx).iter(&ctx) { + for op in block.deref(&ctx).iter(&ctx) { + assert!(Operation::get_op::(op, &ctx).is_none()); + if let Some(call) = Operation::get_op::(op, &ctx) { + call.verify(&ctx).unwrap(); + let scratch = op.deref(&ctx).get_operand(2).get_type(&ctx); + let scratch = scratch.deref(&ctx); + let scratch = scratch + .downcast_ref::() + .unwrap(); + assert_eq!(scratch.address_space, 0); + assert_eq!( + scratch.kind, + dialect_mir::types::MirPointerKind::RawMut + ); + native_arguments.push(op.deref(&ctx).get_num_operands()); + } + if let Some(view) = Operation::get_op::(op, &ctx) + { + assert_eq!(view.get_byte_offset_value(&ctx), 0); + scratch_views += 1; + } + } + } + assert_eq!(native_arguments, vec![3, 4, 4, 4, 4]); + assert_eq!( + scratch_views, 6, + "zero-byte native scratch still retains a fixed view" + ); + } + crate::library::set_semantics(Default::default()); + std::ops::ControlFlow::<(), _>::Continue(()) + }) + }) + .unwrap() + .join() + .unwrap() + .expect("fixture compilation succeeds"); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn foreign_signature_rejects_c_variadic_before_claiming_an_exact_abi() { assert!(ensure_foreign_callee_is_not_variadic("fixed", false).is_ok()); diff --git a/cuda-oxide/crates/mir-importer/src/translator/values.rs b/cuda-oxide/crates/mir-importer/src/translator/values.rs index 35f975f1a0..43640b82da 100644 --- a/cuda-oxide/crates/mir-importer/src/translator/values.rs +++ b/cuda-oxide/crates/mir-importer/src/translator/values.rs @@ -83,6 +83,10 @@ pub struct ValueMap { /// When set, `translate_assert` elides `AssertMessage::BoundsCheck` /// terminators (out-of-bounds indexing becomes UB, like `get_unchecked`). unchecked_indexing: bool, + /// The pipeline selected single-block-cluster semantics for this body. + pub(crate) pre90_cluster_queries: bool, + /// A compatibility query actually used those semantics. + pub(crate) used_cluster_query_fallback: bool, } impl ValueMap { @@ -94,6 +98,8 @@ impl ValueMap { debug_variables: false, direct_values: vec![None; num_locals], unchecked_indexing: false, + pre90_cluster_queries: false, + used_cluster_query_fallback: false, } } diff --git a/cuda-oxide/crates/mir-lower/src/lib.rs b/cuda-oxide/crates/mir-lower/src/lib.rs index 82fa281c4b..0e34ac7ce7 100644 --- a/cuda-oxide/crates/mir-lower/src/lib.rs +++ b/cuda-oxide/crates/mir-lower/src/lib.rs @@ -128,6 +128,7 @@ pub mod conversion_interface; pub mod convert; pub mod helpers; pub mod lowering; +pub use lowering::propagate_kernel_dynamic_shared_alignments; mod packed_shared_local_storage; pub mod scalarize_block_args; pub mod type_conversion_interface; diff --git a/cuda-oxide/crates/mir-lower/src/lowering.rs b/cuda-oxide/crates/mir-lower/src/lowering.rs index b0877b5b7e..f709627221 100644 --- a/cuda-oxide/crates/mir-lower/src/lowering.rs +++ b/cuda-oxide/crates/mir-lower/src/lowering.rs @@ -78,10 +78,7 @@ const RETURN_ABI_ALIGN_ATTR: &str = "cuda_oxide_return_abi_align"; /// shared-memory access can also live in a deeper ordinary helper. Shared /// helpers receive the maximum requirement from every marked root that can /// reach them. -pub(crate) fn propagate_kernel_dynamic_shared_alignments( - ctx: &mut Context, - module_op: Ptr, -) { +pub fn propagate_kernel_dynamic_shared_alignments(ctx: &mut Context, module_op: Ptr) { let mut functions = FxHashMap::default(); for region in module_op.deref(ctx).regions() { for block in region.deref(ctx).iter(ctx) { diff --git a/cuda-oxide/crates/mir-transforms/Cargo.toml b/cuda-oxide/crates/mir-transforms/Cargo.toml index 3d0fefb013..5b3bf74f6b 100644 --- a/cuda-oxide/crates/mir-transforms/Cargo.toml +++ b/cuda-oxide/crates/mir-transforms/Cargo.toml @@ -11,6 +11,8 @@ readme = "README.md" [dependencies] pliron = { workspace = true } dialect-mir = { workspace = true } +dialect-nvvm = { workspace = true } +oxide-artifacts = { workspace = true } reserved-oxide-symbols = { workspace = true } rustc-hash = { workspace = true } thiserror = { workspace = true } diff --git a/cuda-oxide/crates/mir-transforms/src/lib.rs b/cuda-oxide/crates/mir-transforms/src/lib.rs index 187a556927..4e9c5e04c7 100644 --- a/cuda-oxide/crates/mir-transforms/src/lib.rs +++ b/cuda-oxide/crates/mir-transforms/src/lib.rs @@ -3,17 +3,18 @@ * SPDX-License-Identifier: Apache-2.0 */ -//! Optimization passes over the `dialect-mir` IR. +//! Required library analysis and optional transforms over `dialect-mir`. //! -//! These run in the middle of cuda-oxide's pipeline: after `mem2reg` (the pass -//! that promotes memory slots to plain SSA values) and before the IR is lowered -//! to the LLVM dialect on its way to PTX. The first pass here is loop unrolling, -//! switched on by the `#[unroll]` / `#[unroll(N)]` annotation (recorded as a -//! `mir.unroll_hint` operation inside the annotated loop). More loop passes can -//! live here too. +//! The Block library proof consumes prepared SSA after `mem2reg`, or original +//! slots in Full debug, before provider execution and native ABI binding. +//! Optional passes and annotated loop unrolling run afterward, before LLVM +//! lowering. pub mod analyses; mod canonicalize; pub mod forward_compiler_result_bundles; pub mod scalarize_borrowed_aggregate_reads; pub mod unroll; + +/// Required prepared-MIR library proof and native ABI lowering. +pub mod library; diff --git a/cuda-oxide/crates/mir-transforms/src/library.rs b/cuda-oxide/crates/mir-transforms/src/library.rs new file mode 100644 index 0000000000..a65c99ea54 --- /dev/null +++ b/cuda-oxide/crates/mir-transforms/src/library.rs @@ -0,0 +1,210 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Prove prepared Block and storage calls, place storage, and lower their finalized ABI. + +use dialect_mir::{ + attributes::MirCastKindAttr, + library::{LibraryCallPlan, LibraryCallSite, LibraryRoleAttr}, + ops::{MirCastOp, MirLibraryOp, MirLibrarySourceOp, MirLibraryWriteCheckOp}, +}; +use pliron::linked_list::ContainsLinkedList; +use pliron::{ + builtin::{ + attributes::{IntegerAttr, StringAttr}, + types::{IntegerType, Signedness}, + }, + common_traits::Verify, + context::{Context, Ptr}, + graph::walkers::{IRNode, WalkConfig, uninterruptible::immutable::walk_op}, + irbuild::IRStatus, + op::Op, + operation::Operation, + pass::{AnalysisManager, PassResult}, + result::Result, + r#type::TypeHandle, + utils::apint::APInt, + value::Value, +}; +use std::{ + collections::{BTreeMap, BTreeSet}, + num::NonZeroUsize, +}; + +mod admission; +mod callback; +mod context_plans; +mod contexts; +mod planned; +mod stateful; +mod storage; +pub use admission::{admit_library_roots, capture_library_roles}; +pub use callback::admit_library_callbacks; +pub use context_plans::{LibraryAssembly, assemble_library_plans}; +pub use contexts::{materialize_library_contexts, resolve_library_call_target}; +pub use stateful::analyze_libraries; +pub use storage::{RootedNativePlans, Validation, allocate_library_storage}; +#[cfg(test)] +mod admission_tests; + +/// Consume proof events before optional transforms. An unbound native call fails. +pub fn lower_library_operations( + module: Ptr, + ctx: &mut Context, + analyses: &mut AnalysisManager, +) -> Result<()> { + lower(module, ctx, analyses, &BTreeMap::new()) +} + +/// Bind retained native calls to their authenticated provider results. +/// Validate every source site and ABI before rewriting; preserve source locations. +pub fn lower_planned_library_operations( + module: Ptr, + ctx: &mut Context, + analyses: &mut AnalysisManager, + plans: &BTreeMap, +) -> Result<()> { + lower(module, ctx, analyses, plans) +} + +fn lower( + module: Ptr, + ctx: &mut Context, + analyses: &mut AnalysisManager, + plans: &BTreeMap, +) -> Result<()> { + let mut operations = (Vec::new(), Vec::new()); + walk_op( + ctx, + &mut operations, + &WalkConfig::default(), + module, + |ctx, (retained, events), node| { + if let IRNode::Operation(op) = node { + if Operation::get_op::(op, ctx).is_some() { + retained.push(op); + } else if Operation::get_op::(op, ctx).is_some() + || Operation::get_op::(op, ctx).is_some() + { + events.push(op); + } + } + }, + ); + let (retained, events) = operations; + for &op in &events { + op.deref(ctx).verify(ctx)?; + } + let mut sites = BTreeSet::new(); + let mut boundaries = BTreeMap::new(); + let mut natives = BTreeMap::new(); + for &op in &retained { + op.deref(ctx).verify(ctx)?; + let library = MirLibraryOp::new(op); + let site = (*library.get_attr_site(ctx).unwrap()).clone(); + let role = *library.get_attr_role(ctx).unwrap(); + if role == LibraryRoleAttr::NativeOperandBoundary { + if boundaries.insert(site.clone(), op).is_some() { + return pliron::verify_err!( + op.deref(ctx).loc(), + "duplicate native operand boundary" + ); + } + } else if !sites.insert(site.clone()) { + return pliron::verify_err!( + op.deref(ctx).loc(), + "duplicate retained library source site" + ); + } + if matches!(role, LibraryRoleAttr::Native(_)) { + natives.insert(site.clone(), op); + } + if planned::operation(role).is_some() { + let Some(plan) = plans.get(&site) else { + return pliron::verify_err!( + op.deref(ctx).loc(), + "retained native/storage operation has no finalized plan" + ); + }; + planned::validate(ctx, op, plan)?; + } + } + for (site, &native) in &natives { + let Some(&boundary) = boundaries.get(site) else { + return pliron::verify_err!( + native.deref(ctx).loc(), + "native operation is missing its operand boundary" + ); + }; + let same_callee = *MirLibraryOp::new(boundary) + .get_attr_library_callee(ctx) + .unwrap() + == *MirLibraryOp::new(native) + .get_attr_library_callee(ctx) + .unwrap(); + let Some(block) = native.deref(ctx).get_parent_block() else { + return pliron::verify_err!(native.deref(ctx).loc(), "native operation is detached"); + }; + if !same_callee + || boundary.deref(ctx).get_parent_block() != Some(block) + || !block + .deref(ctx) + .iter(ctx) + .take_while(|&op| op != native) + .any(|op| op == boundary) + { + return pliron::verify_err!( + native.deref(ctx).loc(), + "native operand boundary must precede its matching call in the same block" + ); + } + } + for (site, &boundary) in &boundaries { + if !natives.contains_key(site) { + return pliron::verify_err!( + boundary.deref(ctx).loc(), + "native operand boundary is missing its call" + ); + } + } + if retained.is_empty() && events.is_empty() { + return Ok(()); + } + let mut invalidated = PassResult::default(); + invalidated.ir_changed = IRStatus::Changed; + analyses.retain_preserved(&invalidated); + for op in retained { + let site = (*MirLibraryOp::new(op).get_attr_site(ctx).unwrap()).clone(); + let role = *MirLibraryOp::new(op).get_attr_role(ctx).unwrap(); + let result = if role == LibraryRoleAttr::NativeOperandBoundary { + planned::lower_boundary(ctx, op, &plans[&site]); + None + } else { + planned::lower(ctx, op, &plans[&site]) + }; + if let Some(result) = result { + let original = op.deref(ctx).get_result(0); + original.replace_all_uses_with(ctx, &result); + } + Operation::erase(op, ctx); + } + for op in events { + Operation::erase(op, ctx); + } + Ok(()) +} + +fn emit( + ctx: &mut Context, + anchor: Ptr, + results: Vec, + operands: Vec, +) -> Ptr { + let op = Operation::new(ctx, O::get_concrete_op_info(), results, operands, vec![], 0); + let loc = anchor.deref(ctx).loc().clone(); + op.deref_mut(ctx).set_loc(loc); + op.insert_before(ctx, anchor); + op +} diff --git a/cuda-oxide/crates/mir-transforms/src/library/admission.rs b/cuda-oxide/crates/mir-transforms/src/library/admission.rs new file mode 100644 index 0000000000..1a2f13cf4d --- /dev/null +++ b/cuda-oxide/crates/mir-transforms/src/library/admission.rs @@ -0,0 +1,171 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Provider-independent admission using original effects and prepared root IR. + +use dialect_mir::{library::LibraryNativeMetadata, ops::MirFuncOp}; +use dialect_mir::{ + library::{LibraryCallSite, LibraryRoleAttr}, + ops::MirLibraryOp, +}; +use pliron::builtin::op_interfaces::SymbolOpInterface; +use pliron::{ + builtin::attributes::IntegerAttr, + common_traits::Verify, + context::{Context, Ptr}, + graph::walkers::{IRNode, WalkConfig, uninterruptible::immutable::walk_op}, + operation::Operation, + result::Result, +}; +use std::collections::{BTreeMap, BTreeSet}; + +pub(super) fn functions(ctx: &Context, module: Ptr) -> BTreeMap> { + let mut functions = BTreeMap::new(); + walk_op( + ctx, + &mut functions, + &WalkConfig::default(), + module, + |ctx, functions, node| { + if let IRNode::Operation(op) = node + && let Some(function) = MirFuncOp::wrap(ctx, op) + { + functions.insert(function.get_symbol_name(ctx).to_string(), op); + } + }, + ); + functions +} + +pub(super) fn operations(ctx: &Context, function: Ptr) -> Vec> { + let mut operations = Vec::new(); + walk_op( + ctx, + &mut operations, + &WalkConfig::default(), + function, + |_, operations, node| { + if let IRNode::Operation(op) = node { + operations.push(op); + } + }, + ); + operations +} + +/// Capture genuine retained effects before preparation can remove a source site. +/// +/// `symbols` maps original caller identities to their imported physical names. +pub fn capture_library_roles( + module: Ptr, + ctx: &Context, + symbols: &BTreeMap, +) -> Result> { + let functions = functions(ctx, module); + let mut roles = BTreeMap::new(); + for (symbol, function) in functions { + for op in operations(ctx, function) { + let Some(library) = Operation::get_op::(op, ctx) else { + continue; + }; + op.deref(ctx).verify(ctx)?; + let site = (*library.get_attr_site(ctx).unwrap()).clone(); + if symbols.get(&site.caller) != Some(&symbol) { + return pliron::verify_err!( + op.deref(ctx).loc(), + "retained library site has a different source caller" + ); + } + let role = *library.get_attr_role(ctx).unwrap(); + if role == LibraryRoleAttr::NativeOperandBoundary { + continue; + } + if roles.insert(site, role).is_some() { + return pliron::verify_err!( + op.deref(ctx).loc(), + "duplicate retained library source site" + ); + } + } + } + Ok(roles) +} + +fn checked(role: LibraryRoleAttr) -> bool { + matches!( + role, + LibraryRoleAttr::Native( + LibraryNativeMetadata::CheckedBlock { .. } + | LibraryNativeMetadata::CheckedBlockResource { .. } + | LibraryNativeMetadata::CheckedBlockStorage { .. } + ) + ) || super::storage::planned_storage(role) +} + +fn dimension(ctx: &Context, function: Ptr, key: &str) -> Option { + let key = key.try_into().unwrap(); + let function = function.deref(ctx); + let value = function + .attributes + .get::(&key)? + .value() + .to_u64(); + u32::try_from(value).ok().filter(|&value| value != 0) +} + +/// Admit checked root geometry on the actual prepared module before provider I/O. +/// +/// Root reachability and `original_roles` describe the original authenticated +/// source, not an optimized call graph. Removed calls therefore cannot remove a +/// root's obligation. Surviving effects must still match that snapshot. Raw +/// contracts retain their explicit geometry and do not require a +/// checked launch contract. This does not prove uniformity or completion. +pub fn admit_library_roots( + module: Ptr, + ctx: &Context, + roots: &BTreeMap>, + symbols: &BTreeMap, + original_roles: &BTreeMap, +) -> Result> { + for (site, role) in capture_library_roles(module, ctx, symbols)? { + if original_roles.get(&site) != Some(&role) { + return pliron::verify_err!( + module.deref(ctx).loc(), + "retained library effect changed after source capture at {} block {}", + site.caller, + site.block + ); + } + } + let functions = functions(ctx, module); + let mut shapes = BTreeMap::new(); + for (root, reached) in roots { + let roles: Vec<_> = original_roles + .iter() + .filter(|(site, _)| reached.contains(&site.caller)) + .map(|(_, &role)| role) + .collect(); + if !roles.iter().copied().any(checked) { + continue; + } + let Some(&function) = symbols.get(root).and_then(|symbol| functions.get(symbol)) else { + return pliron::verify_err!( + module.deref(ctx).loc(), + "checked library root {root} has no imported function" + ); + }; + let dimensions = + ["reqntid_x", "reqntid_y", "reqntid_z"].map(|key| dimension(ctx, function, key)); + let [Some(x), Some(y), Some(z)] = dimensions else { + return pliron::verify_err!( + function.deref(ctx).loc(), + "checked block collective root {root} requires #[launch_contract(block = (...))] with an exact shape" + ); + }; + let shape = [x, y, z]; + shapes.insert(root.clone(), shape); + } + Ok(shapes) +} diff --git a/cuda-oxide/crates/mir-transforms/src/library/admission_tests.rs b/cuda-oxide/crates/mir-transforms/src/library/admission_tests.rs new file mode 100644 index 0000000000..1dea439dab --- /dev/null +++ b/cuda-oxide/crates/mir-transforms/src/library/admission_tests.rs @@ -0,0 +1,605 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use super::admission::*; +use super::context_plans::*; +use super::contexts::*; +use super::storage::{RootedNativePlans, Validation}; +use dialect_mir::{ + library::{ + LibraryCallPlan, LibraryCallSite, LibraryFunctionSourceAttr, + LibraryNativeMetadata as Native, LibraryRoleAttr, LibraryScalar, NativeSpec, + }, + ops::*, + types::{MirPointerKind, MirPtrType, MirTupleType}, +}; +use pliron::{ + basic_block::BasicBlock, + builtin::{ + attributes::{IntegerAttr, StringAttr, TypeAttr}, + op_interfaces::{SingleBlockRegionInterface, SymbolOpInterface}, + ops::ModuleOp, + types::{FunctionType, IntegerType, Signedness}, + }, + common_traits::Verify, + context::{Context, Ptr}, + graph::dominance::DomInfo, + irbuild::{ + listener::DummyListener, + rewriter::{IRRewriter, Rewriter}, + }, + location::Location, + op::Op, + operation::Operation, + pass::AnalysisManager, + r#type::TypeHandle, + utils::apint::APInt, + value::Value, +}; +use std::{ + collections::{BTreeMap, BTreeSet}, + num::NonZeroUsize, +}; +fn module(ctx: &mut Context) -> Ptr { + dialect_mir::register(ctx); + ModuleOp::new(ctx, "admission".try_into().unwrap()).get_operation() +} +fn function( + ctx: &mut Context, + module: Ptr, + name: &str, + source: &str, + results: Vec, +) -> (Ptr, Ptr) { + let ty = FunctionType::get(ctx, vec![], results); + let op = Operation::new( + ctx, + MirFuncOp::get_concrete_op_info(), + vec![], + vec![], + vec![], + 1, + ); + let f = MirFuncOp::new(ctx, op, TypeAttr::new(ty.into())); + f.set_symbol_name(ctx, name.try_into().unwrap()); + f.set_library_function_source( + ctx, + LibraryFunctionSourceAttr { + export: source.into(), + mangled: format!("mangled_{source}"), + diagnostic: source.into(), + checked_ancestor: true, + }, + ); + Operation::get_op::(module, ctx) + .unwrap() + .append_operation(ctx, op, 0); + let block = BasicBlock::new(ctx, None, vec![]); + block.insert_at_back(op.deref(ctx).get_region(0), ctx); + (op, block) +} +fn ret(ctx: &mut Context, block: Ptr, values: Vec) { + Operation::new( + ctx, + MirReturnOp::get_concrete_op_info(), + vec![], + values, + vec![], + 0, + ) + .insert_at_back(block, ctx); +} +fn shape(ctx: &mut Context, function: Ptr, dimensions: [u32; 3]) { + let ty = IntegerType::get(ctx, 32, Signedness::Unsigned); + for (key, value) in ["reqntid_x", "reqntid_y", "reqntid_z"] + .into_iter() + .zip(dimensions) + { + function.deref_mut(ctx).attributes.set( + key.try_into().unwrap(), + IntegerAttr::new( + ty, + APInt::from_u64(value.into(), NonZeroUsize::new(32).unwrap()), + ), + ); + } +} +fn site(caller: &str, block: usize) -> LibraryCallSite { + LibraryCallSite { + caller: caller.into(), + block, + } +} +const SPEC: NativeSpec = NativeSpec { + scalar: LibraryScalar::Signed(32), + items: 4, +}; +#[allow(clippy::disallowed_methods)] +fn native(ctx: &mut Context, block: Ptr, caller: &str, kind: Native) -> Ptr { + let scalar: TypeHandle = IntegerType::get(ctx, 32, Signedness::Signed).into(); + let pointer = + MirPtrType::get_generic_with_kind(ctx, scalar, true, MirPointerKind::RawMut).into(); + let tuple = MirTupleType::get(ctx, vec![pointer]).into(); + let args = match kind { + Native::Raw { .. } => vec![tuple], + Native::Callback { .. } => vec![ + tuple, + MirPtrType::get_generic_with_kind(ctx, scalar, false, MirPointerKind::SharedRef).into(), + ], + _ => vec![tuple, tuple], + }; + let values = args + .iter() + .map(|&ty| { + let op = Operation::new( + ctx, + MirUndefOp::get_concrete_op_info(), + vec![ty], + vec![], + vec![], + 0, + ); + op.insert_at_back(block, ctx); + op.deref(ctx).get_result(0) + }) + .collect(); + let signature = FunctionType::get(ctx, args, vec![]); + let op = MirLibraryOp::build_call( + ctx, + LibraryRoleAttr::Native(kind), + site(caller, 7), + "genuine_marker".into(), + signature, + values, + ); + op.insert_at_back(block, ctx); + op +} +#[test] +fn admission_keeps_shared_roots_separate_and_original_effects_after_preparation() { + let ctx = &mut Context::new(); + let module = module(ctx); + let (a, ab) = function(ctx, module, "root_a", "root_a", vec![]); + ret(ctx, ab, vec![]); + let (b, bb) = function(ctx, module, "root_b", "root_b", vec![]); + ret(ctx, bb, vec![]); + let (_, hb) = function(ctx, module, "physical_helper", "source::helper", vec![]); + let native = native( + ctx, + hb, + "source::helper", + Native::CheckedBlock { spec: SPEC }, + ); + ret(ctx, hb, vec![]); + let symbols = BTreeMap::from([ + ("root_a".into(), "root_a".into()), + ("root_b".into(), "root_b".into()), + ("source::helper".into(), "physical_helper".into()), + ]); + let roots = BTreeMap::from([ + ( + "root_a".into(), + BTreeSet::from(["root_a".into(), "source::helper".into()]), + ), + ( + "root_b".into(), + BTreeSet::from(["root_b".into(), "source::helper".into()]), + ), + ]); + let roles = capture_library_roles(module, ctx, &symbols).unwrap(); + shape(ctx, a, [8, 4, 2]); + assert!(admit_library_roots(module, ctx, &roots, &symbols, &roles).is_err()); + shape(ctx, b, [33, 1, 1]); + assert_eq!( + admit_library_roots(module, ctx, &roots, &symbols, &roles).unwrap(), + BTreeMap::from([("root_a".into(), [8, 4, 2]), ("root_b".into(), [33, 1, 1])]) + ); + IRRewriter::::default().erase_operation(ctx, native); + shape(ctx, b, [0, 1, 1]); + assert!(admit_library_roots(module, ctx, &roots, &symbols, &roles).is_err()); +} +#[test] +fn raw_contract_keeps_its_explicit_shape_and_authenticates_surviving_effects() { + let ctx = &mut Context::new(); + let module = module(ctx); + let (_, block) = function(ctx, module, "raw", "raw", vec![]); + let op = native( + ctx, + block, + "raw", + Native::Raw { + spec: SPEC, + block_threads: 48, + }, + ); + ret(ctx, block, vec![]); + let symbols = BTreeMap::from([("raw".into(), "raw".into())]); + let roots = BTreeMap::from([("raw".into(), BTreeSet::from(["raw".into()]))]); + let roles = capture_library_roles(module, ctx, &symbols).unwrap(); + assert!( + admit_library_roots(module, ctx, &roots, &symbols, &roles) + .unwrap() + .is_empty() + ); + MirLibraryOp::new(op).set_attr_role( + ctx, + LibraryRoleAttr::Native(Native::Raw { + spec: SPEC, + block_threads: 64, + }), + ); + assert!(admit_library_roots(module, ctx, &roots, &symbols, &roles).is_err()); +} + +#[test] +fn callbacks_preserve_erased_source_obligations_and_actual_thunk_identity() { + use super::callback::admit_library_callbacks; + use dialect_mir::library::{ + LibraryCallbackCall, LibraryCallbackDefinition as D, LibraryCallbackDispatch as Dispatch, + LibraryCallbackFunction, LibraryCallbackSources, + }; + let ctx = &mut Context::new(); + let module = module(ctx); + let (_, root) = function(ctx, module, "root", "root", vec![]); + let callback = native( + ctx, + root, + "root", + Native::Callback { + spec: SPEC, + block_threads: 64, + }, + ); + ret(ctx, root, vec![]); + let (thunk, body) = function(ctx, module, "physical_thunk", "source::thunk", vec![]); + ret(ctx, body, vec![]); + let identity = "mangled_source::thunk".to_string(); + let symbols = BTreeMap::from([ + ("root".into(), "root".into()), + ("source::thunk".into(), "physical_thunk".into()), + ]); + let mut sources = LibraryCallbackSources { + roots: BTreeMap::from([(site("root", 7), identity.clone())]), + functions: BTreeMap::from([( + identity.clone(), + LibraryCallbackFunction { + diagnostic: "source::thunk".into(), + export: Some("source::thunk".into()), + definition: D::Rust, + dispatch: Dispatch::Direct, + has_body: true, + cuda_local_types: vec![], + calls: vec![], + callees: vec![], + }, + )]), + }; + assert!( + admit_library_callbacks(module, ctx, &LibraryCallbackSources::default(), &symbols).is_err() + ); + assert_eq!( + admit_library_callbacks(module, ctx, &sources, &symbols).unwrap(), + BTreeMap::from([(identity.clone(), "physical_thunk".into())]) + ); + // Preparation may remove the original call, but not its source obligations. + Operation::erase(callback, ctx); + sources + .functions + .get_mut(&identity) + .unwrap() + .cuda_local_types + .push("cuda_device::TempStorage".into()); + assert!( + admit_library_callbacks(module, ctx, &sources, &symbols) + .unwrap_err() + .contains("cannot contain CUDA resource type") + ); + sources + .functions + .get_mut(&identity) + .unwrap() + .cuda_local_types + .clear(); + // An expanded scalar intrinsic may leave no ordinary call in prepared MIR. + sources + .functions + .get_mut(&identity) + .unwrap() + .calls + .push(LibraryCallbackCall::Direct { + definition: D::Intrinsic, + diagnostic: "core::intrinsics::opaque".into(), + }); + assert!( + admit_library_callbacks(module, ctx, &sources, &symbols) + .unwrap_err() + .contains("unqualified compiler intrinsic") + ); + sources.functions.get_mut(&identity).unwrap().calls.clear(); + assert!(admit_library_callbacks(module, ctx, &sources, &symbols).is_ok()); + let f = MirFuncOp::wrap(ctx, thunk).unwrap(); + let mut source = f.library_function_source(ctx).unwrap(); + source.mangled = "different_instance".into(); + f.set_library_function_source(ctx, source); + assert!( + admit_library_callbacks(module, ctx, &sources, &symbols) + .unwrap_err() + .contains("does not match its actual definition") + ); +} + +#[test] +fn callbacks_follow_receiver_edges_and_reject_missing_or_unavailable_bodies() { + use super::callback::admit_library_callbacks; + use dialect_mir::library::{ + LibraryCallbackDefinition as D, LibraryCallbackDispatch as Dispatch, + LibraryCallbackFunction, LibraryCallbackSources, + }; + let ctx = &mut Context::new(); + let module = module(ctx); + let (_, body) = function(ctx, module, "physical_thunk", "thunk", vec![]); + ret(ctx, body, vec![]); + let class = || LibraryCallbackFunction { + diagnostic: "helper".into(), + export: None, + definition: D::Rust, + dispatch: Dispatch::Direct, + has_body: true, + cuda_local_types: vec![], + calls: vec![], + callees: vec![], + }; + let mut thunk = class(); + thunk.export = Some("thunk".into()); + // Actual receiver and shim are parallel collector edges from the caller. + thunk.callees = vec!["shim".into(), "receiver".into()]; + let mut shim = class(); + shim.dispatch = Dispatch::FunctionItemShim; + shim.has_body = false; + let mut sources = LibraryCallbackSources { + roots: BTreeMap::from([(site("root", 7), "mangled_thunk".into())]), + functions: BTreeMap::from([ + ("mangled_thunk".into(), thunk), + ("shim".into(), shim), + ("receiver".into(), class()), + ]), + }; + let symbols = BTreeMap::from([("thunk".into(), "physical_thunk".into())]); + assert!(admit_library_callbacks(module, ctx, &sources, &symbols).is_ok()); + sources.functions.get_mut("shim").unwrap().dispatch = Dispatch::ClosureOnceShim; + assert!(admit_library_callbacks(module, ctx, &sources, &symbols).is_ok()); + sources.functions.get_mut("receiver").unwrap().definition = D::Cuda; + assert!( + admit_library_callbacks(module, ctx, &sources, &symbols) + .unwrap_err() + .contains("CUDA resource or collective operation") + ); + sources.functions.remove("receiver"); + assert!( + admit_library_callbacks(module, ctx, &sources, &symbols) + .unwrap_err() + .contains("classification is missing") + ); + sources.functions.insert("receiver".into(), class()); + sources.functions.get_mut("shim").unwrap().dispatch = Dispatch::Direct; + assert!( + admit_library_callbacks(module, ctx, &sources, &symbols) + .unwrap_err() + .contains("body is unavailable") + ); + sources.functions.get_mut("shim").unwrap().dispatch = Dispatch::Indirect; + assert!( + admit_library_callbacks(module, ctx, &sources, &symbols) + .unwrap_err() + .contains("indirect call") + ); +} +#[test] +#[allow(clippy::disallowed_methods)] +fn contexts_preserve_original_calls_statics_and_cross_block_values() { + let ctx = &mut Context::new(); + let module = module(ctx); + let word: TypeHandle = IntegerType::get(ctx, 32, Signedness::Unsigned).into(); + let (helper, hb) = function(ctx, module, "physical_helper", "source::helper", vec![word]); + let loc = Location::Named { + name: "source_helper".into(), + child_loc: Box::new(Location::Unknown), + }; + helper.deref_mut(ctx).set_loc(loc.clone()); + let global_ptr = MirPtrType::get_global(ctx, word, true).into(); + let global = Operation::new( + ctx, + MirGlobalAllocOp::get_concrete_op_info(), + vec![global_ptr], + vec![], + vec![], + 0, + ); + MirGlobalAllocOp::new(global).set_attr_global_type(ctx, TypeAttr::new(word)); + MirGlobalAllocOp::new(global) + .set_attr_global_key(ctx, StringAttr::new("original_static_identity".into())); + global.insert_at_back(hb, ctx); + native( + ctx, + hb, + "source::helper", + Native::CheckedBlock { spec: SPEC }, + ) + .deref_mut(ctx) + .set_loc(loc.clone()); + let value = Operation::new( + ctx, + MirUndefOp::get_concrete_op_info(), + vec![word], + vec![], + vec![], + 0, + ); + value.insert_at_back(hb, ctx); + let successor = BasicBlock::new(ctx, None, vec![]); + successor.insert_at_back(helper.deref(ctx).get_region(0), ctx); + Operation::new( + ctx, + MirGotoOp::get_concrete_op_info(), + vec![], + vec![], + vec![successor], + 0, + ) + .insert_at_back(hb, ctx); + let cross_block_value = value.deref(ctx).get_result(0); + ret(ctx, successor, vec![cross_block_value]); + let mut calls = Vec::new(); + let mut roots = BTreeMap::new(); + let mut symbols = BTreeMap::from([("source::helper".into(), "physical_helper".into())]); + let mut targets = BTreeMap::new(); + let mut native_plans = RootedNativePlans::new(); + for name in ["a", "b"] { + let (_, block) = function(ctx, module, name, name, vec![]); + let call = Operation::new( + ctx, + MirCallOp::get_concrete_op_info(), + vec![word], + vec![], + vec![], + 0, + ); + MirCallOp::new(call).set_attr_callee(ctx, StringAttr::new("physical_helper".into())); + MirCallOp::new(call).set_attr_library_source_site(ctx, site(name, 1)); + call.insert_at_back(block, ctx); + ret(ctx, block, vec![]); + calls.push(call); + roots.insert( + name.into(), + BTreeSet::from([name.into(), "source::helper".into()]), + ); + symbols.insert(name.into(), name.into()); + targets.insert( + site(name, 1), + BTreeSet::from(["source::helper".into(), "callable_shim".into()]), + ); + native_plans.insert( + name.into(), + BTreeMap::from([( + site("source::helper", 7), + LibraryCallPlan { + symbol: format!("native_{name}"), + marker: "genuine_marker".into(), + checked_block: true, + resource_witness: false, + scratch_bytes: 16, + scratch_alignment: 4, + ..LibraryCallPlan::internal( + dialect_mir::library::LibraryOperation::Native, + "genuine_marker".into(), + ) + }, + )]), + ); + } + let validation = Validation { + shapes: BTreeMap::from([("a".into(), [64, 1, 1]), ("b".into(), [33, 1, 1])]), + ..Default::default() + }; + let assembly = assemble_library_plans( + module, + ctx, + &roots, + &targets, + &validation, + &BTreeMap::new(), + &native_plans, + BTreeMap::new(), + ) + .unwrap(); + for emitted in assembly.contexts.keys() { + symbols + .entry(emitted.clone()) + .or_insert_with(|| emitted.replace(':', "_")); + } + assert_eq!( + assembly + .contexts + .values() + .filter(|v| v.as_str() == "source::helper") + .count(), + 2 + ); + let mut analyses = AnalysisManager::default(); + analyses.compute_analysis::(helper, ctx).unwrap(); + let invalid = BTreeMap::from([(site("a", 1), BTreeSet::from(["wrong".into()]))]); + assert!( + materialize_library_contexts( + module, + ctx, + &mut analyses, + &assembly.contexts, + &symbols, + &invalid, + &BTreeMap::new(), + &BTreeMap::new(), + &assembly.redirects + ) + .is_err() + ); + assert!(analyses.try_get_analysis::(helper).is_some()); + assert!(functions(ctx, module).contains_key("physical_helper")); + materialize_library_contexts( + module, + ctx, + &mut analyses, + &assembly.contexts, + &symbols, + &targets, + &BTreeMap::new(), + &BTreeMap::new(), + &assembly.redirects, + ) + .unwrap(); + assert!(analyses.try_get_analysis::(helper).is_none()); + assert!(!functions(ctx, module).contains_key("physical_helper")); + for (name, call) in ["a", "b"].into_iter().zip(calls) { + let emitted = &assembly.redirects[name]["source::helper"]; + let physical = &symbols[emitted]; + let clone = functions(ctx, module)[physical]; + assert_eq!(clone.deref(ctx).loc(), &loc); + assert_eq!( + String::from((*MirCallOp::new(call).get_attr_callee(ctx).unwrap()).clone()), + *physical + ); + assert_eq!( + *MirCallOp::new(call) + .get_attr_library_source_site(ctx) + .unwrap(), + site(name, 1) + ); + for op in operations(ctx, clone) { + if let Some(library) = Operation::get_op::(op, ctx) { + assert_eq!(*library.get_attr_site(ctx).unwrap(), site(emitted, 7)); + } + if let Some(global) = Operation::get_op::(op, ctx) { + assert_eq!( + String::from((*global.get_attr_global_key(ctx).unwrap()).clone()), + "original_static_identity" + ); + } + } + } + assert_eq!( + resolve_library_call_target( + None, + "a", + "foreign", + &symbols, + &targets, + &BTreeMap::new(), + &BTreeMap::new(), + &assembly.redirects + ) + .unwrap(), + "foreign" + ); + module.deref(ctx).verify(ctx).unwrap(); +} diff --git a/cuda-oxide/crates/mir-transforms/src/library/callback.rs b/cuda-oxide/crates/mir-transforms/src/library/callback.rs new file mode 100644 index 0000000000..1b4bbb2226 --- /dev/null +++ b/cuda-oxide/crates/mir-transforms/src/library/callback.rs @@ -0,0 +1,168 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Bounded callback admission from original identities and retained MIR. + +use super::admission; +use dialect_mir::{ + library::{ + LibraryCallbackCall, LibraryCallbackDefinition as Definition, + LibraryCallbackDispatch as Dispatch, LibraryCallbackSources, LibraryNativeMetadata, + LibraryRoleAttr, + }, + ops::MirFuncOp, +}; +use pliron::{ + context::{Context, Ptr}, + linked_list::ContainsLinkedList, + operation::Operation, + printable::Printable, +}; +use std::collections::{BTreeMap, BTreeSet}; + +/// Admit original callback bodies before native provider execution. +/// +/// Source records include all original local-type classifications and only +/// monomorphically reachable call classifications, including erased expansions +/// and callees without imported symbols. They are identities, not safe verdicts. +/// Preparation cannot remove these obligations. This does not prove general +/// purity, termination, capture uniformity or the unsafe provider contract. +/// +/// Return selected thunk identity to physical MIR symbol. The importer applies +/// its existing exporter normalization and retains the same actual definition. +pub fn admit_library_callbacks( + module: Ptr, + ctx: &Context, + sources: &LibraryCallbackSources, + symbols: &BTreeMap, +) -> Result, String> { + let roles = admission::capture_library_roles(module, ctx, symbols) + .map_err(|error| error.disp(ctx).to_string())?; + for (site, role) in &roles { + let callback = matches!( + role, + LibraryRoleAttr::Native(LibraryNativeMetadata::Callback { .. }) + ); + if callback != sources.roots.contains_key(site) { + return Err(format!( + "retained callback source identity disagrees at {} block {}", + site.caller, site.block + )); + } + } + let functions = admission::functions(ctx, module); + let mut visited = BTreeSet::new(); + let mut pending: Vec<_> = sources.roots.values().cloned().collect(); + while let Some(identity) = pending.pop() { + if !visited.insert(identity.clone()) { + continue; + } + let function = sources.functions.get(&identity).ok_or_else(|| { + format!("scalar callback source body classification is missing for `{identity}`") + })?; + pending.extend(function.callees.iter().cloned()); + match function.definition { + Definition::Cuda => { + return Err(format!( + "scalar callback cannot reach CUDA resource or collective operation `{}`", + function.diagnostic + )); + } + Definition::Foreign => { + return Err("scalar callback cannot reach a foreign function".into()); + } + Definition::Intrinsic => { + return Err( + "scalar callback cannot reach an unqualified compiler intrinsic".into(), + ); + } + Definition::Rust => {} + } + if function.dispatch == Dispatch::Indirect { + return Err("scalar callback cannot contain an indirect call".into()); + } + if !function.has_body { + // The collector roots the actual receiver from the caller's site, + // alongside its shim edge, rather than as a child of this shim. + if matches!( + function.dispatch, + Dispatch::FunctionItemShim | Dispatch::ClosureOnceShim + ) { + continue; + } + return Err(format!( + "scalar callback body is unavailable for `{}`", + function.diagnostic + )); + } + if let Some(ty) = function.cuda_local_types.first() { + return Err(format!( + "scalar callback cannot contain CUDA resource type `{ty}`" + )); + } + for call in &function.calls { + match call { + LibraryCallbackCall::InlineAssembly => { + return Err("scalar callback cannot contain inline assembly".into()); + } + LibraryCallbackCall::Indirect => { + return Err("scalar callback cannot contain an indirect call".into()); + } + LibraryCallbackCall::Direct { + definition: Definition::Cuda | Definition::Foreign, + diagnostic, + } => { + return Err(format!( + "scalar callback cannot reach CUDA resources or foreign calls: `{diagnostic}`" + )); + } + LibraryCallbackCall::Direct { + definition: Definition::Intrinsic, + .. + } => { + return Err( + "scalar callback cannot reach an unqualified compiler intrinsic".into(), + ); + } + LibraryCallbackCall::Direct { + definition: Definition::Rust, + .. + } => {} + } + } + } + let mut retained = BTreeMap::new(); + for identity in sources.roots.values() { + let function = &sources.functions[identity]; + let export = function + .export + .as_ref() + .ok_or_else(|| format!("callback thunk `{identity}` has no collected export"))?; + let symbol = symbols + .get(export) + .ok_or_else(|| format!("callback thunk `{export}` has no imported symbol"))?; + let op = functions + .get(symbol) + .ok_or_else(|| format!("callback thunk `{export}` has no imported definition"))?; + let actual = MirFuncOp::wrap(ctx, *op) + .unwrap() + .library_function_source(ctx); + if !actual.is_some_and(|actual| actual.export == *export && actual.mangled == *identity) + || op + .deref(ctx) + .get_region(0) + .deref(ctx) + .iter(ctx) + .next() + .is_none() + { + return Err(format!( + "callback thunk `{export}` does not match its actual definition" + )); + } + retained.insert(identity.clone(), symbol.clone()); + } + Ok(retained) +} diff --git a/cuda-oxide/crates/mir-transforms/src/library/context_plans.rs b/cuda-oxide/crates/mir-transforms/src/library/context_plans.rs new file mode 100644 index 0000000000..94c175dfd5 --- /dev/null +++ b/cuda-oxide/crates/mir-transforms/src/library/context_plans.rs @@ -0,0 +1,592 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Assemble the existing contextual names and finalized plans from MIR proofs. + +use super::storage::{Plan, RootedNativePlans, Site, Validation}; +use dialect_mir::{ + library::{ + LibraryCallKind, LibraryCallPlan, LibraryCallSite, LibraryCallTargetAttr, + LibraryFunctionSourceAttr, LibraryOperation, LibraryRoleAttr, + }, + ops::{MirCallOp, MirFuncOp}, +}; +use oxide_artifacts::KernelLaunchFacts; +use pliron::{ + context::{Context, Ptr}, + graph::walkers::{IRNode, WalkConfig, uninterruptible::immutable::walk_op}, + operation::Operation, +}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +/// Owned final inputs consumed by the existing context materializer and lowerer. +#[derive(Default)] +pub struct LibraryAssembly { + /// Emitted raw export to original raw export; legalization remains importer-owned. + pub contexts: BTreeMap, + pub calls: BTreeMap, + pub redirects: BTreeMap>, + pub roots: BTreeMap, +} + +fn source<'a>( + sources: &'a BTreeMap, + export: &str, +) -> Result<&'a LibraryFunctionSourceAttr, String> { + sources + .get(export) + .ok_or_else(|| format!("library context has no authenticated source {export}")) +} + +fn insert_context( + contexts: &mut BTreeMap, + emitted: String, + original: String, +) -> Result<(), String> { + if let Some(previous) = contexts.insert(emitted, original.clone()) + && previous != original + { + return Err("finalized helper context has conflicting source identities".into()); + } + Ok(()) +} + +fn insert_call( + calls: &mut BTreeMap, + site: LibraryCallSite, + plan: LibraryCallPlan, +) -> Result<(), String> { + if let Some(previous) = calls.get(&site) { + if format!("{previous:?}") != format!("{plan:?}") { + return Err("library call site has conflicting provider specializations".into()); + } + } else { + calls.insert(site, plan); + } + Ok(()) +} + +/// Preserve the old nonstorage naming and bottom-up storage context interning. +/// +/// Provider requests retain original source reachability. Only planned storage +/// roots use trace liveness; their first trace and layout-aware plan are already +/// proved. This function performs no analysis, allocation or IR mutation. +#[allow(clippy::too_many_arguments)] +pub fn assemble_library_plans( + module: Ptr, + ctx: &Context, + rooted_functions: &BTreeMap>, + source_targets: &BTreeMap>, + validation: &Validation, + plans: &BTreeMap, + native_plans: &RootedNativePlans, + provider_roots: BTreeMap, +) -> Result { + let mut captured = ( + BTreeMap::::new(), + BTreeMap::::new(), + false, + ); + walk_op( + ctx, + &mut captured, + &WalkConfig::default(), + module, + |ctx, (sources, selected, conflict), node| { + let IRNode::Operation(op) = node else { return }; + if let Some(function) = Operation::get_op::(op, ctx) + && let Some(identity) = function.library_function_source(ctx) + && let Some(previous) = sources.insert(identity.export.clone(), identity.clone()) + && previous != identity + { + *conflict = true; + } + if let Some(call) = Operation::get_op::(op, ctx) + && let Some(site) = call.get_attr_library_source_site(ctx) + && let Some(target) = call.get_attr_library_call_target(ctx) + && let Some(previous) = selected.insert((*site).clone(), (*target).clone()) + && previous != *target + { + *conflict = true; + } + }, + ); + let (sources, selected, conflict) = captured; + if conflict { + return Err("library context has conflicting authenticated source identities".into()); + } + let mut result = LibraryAssembly { + roots: provider_roots, + ..Default::default() + }; + let roots: BTreeSet<_> = native_plans + .keys() + .chain(validation.shapes.keys()) + .collect(); + for root in roots { + if plans.contains_key(root) { + continue; + } + let mut context = BTreeMap::new(); + if validation.shapes.contains_key(root) { + let root_source = source(&sources, root)?; + for original in rooted_functions + .get(root) + .ok_or("checked root has no source reachability")? + { + let Some(identity) = sources + .get(original) + .filter(|source| source.checked_ancestor) + else { + continue; + }; + let name = if original == root { + root.clone() + } else { + format!( + "{}{}_{}_{}", + reserved_oxide_symbols::LIBRARY_CONTEXT_PREFIX, + root_source.mangled.len(), + root_source.mangled, + identity.mangled, + ) + }; + insert_context(&mut result.contexts, name.clone(), original.clone())?; + context.insert(original.clone(), name); + } + for (original, emitted) in &context { + let redirects = result.redirects.entry(emitted.clone()).or_default(); + for (site, targets) in source_targets { + if site.caller != *original { + continue; + } + for target in targets { + if let Some(target_context) = context.get(target) { + redirects.insert(target.clone(), target_context.clone()); + } + } + } + } + } + if let Some(natives) = native_plans.get(root) { + for (site, plan) in natives { + insert_call( + &mut result.calls, + LibraryCallSite { + caller: context.get(&site.caller).unwrap_or(&site.caller).clone(), + block: site.block, + }, + plan.clone(), + )?; + } + } + } + for (root, plan) in plans { + let trace = validation + .traces + .get(root) + .ok_or("storage root has no validated trace")?; + let mut calls = BTreeMap::::new(); + for (site, event) in &trace.events { + let mut call = if matches!(event.role, LibraryRoleAttr::Native(_)) { + native_plans + .get(root) + .and_then(|natives| natives.get(&event.call)) + .ok_or("missing storage provider plan")? + .clone() + } else { + LibraryCallPlan::internal( + match event.role { + LibraryRoleAttr::StorageNew(_) => LibraryOperation::StorageNew, + LibraryRoleAttr::StorageReconfigure { .. } => { + LibraryOperation::StorageReconfigure + } + LibraryRoleAttr::StorageReserve { .. } => LibraryOperation::StorageReserve, + LibraryRoleAttr::StorageRelease => LibraryOperation::StorageRelease, + _ => return Err("storage trace has an unsupported operation".into()), + }, + event.marker.clone(), + ) + }; + // Even a zero-byte native view binds fixed-prefix/resource checks. + if matches!(event.role, LibraryRoleAttr::Native(_)) { + call.planned_root = Some(root.clone()); + } + if let Some(binding) = plan.sites.get(site) { + call.scratch_offset = binding.offset; + call.scratch_bytes = binding.declared_bytes; + call.scratch_alignment = binding.alignment; + call.planned_root = Some(root.clone()); + call.barrier_before = binding.before; + call.barrier_after = binding.after; + } + calls.insert(site.clone(), call); + } + let mut names = BTreeMap::::new(); + let mut intern = HashMap::<(String, Vec<(usize, String)>), String>::new(); + let mut contexts = Vec::new(); + let mut paths = Vec::new(); + for (path, original) in &trace.contexts { + if source(&sources, original)?.checked_ancestor { + paths.push((path, original)); + } + } + paths.sort_by(|(a, _), (b, _)| b.len().cmp(&a.len()).then(a.cmp(b))); + for (path, original) in paths { + let mut bindings: Vec<_> = calls + .iter() + .filter(|(site, _)| site.len() == path.len() + 1 && site.starts_with(path)) + .map(|(site, call)| (site[path.len()], format!("{call:?}"))) + .collect(); + let children: Vec<_> = names + .iter() + .filter(|(child, _)| child.len() == path.len() + 1 && child.starts_with(path)) + .map(|(child, name)| (child[path.len()], name.clone())) + .collect(); + bindings.extend( + children + .iter() + .map(|(bb, name)| (*bb, format!("child:{name}"))), + ); + bindings.sort(); + let key = (original.clone(), bindings); + let name = if path.is_empty() { + root.clone() + } else if let Some(name) = intern.get(&key) { + name.clone() + } else { + format!( + "{}storage_{}_{}_{}", + reserved_oxide_symbols::LIBRARY_CONTEXT_PREFIX, + root.len(), + root, + contexts.len() + ) + }; + let is_new = !intern.contains_key(&key); + intern.insert(key, name.clone()); + names.insert(path.clone(), name.clone()); + if !is_new { + continue; + } + contexts.push((original, name.clone())); + insert_context(&mut result.contexts, name.clone(), original.clone())?; + for (site, call) in &calls { + if site.len() == path.len() + 1 && site.starts_with(path) { + insert_call( + &mut result.calls, + LibraryCallSite { + caller: name.clone(), + block: trace.events[site].call.block, + }, + call.clone(), + )?; + } + } + for (bb, target) in children { + let edge = selected + .get(&LibraryCallSite { + caller: original.clone(), + block: bb, + }) + .ok_or("storage child call edge is missing")?; + if edge.kind != LibraryCallKind::Direct { + return Err("storage bindings require a direct helper call".into()); + } + let mut call = + LibraryCallPlan::internal(LibraryOperation::Redirect, edge.marker.clone()); + call.symbol = target; + insert_call( + &mut result.calls, + LibraryCallSite { + caller: name.clone(), + block: bb, + }, + call, + )?; + } + } + let shape = validation + .shapes + .get(root) + .ok_or("storage root has no admitted shape")?; + result.roots.insert( + root.clone(), + KernelLaunchFacts { + exact_block: Some((shape[0], shape[1], shape[2])), + dynamic_shared_bytes: plan.bytes, + dynamic_shared_alignment: plan.alignment, + }, + ); + } + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::super::storage::{Binding, Event, Policy, Trace}; + use super::*; + use dialect_mir::library::{LibraryNativeMetadata, LibraryScalar, NativeSpec}; + use pliron::{ + basic_block::BasicBlock, + builtin::{ + attributes::{StringAttr, TypeAttr}, + op_interfaces::{SingleBlockRegionInterface, SymbolOpInterface}, + ops::ModuleOp, + types::FunctionType, + }, + op::Op, + }; + + fn function(ctx: &mut Context, module: Ptr, raw: &str) -> Ptr { + let ty = FunctionType::get(ctx, vec![], vec![]); + let op = Operation::new( + ctx, + MirFuncOp::get_concrete_op_info(), + vec![], + vec![], + vec![], + 1, + ); + let function = MirFuncOp::new(ctx, op, TypeAttr::new(ty.into())); + function.set_symbol_name(ctx, format!("physical_{raw}").try_into().unwrap()); + function.set_library_function_source( + ctx, + LibraryFunctionSourceAttr { + export: raw.into(), + mangled: format!("mangled_{raw}"), + diagnostic: raw.into(), + checked_ancestor: true, + }, + ); + Operation::get_op::(module, ctx) + .unwrap() + .append_operation(ctx, op, 0); + let block = BasicBlock::new(ctx, None, vec![]); + block.insert_at_back(op.deref(ctx).get_region(0), ctx); + block + } + + fn site(caller: &str, block: usize) -> LibraryCallSite { + LibraryCallSite { + caller: caller.into(), + block, + } + } + + fn call( + ctx: &mut Context, + block: Ptr, + bb: usize, + target: &str, + kind: LibraryCallKind, + ) -> Ptr { + let op = Operation::new( + ctx, + MirCallOp::get_concrete_op_info(), + vec![], + vec![], + vec![], + 0, + ); + let call = MirCallOp::new(op); + call.set_attr_callee(ctx, StringAttr::new(format!("physical_{target}"))); + call.set_attr_library_source_site(ctx, site("root", bb)); + call.set_attr_library_call_target( + ctx, + LibraryCallTargetAttr { + kind, + export: target.into(), + marker: format!("mangled_{target}"), + }, + ); + op.insert_at_back(block, ctx); + op + } + + fn native_plan() -> LibraryCallPlan { + let mut plan = LibraryCallPlan::internal(LibraryOperation::Native, "native_marker".into()); + plan.symbol = "native_symbol".into(); + plan.checked_block = true; + plan.scratch_alignment = 1; + plan + } + + #[test] + fn nonstorage_redirects_keep_all_source_targets_and_mangled_names() { + let ctx = &mut Context::new(); + dialect_mir::register(ctx); + let module = ModuleOp::new(ctx, "contexts".try_into().unwrap()).get_operation(); + let root_block = function(ctx, module, "root"); + function(ctx, module, "callable"); + function(ctx, module, "shim"); + call(ctx, root_block, 7, "callable", LibraryCallKind::Callable); + let reached = BTreeMap::from([( + "root".into(), + BTreeSet::from(["root".into(), "callable".into(), "shim".into()]), + )]); + let targets = BTreeMap::from([( + site("root", 7), + BTreeSet::from(["callable".into(), "shim".into()]), + )]); + let validation = Validation { + shapes: BTreeMap::from([("root".into(), [64, 1, 1])]), + ..Default::default() + }; + let natives = BTreeMap::from([( + "root".into(), + BTreeMap::from([(site("shim", 3), native_plan())]), + )]); + let result = assemble_library_plans( + module, + ctx, + &reached, + &targets, + &validation, + &BTreeMap::new(), + &natives, + BTreeMap::new(), + ) + .unwrap(); + let expected = format!( + "{}12_mangled_root_mangled_shim", + reserved_oxide_symbols::LIBRARY_CONTEXT_PREFIX + ); + assert_eq!(result.redirects["root"].len(), 2); + assert_eq!(result.redirects["root"]["shim"], expected); + assert_eq!(result.contexts[&expected], "shim"); + assert_eq!(result.calls[&site(&expected, 3)].symbol, "native_symbol"); + } + + #[test] + fn storage_interning_uses_final_bindings_and_keeps_zero_layout_root() { + let ctx = &mut Context::new(); + dialect_mir::register(ctx); + let module = ModuleOp::new(ctx, "storage_contexts".try_into().unwrap()).get_operation(); + let root_block = function(ctx, module, "root"); + function(ctx, module, "helper"); + let first = call(ctx, root_block, 1, "helper", LibraryCallKind::Direct); + call(ctx, root_block, 2, "helper", LibraryCallKind::Direct); + call(ctx, root_block, 3, "helper", LibraryCallKind::Direct); + let role = LibraryRoleAttr::Native(LibraryNativeMetadata::CheckedBlockStorage { + spec: NativeSpec { + scalar: LibraryScalar::Signed(32), + items: 1, + }, + storage: dialect_mir::library::LibraryStorageArg::Implicit, + access: dialect_mir::library::LibraryAccess::Read, + witness: dialect_mir::library::LibraryWitness::ReadOnlySlice, + }); + let event = |caller, bb| Event { + call: site(caller, bb), + role, + marker: "native_marker".into(), + origin: None, + policy: Policy::default(), + }; + let trace = Trace { + contexts: BTreeMap::from([ + (vec![], "root".into()), + (vec![1], "helper".into()), + (vec![2], "helper".into()), + (vec![3], "helper".into()), + ]), + events: BTreeMap::from([ + (vec![1, 4], event("helper", 4)), + (vec![2, 4], event("helper", 4)), + (vec![3, 4], event("helper", 4)), + (vec![9], event("root", 9)), + ]), + }; + let validation = Validation { + shapes: BTreeMap::from([("root".into(), [64, 1, 1])]), + traces: BTreeMap::from([("root".into(), trace)]), + }; + let binding = |offset| Binding { + offset, + declared_bytes: 16, + bytes: 16, + alignment: 16, + before: true, + ..Default::default() + }; + let plans = BTreeMap::from([( + "root".into(), + Plan { + sites: BTreeMap::from([ + (vec![1, 4], binding(0)), + (vec![2, 4], binding(0)), + (vec![3, 4], binding(16)), + ]), + bytes: 32, + alignment: 16, + }, + )]); + let natives = BTreeMap::from([( + "root".into(), + BTreeMap::from([ + (site("helper", 4), native_plan()), + (site("root", 9), native_plan()), + ]), + )]); + let reached = BTreeMap::from([( + "root".into(), + BTreeSet::from(["root".into(), "helper".into()]), + )]); + let targets = [1, 2, 3] + .into_iter() + .map(|bb| (site("root", bb), BTreeSet::from(["helper".into()]))) + .collect(); + let result = assemble_library_plans( + module, + ctx, + &reached, + &targets, + &validation, + &plans, + &natives, + BTreeMap::new(), + ) + .unwrap(); + assert_eq!( + result.calls[&site("root", 1)].symbol, + result.calls[&site("root", 2)].symbol + ); + assert_ne!( + result.calls[&site("root", 1)].symbol, + result.calls[&site("root", 3)].symbol + ); + assert_eq!(result.contexts.len(), 3); + assert_eq!(result.calls[&site("root", 9)].scratch_bytes, 0); + assert_eq!( + result.calls[&site("root", 9)].planned_root.as_deref(), + Some("root") + ); + assert_eq!(result.roots["root"].dynamic_shared_bytes, 32); + assert_eq!(result.roots["root"].dynamic_shared_alignment, 16); + let helper = &result.calls[&site("root", 3)].symbol; + assert!(result.calls[&site(helper, 4)].barrier_before); + assert_eq!(result.calls[&site(helper, 4)].scratch_offset, 16); + let mut selected = (*MirCallOp::new(first) + .get_attr_library_call_target(ctx) + .unwrap()) + .clone(); + selected.kind = LibraryCallKind::Callable; + MirCallOp::new(first).set_attr_library_call_target(ctx, selected); + let error = assemble_library_plans( + module, + ctx, + &reached, + &targets, + &validation, + &plans, + &natives, + BTreeMap::new(), + ) + .err() + .unwrap(); + assert_eq!(error, "storage bindings require a direct helper call"); + } +} diff --git a/cuda-oxide/crates/mir-transforms/src/library/contexts.rs b/cuda-oxide/crates/mir-transforms/src/library/contexts.rs new file mode 100644 index 0000000000..c6282332b3 --- /dev/null +++ b/cuda-oxide/crates/mir-transforms/src/library/contexts.rs @@ -0,0 +1,332 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Realize the existing finalized helper contexts without importing bodies again. + +use super::*; +use dialect_mir::{ + library::LibraryOperation, + ops::{MirCallOp, MirFuncOp}, +}; +use pliron::{ + builtin::op_interfaces::SymbolOpInterface, + irbuild::{ + cloning::{IrMapping, clone_operation}, + listener::DummyListener, + rewriter::IRRewriter, + }, +}; + +struct Rewrite { + op: Ptr, + callee: Option, + site: Option, +} + +struct ContextRewrite { + function: Ptr, + emitted_symbol: String, + clone: bool, + operations: Vec, +} + +/// Resolve one original ordinary call under an existing finalized context. +/// +/// Used for both actual calls and saved pre-preparation resource edges. An exact +/// call-site Redirect takes precedence over the context's callee redirects. +/// Foreign calls without a collected target remain unchanged; contextual +/// rewrites require the actual original target to be authenticated. Mangled +/// marker identities are resolved separately from collected export names. +#[allow(clippy::too_many_arguments)] +pub fn resolve_library_call_target( + source_site: Option<&LibraryCallSite>, + emitted_caller: &str, + actual_callee: &str, + symbols: &BTreeMap, + source_targets: &BTreeMap>, + marker_exports: &BTreeMap, + plans: &BTreeMap, + redirects: &BTreeMap>, +) -> std::result::Result { + let mut target = None; + let mut authenticated = false; + if let Some(site) = source_site { + if let Some(targets) = source_targets.get(site) { + authenticated = targets.iter().any(|target| { + symbols + .get(target) + .is_some_and(|symbol| symbol == actual_callee) + }); + if !authenticated { + return Err("ordinary call does not match its authenticated source targets".into()); + } + } + let emitted_site = LibraryCallSite { + caller: emitted_caller.into(), + block: site.block, + }; + if let Some(plan) = plans.get(&emitted_site) + && plan.operation == LibraryOperation::Redirect + { + if !marker_exports + .get(&plan.marker) + .and_then(|original| symbols.get(original)) + .is_some_and(|symbol| symbol == actual_callee) + { + return Err("library redirect does not match its original callee".into()); + } + target = Some(&plan.symbol); + } + } + if target.is_none() + && let Some(redirects) = redirects.get(emitted_caller) + { + target = redirects + .iter() + .find(|(original, _)| { + symbols + .get(*original) + .is_some_and(|symbol| symbol == actual_callee) + }) + .map(|(_, target)| target); + } + let Some(target) = target else { + return Ok(actual_callee.into()); + }; + if !authenticated { + return Err("library redirect requires an authenticated original call site".into()); + } + symbols + .get(target) + .cloned() + .ok_or_else(|| format!("library redirect target {target} has no legalized symbol")) +} + +/// Realize finalized context names and redirects in the same prepared module. +/// +/// `contexts` maps emitted raw names to original raw names. `symbols` contains +/// both, legalized by the importer's original Legaliser. Plans and redirects +/// retain their existing raw-name conventions. Ordinary call-site attributes +/// remain original; library site callers are rebound to the emitted context. +/// Only affected functions are cloned. Module globals, static source keys, +/// locations and debug attributes are preserved by Pliron's same-context clone. +/// All requested rewrites are checked before mutation; cache entries are then +/// invalidated before cloning. The importer separately remaps saved alignments +/// and reattaches debug-global owner names before resource completion. +#[allow(clippy::too_many_arguments)] +pub fn materialize_library_contexts( + module: Ptr, + ctx: &mut Context, + analyses: &mut AnalysisManager, + contexts: &BTreeMap, + symbols: &BTreeMap, + source_targets: &BTreeMap>, + marker_exports: &BTreeMap, + plans: &BTreeMap, + redirects: &BTreeMap>, +) -> Result<()> { + let functions = admission::functions(ctx, module); + let sources: BTreeSet<_> = contexts.values().cloned().collect(); + let mut targets: BTreeMap<_, _> = symbols + .iter() + .filter(|(source, symbol)| !sources.contains(*source) && functions.contains_key(*symbol)) + .map(|(source, _)| (source.clone(), source.clone())) + .collect(); + targets.extend(contexts.clone()); + let final_symbols: BTreeSet<_> = targets + .keys() + .filter_map(|target| symbols.get(target)) + .collect(); + let removed: BTreeSet<_> = sources + .iter() + .filter(|source| !targets.contains_key(*source)) + .filter_map(|source| symbols.get(source).cloned()) + .collect(); + let mut emitted_symbols = BTreeSet::new(); + let mut rewrites = Vec::new(); + let mut seen_redirects = BTreeSet::new(); + for (emitted, source) in &targets { + let Some(&function) = symbols.get(source).and_then(|symbol| functions.get(symbol)) else { + return pliron::verify_err!( + module.deref(ctx).loc(), + "library context source {source} has no imported function" + ); + }; + let Some(emitted_symbol) = symbols.get(emitted) else { + return pliron::verify_err!( + function.deref(ctx).loc(), + "library context {emitted} has no legalized symbol" + ); + }; + if !emitted_symbols.insert(emitted_symbol.clone()) { + return pliron::verify_err!( + function.deref(ctx).loc(), + "duplicate emitted library context symbol {emitted_symbol}" + ); + } + let clone = source != emitted; + if clone && functions.contains_key(emitted_symbol) { + return pliron::verify_err!( + function.deref(ctx).loc(), + "library context collides with existing function {emitted_symbol}" + ); + } + if clone + && function + .deref(ctx) + .attributes + .get::(&"gpu_kernel".try_into().unwrap()) + .is_some() + { + return pliron::verify_err!( + function.deref(ctx).loc(), + "library contexts cannot clone a kernel entry" + ); + } + let mut operations = Vec::new(); + for op in admission::operations(ctx, function) { + if let Some(library) = Operation::get_op::(op, ctx) { + op.deref(ctx).verify(ctx)?; + let mut site = (*library.get_attr_site(ctx).unwrap()).clone(); + if site.caller != *source { + return pliron::verify_err!( + op.deref(ctx).loc(), + "library context contains a different source caller" + ); + } + site.caller = emitted.clone(); + operations.push(Rewrite { + op, + callee: None, + site: Some(site), + }); + } + let Some(call) = Operation::get_op::(op, ctx) else { + continue; + }; + let Some(callee) = call.get_attr_callee(ctx) else { + return pliron::verify_err!( + op.deref(ctx).loc(), + "library context call has no callee" + ); + }; + let original_callee = String::from((*callee).clone()); + let source_site = call + .get_attr_library_source_site(ctx) + .map(|site| (*site).clone()); + if let Some(site) = &source_site { + if site.caller != *source { + return pliron::verify_err!( + op.deref(ctx).loc(), + "ordinary call site has a different source caller" + ); + } + let emitted_site = LibraryCallSite { + caller: emitted.clone(), + block: site.block, + }; + if let Some(plan) = plans.get(&emitted_site) + && plan.operation == LibraryOperation::Redirect + && !seen_redirects.insert(emitted_site) + { + return pliron::verify_err!( + op.deref(ctx).loc(), + "duplicate original library helper call site" + ); + } + } + let callee = match resolve_library_call_target( + source_site.as_ref(), + emitted, + &original_callee, + symbols, + source_targets, + marker_exports, + plans, + redirects, + ) { + Ok(callee) => callee, + Err(message) => return pliron::verify_err!(op.deref(ctx).loc(), "{message}"), + }; + if callee != original_callee && !final_symbols.contains(&callee) { + return pliron::verify_err!( + op.deref(ctx).loc(), + "library redirect target {callee} has no emitted function" + ); + } + if removed.contains(&callee) { + return pliron::verify_err!( + op.deref(ctx).loc(), + "ordinary call still refers to replaced library helper {callee}" + ); + } + operations.push(Rewrite { + op, + callee: Some(callee), + site: None, + }); + } + rewrites.push(ContextRewrite { + function, + emitted_symbol: emitted_symbol.clone(), + clone, + operations, + }); + } + for (site, plan) in plans { + if plan.operation == LibraryOperation::Redirect && !seen_redirects.contains(site) { + return pliron::verify_err!( + module.deref(ctx).loc(), + "finalized library redirect has no original call site {} block {}", + site.caller, + site.block + ); + } + } + if contexts.is_empty() + && redirects.is_empty() + && !plans + .values() + .any(|plan| plan.operation == LibraryOperation::Redirect) + { + return Ok(()); + } + let mut invalidated = PassResult::default(); + invalidated.ir_changed = IRStatus::Changed; + analyses.retain_preserved(&invalidated); + let mut rewriter = IRRewriter::::default(); + for rewrite in rewrites { + let mut mapping = IrMapping::new(); + let function = if rewrite.clone { + let clone = clone_operation(rewrite.function, ctx, &mut rewriter, &mut mapping); + clone.insert_before(ctx, rewrite.function); + MirFuncOp::wrap(ctx, clone) + .unwrap() + .set_symbol_name(ctx, rewrite.emitted_symbol.try_into().unwrap()); + clone + } else { + rewrite.function + }; + for update in rewrite.operations { + let op = if rewrite.clone { + mapping.lookup_op(update.op).unwrap() + } else { + update.op + }; + if let Some(callee) = update.callee { + MirCallOp::new(op).set_attr_callee(ctx, StringAttr::new(callee)); + } + if let Some(site) = update.site { + MirLibraryOp::new(op).set_attr_site(ctx, site); + } + } + // Verification of cross-context calls waits until every clone exists. + debug_assert!(function.deref(ctx).get_parent_block().is_some()); + } + for symbol in removed { + Operation::erase(functions[&symbol], ctx); + } + module.deref(ctx).verify(ctx) +} diff --git a/cuda-oxide/crates/mir-transforms/src/library/planned.rs b/cuda-oxide/crates/mir-transforms/src/library/planned.rs new file mode 100644 index 0000000000..ea8d7fce4d --- /dev/null +++ b/cuda-oxide/crates/mir-transforms/src/library/planned.rs @@ -0,0 +1,784 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Emit validated native/storage plans from original typed library operands. + +use super::*; +use dialect_mir::{ + attributes::{FieldIndexAttr, MirPointerKindAuthorityAttr}, + library::{LibraryCallPlan, LibraryNativeMetadata, LibraryOperation}, + ops::{ + MirCallOp, MirConstantOp, MirExternSharedOp, MirExtractFieldOp, MirInsertFieldOp, + MirUndefOp, + }, + types::{MirPointerKind, MirPtrType, MirStructType, MirTupleType}, +}; +use dialect_nvvm::ops::Barrier0Op; +use pliron::{ + builtin::{type_interfaces::FunctionTypeInterface, types::FunctionType}, + r#type::Typed, + verify_err, +}; + +const FIXED_SHARED_BYTES: &str = "cuda_oxide_fixed_shared_bytes"; +const PLANNED_ROOT: &str = "cuda_oxide_planned_storage_root"; + +pub(super) fn operation(role: LibraryRoleAttr) -> Option { + Some(match role { + LibraryRoleAttr::Native(_) => LibraryOperation::Native, + LibraryRoleAttr::StorageNew(_) => LibraryOperation::StorageNew, + LibraryRoleAttr::StorageReconfigure { .. } => LibraryOperation::StorageReconfigure, + LibraryRoleAttr::StorageReserve { .. } => LibraryOperation::StorageReserve, + LibraryRoleAttr::StorageRelease => LibraryOperation::StorageRelease, + _ => return None, + }) +} + +pub(super) fn validate(ctx: &Context, op: Ptr, plan: &LibraryCallPlan) -> Result<()> { + let library = MirLibraryOp::new(op); + let role = *library.get_attr_role(ctx).unwrap(); + if operation(role).as_ref() != Some(&plan.operation) + || String::from((*library.get_attr_library_callee(ctx).unwrap()).clone()) != plan.marker + { + return verify_err!( + op.deref(ctx).loc(), + "retained library role/callee does not match its finalized plan" + ); + } + if !plan.scratch_alignment.is_power_of_two() { + return verify_err!(op.deref(ctx).loc(), "invalid library scratch alignment"); + } + if let LibraryRoleAttr::Native(native) = role { + use LibraryNativeMetadata::*; + if matches!(native, Raw { .. } | Callback { .. }) { + let signature = library + .get_attr_library_signature(ctx) + .unwrap() + .get_type(ctx); + let args = signature + .deref(ctx) + .downcast_ref::() + .unwrap() + .arg_types(); + if args[0] + .deref(ctx) + .downcast_ref::() + .unwrap() + .get_types() + .len() + < 2 + { + return verify_err!( + op.deref(ctx).loc(), + "native scratch ABI requires at least two arguments" + ); + } + } + let (checked, resource, storage) = match native { + Raw { .. } | Callback { .. } => (false, false, false), + CheckedBlock { .. } => (true, false, false), + CheckedBlockResource { .. } => (true, true, false), + CheckedBlockStorage { .. } => (true, true, true), + }; + if ( + plan.checked_block, + plan.resource_witness, + plan.storage_witness, + ) != (checked, resource, storage) + || plan.callback != matches!(native, Callback { .. }) + || plan.symbol.is_empty() + { + return verify_err!( + op.deref(ctx).loc(), + "native plan does not match the retained source contract" + ); + } + if resource && !storage && plan.scratch_bytes != 0 { + return verify_err!( + op.deref(ctx).loc(), + "checked resource library calls require scratch-free artifacts" + ); + } + } + Ok(()) +} + +pub(super) fn lower( + ctx: &mut Context, + anchor: Ptr, + plan: &LibraryCallPlan, +) -> Option { + let role = *MirLibraryOp::new(anchor).get_attr_role(ctx).unwrap(); + match role { + LibraryRoleAttr::Native(native) => { + native_call(ctx, anchor, native, plan); + None + } + LibraryRoleAttr::StorageRelease => None, + _ => Some(storage(ctx, anchor, plan)), + } +} + +fn integer(ctx: &mut Context, anchor: Ptr, width: u32, value: u64) -> Value { + let ty = IntegerType::get(ctx, width, Signedness::Unsigned); + let op = emit::(ctx, anchor, vec![ty.into()], vec![]); + MirConstantOp::new(op).set_attr_value( + ctx, + IntegerAttr::new( + ty, + APInt::from_u64(value, NonZeroUsize::new(width as usize).unwrap()), + ), + ); + op.deref(ctx).get_result(0) +} + +fn pointer_cast( + ctx: &mut Context, + anchor: Ptr, + value: Value, + target: TypeHandle, + authority: MirPointerKindAuthorityAttr, +) -> Value { + if value.get_type(ctx) == target { + return value; + } + let op = emit::(ctx, anchor, vec![target], vec![value]); + let cast = MirCastOp::new(op); + cast.set_attr_cast_kind(ctx, MirCastKindAttr::PtrToPtr); + cast.set_pointer_kind_authority(ctx, authority); + op.deref(ctx).get_result(0) +} + +/// The unsafe native ABI grants only its raw scratch pointer, never a reference. +#[allow(clippy::disallowed_methods)] +fn scratch_pointer(ctx: &mut Context, pointee: TypeHandle, shared: bool) -> TypeHandle { + if shared { + MirPtrType::get_shared_with_kind(ctx, pointee, true, MirPointerKind::RawMut).into() + } else { + MirPtrType::get_generic_with_kind(ctx, pointee, true, MirPointerKind::RawMut).into() + } +} + +#[allow(clippy::disallowed_methods)] +fn environment_pointer(ctx: &mut Context, pointee: TypeHandle) -> TypeHandle { + MirPtrType::get_generic_with_kind(ctx, pointee, false, MirPointerKind::RawConst).into() +} + +fn shared_view( + ctx: &mut Context, + anchor: Ptr, + pointee: TypeHandle, + plan: &LibraryCallPlan, +) -> Value { + let ty = MirPtrType::get_shared(ctx, pointee, true).into(); + let op = emit::(ctx, anchor, vec![ty], vec![]); + let shared = MirExternSharedOp::new(op); + shared.set_byte_offset_value(ctx, u64::from(plan.scratch_offset)); + shared.set_alignment_value(ctx, u64::from(plan.scratch_alignment)); + let layout_ty = IntegerType::get(ctx, 64, Signedness::Unsigned); + op.deref_mut(ctx).attributes.set( + FIXED_SHARED_BYTES.try_into().unwrap(), + IntegerAttr::new( + layout_ty, + APInt::from_u64( + u64::from(plan.scratch_bytes), + NonZeroUsize::new(64).unwrap(), + ), + ), + ); + if let Some(root) = &plan.planned_root { + op.deref_mut(ctx).attributes.set( + PLANNED_ROOT.try_into().unwrap(), + StringAttr::new(root.clone()), + ); + } + op.deref(ctx).get_result(0) +} + +fn block_barrier(ctx: &mut Context, anchor: Ptr) { + let barrier = emit::(ctx, anchor, vec![], vec![]); + barrier.deref_mut(ctx).attributes.set( + "cuda_oxide_intrinsic_marker".try_into().unwrap(), + StringAttr::new("v1:i0034".into()), + ); +} + +fn flatten(ctx: &mut Context, anchor: Ptr, value: Value) -> Vec { + let fields = value + .get_type(ctx) + .deref(ctx) + .downcast_ref::() + .unwrap() + .get_types() + .to_vec(); + fields + .into_iter() + .enumerate() + .map(|(index, ty)| { + let op = emit::(ctx, anchor, vec![ty], vec![value]); + MirExtractFieldOp::new(op).set_attr_index(ctx, FieldIndexAttr(index as u32)); + op.deref(ctx).get_result(0) + }) + .collect() +} + +pub(super) fn lower_boundary(ctx: &mut Context, anchor: Ptr, plan: &LibraryCallPlan) { + if plan.barrier_before { + block_barrier(ctx, anchor); + } +} + +fn native_call( + ctx: &mut Context, + anchor: Ptr, + native: LibraryNativeMetadata, + plan: &LibraryCallPlan, +) { + use LibraryNativeMetadata::*; + let library = MirLibraryOp::new(anchor); + let signature = library + .get_attr_library_signature(ctx) + .unwrap() + .get_type(ctx); + let types = signature + .deref(ctx) + .downcast_ref::() + .unwrap() + .arg_types(); + let operands = anchor.deref(ctx).operands().collect::>(); + let indices: &[usize] = match native { + Raw { .. } | Callback { .. } => &[0], + CheckedBlock { .. } => &[0, 1], + CheckedBlockResource { .. } => &[1, 2], + CheckedBlockStorage { .. } => &[2, 3], + }; + let mut arguments = Vec::new(); + let mut declared = Vec::new(); + let mut uniform_len = 0; + for &index in indices { + let fields = types[index] + .deref(ctx) + .downcast_ref::() + .unwrap() + .get_types() + .to_vec(); + if uniform_len == 0 { + uniform_len = fields.len(); + } + declared.extend(fields); + arguments.extend(flatten(ctx, anchor, operands[index])); + } + if matches!( + native, + CheckedBlock { .. } | CheckedBlockResource { .. } | CheckedBlockStorage { .. } + ) { + arguments[1..].rotate_left(uniform_len - 1); + declared[1..].rotate_left(uniform_len - 1); + } + let byte = IntegerType::get(ctx, 8, Signedness::Unsigned).into(); + let mut scratch = shared_view(ctx, anchor, byte, plan); + let raw_shared = scratch_pointer(ctx, byte, true); + scratch = pointer_cast( + ctx, + anchor, + scratch, + raw_shared, + MirPointerKindAuthorityAttr::RawAddress, + ); + arguments.insert(2, scratch); + declared.insert(2, scratch_pointer(ctx, byte, false)); + if matches!(native, Callback { .. }) { + let environment = operands[1]; + let pointee = environment + .get_type(ctx) + .deref(ctx) + .downcast_ref::() + .unwrap() + .pointee; + let typed_raw = environment_pointer(ctx, pointee); + let environment = pointer_cast( + ctx, + anchor, + environment, + typed_raw, + MirPointerKindAuthorityAttr::RawAddress, + ); + let opaque = environment_pointer(ctx, byte); + // Pointee erasure belongs to this explicit unsafe callback ABI. + let cast = emit::(ctx, anchor, vec![opaque], vec![environment]); + MirCastOp::new(cast).set_attr_cast_kind(ctx, MirCastKindAttr::PtrToPtr); + arguments.push(cast.deref(ctx).get_result(0)); + declared.push(opaque); + } + for (argument, &expected) in arguments.iter_mut().zip(&declared) { + let actual = argument.get_type(ctx); + let address_only = { + let actual = actual.deref(ctx); + let expected = expected.deref(ctx); + match ( + actual.downcast_ref::(), + expected.downcast_ref::(), + ) { + (Some(actual), Some(expected)) => { + actual.pointee == expected.pointee + && actual.is_mutable == expected.is_mutable + && actual.kind == expected.kind + && actual.address_space != expected.address_space + } + _ => false, + } + }; + if address_only { + *argument = pointer_cast( + ctx, + anchor, + *argument, + expected, + MirPointerKindAuthorityAttr::AbiBoundary, + ); + } + } + let signature = FunctionType::get(ctx, declared, vec![]).into(); + let unit = MirTupleType::get(ctx, vec![]).into(); + let call = emit::(ctx, anchor, vec![unit], arguments); + let call = MirCallOp::new(call); + call.set_attr_callee(ctx, StringAttr::new(plan.symbol.clone())); + call.set_external_callee_signature(ctx, signature); + if plan.barrier_after { + block_barrier(ctx, anchor); + } +} + +fn storage(ctx: &mut Context, anchor: Ptr, plan: &LibraryCallPlan) -> Value { + let ty = anchor.deref(ctx).get_result(0).get_type(ctx); + let field = ty + .deref(ctx) + .downcast_ref::() + .unwrap() + .get_field_type(0) + .unwrap(); + let value = if plan.operation == LibraryOperation::StorageReserve { + let pointee = field + .deref(ctx) + .downcast_ref::() + .unwrap() + .pointee; + let value = shared_view(ctx, anchor, pointee, plan); + pointer_cast( + ctx, + anchor, + value, + field, + MirPointerKindAuthorityAttr::RawAddress, + ) + } else { + integer(ctx, anchor, 8, 0) + }; + let undef = emit::(ctx, anchor, vec![ty], vec![]) + .deref(ctx) + .get_result(0); + let result = emit::(ctx, anchor, vec![ty], vec![undef, value]); + MirInsertFieldOp::new(result).set_attr_insert_index(ctx, FieldIndexAttr(0)); + result.deref(ctx).get_result(0) +} + +#[cfg(test)] +#[allow(clippy::disallowed_methods)] +mod tests { + use super::*; + use dialect_mir::{ + library::{ + LibraryAccess, LibraryCallSite, LibraryReserveLayout, LibraryScalar, LibraryStorageArg, + LibraryStoragePolicy, LibraryWitness, NativeSpec, + }, + ops::MirReturnOp, + types::MirSliceType, + }; + use pliron::{ + basic_block::BasicBlock, + builtin::{op_interfaces::SingleBlockRegionInterface, ops::ModuleOp}, + linked_list::ContainsLinkedList, + }; + + fn undef(ctx: &mut Context, block: Ptr, ty: TypeHandle) -> Value { + let op = MirUndefOp::new(ctx, ty).get_operation(); + op.insert_at_back(block, ctx); + op.deref(ctx).get_result(0) + } + + fn root(ctx: &mut Context) -> (Ptr, Ptr) { + dialect_mir::register(ctx); + dialect_nvvm::register(ctx); + let module = ModuleOp::new(ctx, "planned".try_into().unwrap()); + let block = module.get_body(ctx, 0); + (module.get_operation(), block) + } + + fn origin(ctx: &Context, mut value: Value) -> (Value, usize) { + loop { + let op = value.defining_op().unwrap(); + if Operation::get_op::(op, ctx).is_some() { + value = op.deref(ctx).get_operand(0); + } else { + let extract = MirExtractFieldOp::new(op); + return ( + op.deref(ctx).get_operand(0), + extract.get_attr_index(ctx).unwrap().0 as usize, + ); + } + } + } + + #[test] + fn native_roles_preserve_argument_order_and_foreign_pointer_authority() { + use LibraryNativeMetadata::*; + let spec = NativeSpec { + scalar: LibraryScalar::Signed(32), + items: 4, + }; + let access = LibraryAccess::Read; + let witness = LibraryWitness::ReadOnlySlice; + let storage = LibraryStorageArg::Implicit; + let roles = [ + Raw { + spec, + block_threads: 64, + }, + Callback { + spec, + block_threads: 64, + }, + CheckedBlock { spec }, + CheckedBlockResource { + spec, + access, + witness, + }, + CheckedBlockStorage { + spec, + access, + witness, + storage, + }, + ]; + for native in roles { + let ctx = &mut Context::new(); + let (module, block) = root(ctx); + let scalar = IntegerType::get(ctx, 32, Signedness::Signed).into(); + let byte = IntegerType::get(ctx, 8, Signedness::Unsigned).into(); + let input = + MirPtrType::get_generic_with_kind(ctx, scalar, false, MirPointerKind::RawConst) + .into(); + let output = + MirPtrType::get_generic_with_kind(ctx, scalar, true, MirPointerKind::RawMut).into(); + let flags = + MirPtrType::get_generic_with_kind(ctx, byte, true, MirPointerKind::RawMut).into(); + let shared_output = + MirPtrType::get_shared_with_kind(ctx, scalar, true, MirPointerKind::RawMut).into(); + let uniform = MirTupleType::get(ctx, vec![input, scalar]).into(); + let payload = MirTupleType::get(ctx, vec![output, flags]).into(); + let actual_payload = MirTupleType::get(ctx, vec![shared_output, flags]).into(); + let raw = MirTupleType::get(ctx, vec![input, output, scalar]).into(); + let empty = MirTupleType::get(ctx, vec![]).into(); + let borrowed = + MirSliceType::get_with_kind(ctx, scalar, MirPointerKind::SharedRef).into(); + let environment = MirTupleType::get(ctx, vec![scalar]).into(); + let environment = MirPtrType::get_generic_with_kind( + ctx, + environment, + false, + MirPointerKind::SharedRef, + ) + .into(); + let types = match native { + Raw { .. } => vec![raw], + Callback { .. } => vec![raw, environment], + CheckedBlock { .. } => vec![uniform, payload], + CheckedBlockResource { .. } => { + vec![borrowed, uniform, payload] + } + CheckedBlockStorage { .. } => vec![borrowed, empty, uniform, payload], + }; + let site = LibraryCallSite { + caller: "root".into(), + block: 7, + }; + let boundary = MirLibraryOp::build( + ctx, + LibraryRoleAttr::NativeOperandBoundary, + site.clone(), + "marker".into(), + ); + boundary.insert_at_back(block, ctx); + let mut operand_read = None; + let operands: Vec<_> = types + .iter() + .map(|&ty| { + if ty == payload { + let address_type = MirPtrType::get_generic_with_kind( + ctx, + actual_payload, + false, + MirPointerKind::RawConst, + ) + .into(); + let address = undef(ctx, block, address_type); + let read = Operation::new( + ctx, + dialect_mir::ops::MirLoadOp::get_concrete_op_info(), + vec![actual_payload], + vec![address], + vec![], + 0, + ); + read.insert_at_back(block, ctx); + operand_read = Some(read); + read.deref(ctx).get_result(0) + } else { + undef(ctx, block, ty) + } + }) + .collect(); + let signature = FunctionType::get(ctx, types, vec![]); + let op = MirLibraryOp::build_call( + ctx, + LibraryRoleAttr::Native(native), + site.clone(), + "marker".into(), + signature, + operands.clone(), + ); + op.insert_at_back(block, ctx); + let mut plan = LibraryCallPlan::internal(LibraryOperation::Native, "marker".into()); + plan.symbol = "native_body".into(); + plan.checked_block = !matches!(native, Raw { .. } | Callback { .. }); + plan.callback = matches!(native, Callback { .. }); + plan.resource_witness = matches!( + native, + CheckedBlockResource { .. } | CheckedBlockStorage { .. } + ); + plan.storage_witness = matches!(native, CheckedBlockStorage { .. }); + plan.scratch_bytes = if matches!(native, CheckedBlockResource { .. }) { + 0 + } else { + 32 + }; + plan.scratch_alignment = 4; + plan.barrier_before = true; + let mut wrong_kind = plan.clone(); + wrong_kind.callback = !plan.callback; + assert!(validate(ctx, op, &wrong_kind).is_err()); + super::super::lower_planned_library_operations( + module, + ctx, + &mut AnalysisManager::default(), + &BTreeMap::from([(site, plan)]), + ) + .unwrap(); + let operations: Vec<_> = block.deref(ctx).iter(ctx).collect(); + assert!( + operations + .iter() + .all(|&op| Operation::get_op::(op, ctx).is_none()) + ); + if let Some(read) = operand_read { + let barrier = operations + .iter() + .position(|&op| Operation::get_op::(op, ctx).is_some()) + .unwrap(); + assert!(barrier < operations.iter().position(|&op| op == read).unwrap()); + } + let call = operations + .iter() + .find_map(|&op| Operation::get_op::(op, ctx)) + .unwrap(); + call.get_operation().deref(ctx).verify(ctx).unwrap(); + let call_op = call.get_operation(); + let expected = match native { + Raw { .. } | Callback { .. } => { + vec![(operands[0], 0), (operands[0], 1), (operands[0], 2)] + } + CheckedBlock { .. } => vec![ + (operands[0], 0), + (operands[1], 0), + (operands[1], 1), + (operands[0], 1), + ], + CheckedBlockResource { .. } => vec![ + (operands[1], 0), + (operands[2], 0), + (operands[2], 1), + (operands[1], 1), + ], + CheckedBlockStorage { .. } => vec![ + (operands[2], 0), + (operands[3], 0), + (operands[3], 1), + (operands[2], 1), + ], + }; + for (index, expected) in expected.into_iter().enumerate() { + let index = index + usize::from(index >= 2); + assert_eq!( + origin(ctx, call_op.deref(ctx).get_operand(index)), + expected, + "{native:?}" + ); + } + if matches!(native, Callback { .. }) { + assert_eq!(call_op.deref(ctx).get_num_operands(), 5); + let environment = call_op.deref(ctx).get_operand(4); + let ty = environment.get_type(ctx); + let ty = ty.deref(ctx); + let pointer = ty.downcast_ref::().unwrap(); + assert_eq!( + (pointer.pointee, pointer.kind, pointer.address_space), + (byte, MirPointerKind::RawConst, 0) + ); + assert!(!pointer.is_mutable); + let erase = environment.defining_op().unwrap(); + erase.deref(ctx).verify(ctx).unwrap(); + let raw = erase.deref(ctx).get_operand(0).defining_op().unwrap(); + raw.deref(ctx).verify(ctx).unwrap(); + assert_eq!(raw.deref(ctx).get_operand(0), operands[1]); + assert_eq!( + *MirCastOp::new(raw) + .get_attr_pointer_kind_authority(ctx) + .unwrap(), + MirPointerKindAuthorityAttr::RawAddress + ); + } + let scratch = call_op.deref(ctx).get_operand(2); + let cast = MirCastOp::new(scratch.defining_op().unwrap()); + assert_eq!( + cast.get_attr_pointer_kind_authority(ctx) + .map(|a| (*a).clone()), + Some(MirPointerKindAuthorityAttr::AbiBoundary) + ); + let scratch_type = scratch.get_type(ctx); + let pointer = scratch_type.deref(ctx); + let pointer = pointer.downcast_ref::().unwrap(); + assert_eq!( + (pointer.address_space, pointer.kind), + (0, MirPointerKind::RawMut) + ); + } + } + + #[test] + fn reservation_preserves_typed_result_and_release_adds_no_barrier() { + let ctx = &mut Context::new(); + let (module, block) = root(ctx); + let byte = IntegerType::get(ctx, 8, Signedness::Unsigned).into(); + let scalar = IntegerType::get(ctx, 32, Signedness::Signed).into(); + let descriptor = + MirStructType::get(ctx, "descriptor".into(), vec!["private".into()], vec![byte]).into(); + let borrow = + MirPtrType::get_generic_with_kind(ctx, descriptor, true, MirPointerKind::UniqueRef) + .into(); + let field = + MirPtrType::get_generic_with_kind(ctx, scalar, true, MirPointerKind::RawMut).into(); + let result = MirStructType::get( + ctx, + "reservation".into(), + vec!["private".into()], + vec![field], + ) + .into(); + let borrowed = undef(ctx, block, borrow); + let policy = LibraryStoragePolicy { + capacity: 32, + alignment: 4, + auto_sync: false, + exclusive: true, + }; + let role = LibraryRoleAttr::StorageReserve { + policy, + layout: LibraryReserveLayout { + scalar: LibraryScalar::Signed(32), + count: 4, + bytes: 16, + alignment: 4, + }, + }; + let signature = FunctionType::get(ctx, vec![borrow], vec![result]); + let site = LibraryCallSite { + caller: "root".into(), + block: 1, + }; + let reserve = MirLibraryOp::build_call( + ctx, + role, + site.clone(), + "reserve".into(), + signature, + vec![borrowed], + ); + reserve.insert_at_back(block, ctx); + let token = reserve.deref(ctx).get_result(0); + let signature = FunctionType::get(ctx, vec![result], vec![]); + let release_site = LibraryCallSite { + caller: "root".into(), + block: 2, + }; + let release = MirLibraryOp::build_call( + ctx, + LibraryRoleAttr::StorageRelease, + release_site.clone(), + "release".into(), + signature, + vec![token], + ); + release.insert_at_back(block, ctx); + let ret = Operation::new( + ctx, + MirReturnOp::get_concrete_op_info(), + vec![], + vec![token], + vec![], + 0, + ); + ret.insert_at_back(block, ctx); + let mut plan = + LibraryCallPlan::internal(LibraryOperation::StorageReserve, "reserve".into()); + plan.scratch_offset = 12; + plan.scratch_bytes = 16; + plan.scratch_alignment = 4; + plan.planned_root = Some("root".into()); + let release_plan = + LibraryCallPlan::internal(LibraryOperation::StorageRelease, "release".into()); + super::super::lower_planned_library_operations( + module, + ctx, + &mut AnalysisManager::default(), + &BTreeMap::from([(site, plan), (release_site, release_plan)]), + ) + .unwrap(); + let result_value = ret.deref(ctx).get_operand(0); + assert_eq!(result_value.get_type(ctx), result); + let insert = result_value.defining_op().unwrap(); + assert!(Operation::get_op::(insert, ctx).is_some()); + let raw = insert.deref(ctx).get_operand(1); + assert_eq!(raw.get_type(ctx), field); + assert_eq!( + MirCastOp::new(raw.defining_op().unwrap()) + .get_attr_pointer_kind_authority(ctx) + .map(|a| (*a).clone()), + Some(MirPointerKindAuthorityAttr::RawAddress) + ); + let operations: Vec<_> = block.deref(ctx).iter(ctx).collect(); + let shared = operations + .iter() + .find_map(|&op| Operation::get_op::(op, ctx)) + .unwrap(); + assert_eq!(shared.get_byte_offset_value(ctx), 12); + assert!( + operations + .iter() + .all(|&op| Operation::get_op::(op, ctx).is_none() + && Operation::get_op::(op, ctx).is_none()) + ); + } +} diff --git a/cuda-oxide/crates/mir-transforms/src/library/stateful.rs b/cuda-oxide/crates/mir-transforms/src/library/stateful.rs new file mode 100644 index 0000000000..cf1b1b9fef --- /dev/null +++ b/cuda-oxide/crates/mir-transforms/src/library/stateful.rs @@ -0,0 +1,1752 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Bounded per-context collective proof over prepared MIR values and local slots. +use super::storage::{Event, Plan, Policy, Site, Trace, Validation, planned_storage}; +use dialect_mir::{ + library::{ + self, LibraryAccess, LibraryCallKind, LibraryCallSite, LibraryNativeMetadata as Native, + LibraryRoleAttr as Role, LibraryRootArgumentClass, LibrarySourceEffectKind as SourceKind, + LibraryStorageArg, LibraryWriteKind, + }, + ops::*, + types::MirPtrType, +}; +use pliron::linked_list::ContainsLinkedList; +use pliron::{ + basic_block::BasicBlock, + builtin::op_interfaces::BranchOpInterface, + common_traits::Verify, + context::{Context, Ptr}, + op::op_cast, + operation::Operation, + pass::{Analysis, AnalysisManager}, + r#type::Typed, + value::Value, +}; +use rustc_hash::FxHashMap; +use std::borrow::Cow; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +#[derive(Clone, Debug, PartialEq, Eq)] +enum Fact { + Unset, + Uniform(Domain), + Kernel(String, Domain), + Resource(String, Domain), + Descriptor(Site, Policy), + Reservation(Site), + Local(usize, Vec), + Referent(Box), + Fields(Vec), + // Fields are conditional on their presence in the active variant. The + // tag's uniformity is independent of every payload field's uniformity. + Enum { + tag_uniform: Domain, + fields: Vec, + }, + Varying, +} +impl Fact { + fn precision(&self) -> Option { + match self { + Self::Uniform(scope) | Self::Kernel(_, scope) | Self::Resource(_, scope) => { + Some(*scope) + } + Self::Descriptor(..) | Self::Reservation(..) => Some(Domain::BLOCK), + _ => None, + } + } + fn uniform_at(&self, scope: Domain) -> bool { + self.precision().is_some_and(|proved| proved.covers(scope)) + } + fn merge(&self, other: &Self) -> Self { + if self == other { + return self.clone(); + } + match (self, other) { + (Self::Unset, value) | (value, Self::Unset) => value.clone(), + (Self::Uniform(a), Self::Uniform(b)) + | (Self::Uniform(a), Self::Kernel(_, b)) + | (Self::Kernel(_, a), Self::Uniform(b)) => Self::Uniform((*a).meet(*b)), + (Self::Kernel(a, x), Self::Kernel(b, y)) if a == b => { + Self::Kernel(a.clone(), (*x).meet(*y)) + } + (Self::Resource(a, x), Self::Resource(b, y)) if a == b => { + Self::Resource(a.clone(), (*x).meet(*y)) + } + (Self::Fields(a), Self::Fields(b)) if a.len() == b.len() => { + Self::Fields(a.iter().zip(b).map(|(a, b)| a.merge(b)).collect()) + } + ( + Self::Enum { + tag_uniform: a, + fields: af, + }, + Self::Enum { + tag_uniform: b, + fields: bf, + }, + ) => Self::Enum { + tag_uniform: (*a).meet(*b), + fields: (0..af.len().max(bf.len())) + .map(|i| self.variant_field(i).merge(&other.variant_field(i))) + .collect(), + }, + ( + Self::Enum { + tag_uniform, + fields, + }, + Self::Uniform(scope) | Self::Kernel(_, scope), + ) + | ( + Self::Uniform(scope) | Self::Kernel(_, scope), + Self::Enum { + tag_uniform, + fields, + }, + ) => Self::Enum { + tag_uniform: (*tag_uniform).meet(*scope), + fields: fields + .iter() + .map(|f| f.merge(&Self::Uniform(*scope))) + .collect(), + }, + _ => Self::Varying, + } + } + fn taint(self, control: Domain) -> Self { + match self { + Self::Uniform(scope) => Self::Uniform(scope.meet(control)), + Self::Kernel(origin, scope) => Self::Kernel(origin, scope.meet(control)), + Self::Resource(origin, scope) => Self::Resource(origin, scope.meet(control)), + Self::Fields(v) => Self::Fields(v.into_iter().map(|f| f.taint(control)).collect()), + Self::Enum { + tag_uniform, + fields, + } => Self::Enum { + tag_uniform: tag_uniform.meet(control), + fields: fields.into_iter().map(|f| f.taint(control)).collect(), + }, + Self::Local(..) => self, + Self::Referent(value) => Self::Referent(Box::new(value.taint(control))), + _ if control == Domain::BLOCK => self, + _ => Self::Varying, + } + } + fn has_resource(&self) -> bool { + match self { + Self::Resource(..) | Self::Descriptor(..) | Self::Reservation(..) => true, + Self::Referent(value) => value.has_resource(), + Self::Fields(values) | Self::Enum { fields: values, .. } => { + values.iter().any(Self::has_resource) + } + _ => false, + } + } + fn has_local(&self) -> bool { + match self { + Self::Local(..) => true, + Self::Referent(value) => value.has_local(), + Self::Fields(values) | Self::Enum { fields: values, .. } => { + values.iter().any(Self::has_local) + } + _ => false, + } + } + fn invalidate_borrow(&mut self) { + match self { + Self::Resource(..) | Self::Descriptor(..) | Self::Reservation(..) => {} + Self::Referent(value) if value.has_resource() => value.invalidate_borrow(), + Self::Fields(values) if values.iter().any(Self::has_resource) => { + values.iter_mut().for_each(Self::invalidate_borrow); + } + Self::Enum { + tag_uniform, + fields, + } if fields.iter().any(Self::has_resource) => { + *tag_uniform = Domain::LANE; + fields.iter_mut().for_each(Self::invalidate_borrow); + } + _ => *self = Self::Varying, + } + } + fn descriptor(&self) -> Option<(Site, Policy)> { + match self { + Self::Descriptor(origin, policy) => Some((origin.clone(), *policy)), + Self::Referent(value) => value.descriptor(), + _ => None, + } + } + fn reservation(&self) -> Option { + match self { + Self::Reservation(site) => Some(site.clone()), + Self::Referent(value) => value.reservation(), + _ => None, + } + } + fn resource_origin(&self) -> Option<&str> { + match self { + Self::Resource(origin, _) => Some(origin), + Self::Referent(value) => value.resource_origin(), + _ => None, + } + } + fn invalidate_referents(&mut self) { + match self { + Self::Referent(_) => self.invalidate_borrow(), + Self::Fields(fields) | Self::Enum { fields, .. } => { + fields.iter_mut().for_each(Self::invalidate_referents); + } + _ => {} + } + } + fn field(&self, index: usize) -> Self { + match self { + Self::Fields(v) => v.get(index).cloned().unwrap_or(Self::Varying), + Self::Kernel(origin, scope) | Self::Resource(origin, scope) => { + Self::Kernel(format!("{origin}.{index}"), *scope) + } + Self::Reservation(_) => Self::Uniform(Domain::BLOCK), + Self::Uniform(scope) => Self::Uniform(*scope), + _ => Self::Varying, + } + } + fn variant_field(&self, index: usize) -> Self { + match self { + // This conditional analysis fact is only valid at enum joins or + // after a Downcast. It never reads an inactive variant's bytes. + Self::Enum { fields, .. } => fields + .get(index) + .cloned() + .unwrap_or(Self::Uniform(Domain::BLOCK)), + Self::Uniform(_) | Self::Kernel(..) | Self::Resource(..) => self.field(index), + _ => Self::Varying, + } + } +} +// This prefix has only complete Block uniformity versus lane-varying values. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Domain { + Block, + Lane, +} +impl Domain { + const BLOCK: Self = Self::Block; + const LANE: Self = Self::Lane; + fn covers(self, other: Self) -> bool { + self == Self::Block || other == Self::Lane + } + fn meet(self, other: Self) -> Self { + if self == other { self } else { Self::Lane } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Control { + domain: Domain, +} +impl Control { + const FULL: Self = Self { + domain: Domain::BLOCK, + }; + fn admits(self, domain: Domain) -> bool { + self.domain.covers(domain) + } + fn narrow(self, precision: Domain) -> Self { + Self { + domain: self.domain.meet(precision), + } + } +} +// Pending read=1/write=2 accesses retain every possible access at a join. +// Only a proved complete Block barrier clears them. +type Pending = BTreeMap; +fn merge_pending(into: &mut Pending, other: &Pending) -> bool { + let mut changed = false; + for (origin, bits) in other { + let old = into.entry(origin.clone()).or_default(); + if *old | bits != *old { + *old |= bits; + changed = true; + } + } + changed +} +#[derive(Clone, Default)] +struct Epochs { + dirty: BTreeSet, + live: BTreeMap, + used: BTreeMap>, +} +impl Epochs { + fn merge(&mut self, other: &Self) -> bool { + let before = (self.dirty.len(), self.live.len()); + self.dirty.extend(other.dirty.iter().cloned()); + self.live + .extend(other.live.iter().map(|(a, b)| (a.clone(), b.clone()))); + let mut changed = before != (self.dirty.len(), self.live.len()); + for (origin, policies) in &other.used { + let current = self.used.entry(origin.clone()).or_default(); + let before = current.len(); + current.extend(policies.iter().cloned()); + changed |= before != current.len(); + } + changed + } +} +#[derive(Clone)] +struct State { + values: FxHashMap, + slots: Vec, + borrowed: BTreeSet, + pending: Pending, + epochs: Epochs, +} +impl State { + fn fact(&self, value: Value) -> Fact { + self.values.get(&value).cloned().unwrap_or(Fact::Unset) + } + fn merge(&mut self, other: &Self) -> bool { + let mut changed = merge_pending(&mut self.pending, &other.pending); + changed |= self.epochs.merge(&other.epochs); + let n = self.borrowed.len(); + self.borrowed.extend(&other.borrowed); + changed |= n != self.borrowed.len(); + for (key, value) in &other.values { + let old = self.values.entry(*key).or_insert(Fact::Unset); + let merged = old.merge(value); + changed |= merged != *old; + *old = merged; + } + for (old, value) in self.slots.iter_mut().zip(&other.slots) { + let merged = old.merge(value); + changed |= merged != *old; + *old = merged; + } + changed + } + fn expose(&mut self, fact: &Fact) { + match fact { + Fact::Local(slot, _) => { + self.borrowed.insert(*slot); + } + Fact::Fields(fields) | Fact::Enum { fields, .. } => { + fields.iter().for_each(|f| self.expose(f)) + } + Fact::Referent(f) => self.expose(f), + _ => {} + } + } +} +fn local_read(state: &State, slot: usize, fields: &[usize]) -> Fact { + fields + .iter() + .fold(state.slots[slot].clone(), |fact, field| fact.field(*field)) +} +fn invalidate_borrows(state: &mut State) { + for (slot, fact) in state.slots.iter_mut().enumerate() { + if state.borrowed.contains(&slot) { + fact.invalidate_borrow(); + } else { + fact.invalidate_referents(); + } + } + for fact in state.values.values_mut() { + fact.invalidate_referents(); + } +} +fn complete(state: &mut State) { + state.pending.clear(); + state.epochs.dirty.clear(); +} +fn dereference(fact: Fact, state: &State) -> Fact { + match fact { + Fact::Local(slot, fields) => local_read(state, slot, &fields), + Fact::Referent(f) => *f, + _ => Fact::Varying, + } +} +fn store(address: Fact, value: Fact, state: &mut State) { + if let Fact::Local(slot, fields) = address { + write_fields(&mut state.slots[slot], &fields, value); + } else { + invalidate_borrows(state); + } +} +fn write_fields(target: &mut Fact, fields: &[usize], value: Fact) { + if let Some((index, rest)) = fields.split_first() { + if let Fact::Fields(values) = target + && let Some(field) = values.get_mut(*index) + { + write_fields(field, rest, value); + return; + } + *target = Fact::Varying; + } else { + *target = value; + } +} +// Local indices belong to one MIR body. Resolve the entire reference graph +// before crossing a call boundary; a nested Local must never index the callee. +fn call_fact(fact: Fact, state: &State) -> Fact { + fn resolve(fact: Fact, state: &State, visiting: &mut HashSet) -> Fact { + match fact { + Fact::Local(local, fields) => { + if !visiting.insert(local) { + return Fact::Varying; + } + let value = resolve(local_read(state, local, &fields), state, visiting); + visiting.remove(&local); + Fact::Referent(Box::new(value)) + } + Fact::Referent(value) => Fact::Referent(Box::new(resolve(*value, state, visiting))), + Fact::Fields(values) => Fact::Fields( + values + .into_iter() + .map(|value| resolve(value, state, visiting)) + .collect(), + ), + Fact::Enum { + tag_uniform, + fields, + } => Fact::Enum { + tag_uniform, + fields: fields + .into_iter() + .map(|value| resolve(value, state, visiting)) + .collect(), + }, + other => other, + } + } + resolve(fact, state, &mut HashSet::new()) +} + +// Worklist bottom and callee-owned addresses are not caller facts. This bounded +// analysis does not prove references newly created by a helper. +fn return_fact(fact: Fact) -> Fact { + match fact { + Fact::Unset | Fact::Local(..) => Fact::Varying, + Fact::Fields(values) => Fact::Fields(values.into_iter().map(return_fact).collect()), + Fact::Enum { + tag_uniform, + fields, + } => Fact::Enum { + tag_uniform, + fields: fields.into_iter().map(return_fact).collect(), + }, + Fact::Referent(value) => Fact::Referent(Box::new(return_fact(*value))), + other => other, + } +} +fn reachable(edges: &[Vec], start: usize, stop: Option) -> BTreeSet { + let mut result = BTreeSet::new(); + let mut pending = vec![start]; + while let Some(block) = pending.pop() { + if Some(block) != stop && result.insert(block) { + pending.extend(edges[block].iter().copied()); + } + } + result +} +// Preserve the prefix's conservative participation rule only before this +// branch's reconvergence point, not across a surrounding uniform backedge. +fn has_pre_join_cycle(edges: &[Vec], block: usize, join: Option) -> bool { + let targets: BTreeSet<_> = edges[block].iter().copied().collect(); + targets.len() > 1 + && targets.into_iter().any(|target| { + let region = reachable(edges, target, join); + region.contains(&block) + || region.iter().any(|&member| { + edges[member].iter().any(|&next| { + region.contains(&next) && reachable(edges, next, join).contains(&member) + }) + }) + }) +} +fn joins(edges: &[Vec]) -> Vec> { + let exit = edges.len(); + let all: BTreeSet<_> = (0..=exit).collect(); + let mut post = vec![all; exit + 1]; + post[exit] = BTreeSet::from([exit]); + loop { + let mut changed = false; + for block in (0..exit).rev() { + let targets = if edges[block].is_empty() { + vec![exit] + } else { + edges[block].clone() + }; + let mut next = post[targets[0]].clone(); + for target in &targets[1..] { + next = next.intersection(&post[*target]).copied().collect(); + } + next.insert(block); + if next != post[block] { + post[block] = next; + changed = true; + } + } + if !changed { + break; + } + } + (0..exit) + .map(|block| { + post[block] + .iter() + .copied() + .filter(|v| *v != block && *v != exit) + .find(|candidate| { + post[block].iter().all(|other| { + *other == block || *other == *candidate || post[*candidate].contains(other) + }) + }) + }) + .collect() +} + +// Only immutable structure is cached. Context-dependent facts are recomputed +// for each call on this same prepared module. +#[derive(Clone, Copy, PartialEq, Eq)] +enum TransferKind { + Library, + Source, + WriteCheck, + Call, + Store, + Bulk, + SetDiscriminant, + Unreachable, + NoResult, + Alloca, + Load, + Ref, + FieldAddr, + ExtractField, + InsertField, + ConstructEnum, + GetDiscriminant, + EnumPayload, + Aggregate, + Constant, + Undef, + Cast, + PointerOffset, + ArrayElementAddr, + ExtractArrayElement, + Eq, + Ne, + Not, + CheckedArithmetic, + Scalar, + Unknown, +} + +impl TransferKind { + fn classify(ctx: &Context, op: Ptr) -> Self { + if Operation::get_op::(op, ctx).is_some() { + Self::Library + } else if Operation::get_op::(op, ctx).is_some() { + Self::Source + } else if Operation::get_op::(op, ctx).is_some() { + Self::WriteCheck + } else if Operation::get_op::(op, ctx).is_some() { + Self::Call + } else if Operation::get_op::(op, ctx).is_some() { + Self::Store + } else if Operation::get_op::(op, ctx).is_some() + || Operation::get_op::(op, ctx).is_some() + { + Self::Bulk + } else if Operation::get_op::(op, ctx).is_some() { + Self::SetDiscriminant + } else if Operation::get_op::(op, ctx).is_some() { + Self::Unreachable + } else if op.deref(ctx).get_num_results() == 0 { + Self::NoResult + } else if Operation::get_op::(op, ctx).is_some() { + Self::Alloca + } else if Operation::get_op::(op, ctx).is_some() { + Self::Load + } else if Operation::get_op::(op, ctx).is_some() { + Self::Ref + } else if Operation::get_op::(op, ctx).is_some() { + Self::FieldAddr + } else if Operation::get_op::(op, ctx).is_some() { + Self::ExtractField + } else if Operation::get_op::(op, ctx).is_some() { + Self::InsertField + } else if Operation::get_op::(op, ctx).is_some() { + Self::ConstructEnum + } else if Operation::get_op::(op, ctx).is_some() { + Self::GetDiscriminant + } else if Operation::get_op::(op, ctx).is_some() { + Self::EnumPayload + } else { + match Operation::get_opid(op, ctx).to_string().as_str() { + "mir.construct_struct" + | "mir.construct_tuple" + | "mir.construct_array" + | "mir.construct_slice" + | "mir.construct_disjoint_slice" => Self::Aggregate, + "mir.constant" | "mir.float_constant" => Self::Constant, + "mir.undef" => Self::Undef, + "mir.cast" => Self::Cast, + "mir.ptr_offset" => Self::PointerOffset, + "mir.array_element_addr" => Self::ArrayElementAddr, + "mir.extract_array_element" => Self::ExtractArrayElement, + "mir.eq" => Self::Eq, + "mir.ne" => Self::Ne, + "mir.not" => Self::Not, + "mir.checked_add" | "mir.checked_sub" | "mir.checked_mul" => { + Self::CheckedArithmetic + } + "mir.add" | "mir.sub" | "mir.mul" | "mir.div" | "mir.rem" | "mir.neg" + | "mir.shr" | "mir.shl" | "mir.bitand" | "mir.bitor" | "mir.bitxor" | "mir.lt" + | "mir.le" | "mir.gt" | "mir.ge" | "mir.cmp" => Self::Scalar, + _ => Self::Unknown, + } + } + } +} + +struct PreparedOperation { + op: Ptr, + operands: Vec, + results: Vec, + kind: TransferKind, +} + +impl PreparedOperation { + fn new(ctx: &Context, op: Ptr) -> Self { + Self { + op, + operands: op.deref(ctx).operands().collect(), + results: op.deref(ctx).results().collect(), + kind: TransferKind::classify(ctx, op), + } + } +} + +struct FunctionGraph { + op: Ptr, + blocks: Vec>, + operations: Vec>, + edges: Vec>, + successor_ordinals: Vec>, + joins: Vec>, + cyclic_branches: BTreeSet, + slots: HashMap, usize>, + escaped_slots: BTreeSet, + value_count: usize, +} +struct PreparedGraph { + functions: BTreeMap, +} + +// Ordinary importer slots are not source borrows. Mark only addresses used +// beyond direct memory access, before joins or opaque operations lose Local. +fn slot_escapes(ctx: &Context, address: Value) -> bool { + let mut pending = vec![address]; + let mut visited = HashSet::new(); + while let Some(value) = pending.pop() { + if !visited.insert(value) { + continue; + } + for use_ in value.uses(ctx) { + let op = use_.user_op(); + let index = use_.find_index(ctx); + match TransferKind::classify(ctx, op) { + TransferKind::Load | TransferKind::Store | TransferKind::SetDiscriminant + if index == 0 => {} + TransferKind::WriteCheck => {} + TransferKind::FieldAddr + | TransferKind::PointerOffset + | TransferKind::ArrayElementAddr + | TransferKind::Cast + if index == 0 => + { + pending.extend(op.deref(ctx).results()); + } + _ => return true, + } + } + } + false +} + +impl Analysis for PreparedGraph { + fn name(&self) -> &str { + "prepared-library-cfg" + } + fn compute( + module: Ptr, + ctx: &Context, + _: &mut AnalysisManager, + ) -> pliron::result::Result { + let mut functions = BTreeMap::new(); + for (name, op) in super::admission::functions(ctx, module) { + if op.deref(ctx).num_regions() == 0 { + continue; + } + let blocks: Vec<_> = op.deref(ctx).get_region(0).deref(ctx).iter(ctx).collect(); + if blocks.is_empty() { + continue; + } + let indices: HashMap<_, _> = blocks.iter().enumerate().map(|(i, b)| (*b, i)).collect(); + let operations: Vec> = blocks + .iter() + .map(|b| { + b.deref(ctx) + .iter(ctx) + .map(|op| PreparedOperation::new(ctx, op)) + .collect() + }) + .collect(); + let mut edges = Vec::new(); + let mut successor_ordinals = Vec::new(); + let mut slots = HashMap::new(); + let mut value_count = 0; + for (block, ops) in blocks.iter().zip(&operations) { + value_count += block.deref(ctx).get_num_arguments(); + for operation in ops { + if matches!( + operation.kind, + TransferKind::Library | TransferKind::Source | TransferKind::WriteCheck + ) { + operation.op.deref(ctx).verify(ctx)?; + } + value_count += operation.results.len(); + if operation.kind == TransferKind::Alloca { + let n = slots.len(); + slots.insert(operation.op, n); + } + } + let Some(term) = ops.last() else { + return pliron::verify_err!( + op.deref(ctx).loc(), + "library analysis requires terminated MIR blocks" + ); + }; + let term = term.op; + let impossible = library::library_impossible_successor(ctx, term); + let mut targets = Vec::new(); + let mut ordinals = Vec::new(); + for (ordinal, target) in term.deref(ctx).successors().enumerate() { + if impossible == Some(ordinal as u32) { + continue; + } + let Some(&index) = indices.get(&target) else { + return pliron::verify_err!( + term.deref(ctx).loc(), + "library CFG successor escapes its function" + ); + }; + targets.push(index); + ordinals.push(ordinal); + } + edges.push(targets); + successor_ordinals.push(ordinals); + } + let escaped_slots = slots + .iter() + .filter_map(|(&op, &slot)| { + slot_escapes(ctx, op.deref(ctx).get_result(0)).then_some(slot) + }) + .collect(); + let joins = joins(&edges); + let cyclic_branches = (0..blocks.len()) + .filter(|&block| { + Operation::get_op::(operations[block].last().unwrap().op, ctx) + .is_some() + && has_pre_join_cycle(&edges, block, joins[block]) + }) + .collect(); + functions.insert( + name, + FunctionGraph { + op, + blocks, + operations, + edges, + successor_ordinals, + joins, + cyclic_branches, + slots, + escaped_slots, + value_count, + }, + ); + } + Ok(Self { functions }) + } +} + +fn branch_controls( + fact: Fact, + targets: &[(usize, bool)], + control: Control, +) -> Vec<(usize, Control)> { + let unique: BTreeSet<_> = targets.iter().map(|(target, _)| *target).collect(); + let restricted = if unique.len() > 1 { + control.narrow(fact.precision().unwrap_or(Domain::LANE)) + } else { + control + }; + unique + .into_iter() + .map(|target| (target, restricted)) + .collect() +} +fn resource(role: Role) -> Option { + match role { + Role::Native( + Native::CheckedBlockResource { access, .. } + | Native::CheckedBlockStorage { access, .. }, + ) => Some(access), + _ => None, + } +} +fn checked(role: Role) -> bool { + matches!( + role, + Role::Native( + Native::CheckedBlock { .. } + | Native::CheckedBlockResource { .. } + | Native::CheckedBlockStorage { .. } + ) + ) +} +fn controls_index(native: Native) -> Option { + match native { + Native::Raw { .. } | Native::Callback { .. } => None, + Native::CheckedBlock { .. } => Some(0), + Native::CheckedBlockResource { .. } => Some(1), + Native::CheckedBlockStorage { .. } => Some(2), + } +} +fn scalar_precision(mut facts: impl Iterator) -> Fact { + facts + .try_fold(Domain::BLOCK, |domain, fact| { + fact.precision().map(|p| domain.meet(p)) + }) + .map_or(Fact::Varying, Fact::Uniform) +} +fn integer(ctx: &Context, value: Value) -> Option { + let op = value.defining_op()?; + Operation::get_op::(op, ctx)? + .get_attr_value(ctx) + .map(|a| a.value().to_u64()) +} +struct Analyzer<'a> { + ctx: &'a Context, + graph: &'a PreparedGraph, + symbols: &'a BTreeMap, + root: &'a str, + resource_root: bool, + storage_root: bool, + stack: Vec, + path: Site, + trace: &'a mut Trace, + plan: Option<&'a mut Plan>, +} +impl Analyzer<'_> { + fn error(&self, function: &str, block: usize, reason: &str) -> String { + let diagnostic = self + .symbols + .get(function) + .and_then(|s| self.graph.functions.get(s)) + .and_then(|f| { + MirFuncOp::wrap(self.ctx, f.op) + .unwrap() + .library_function_source(self.ctx) + }) + .map_or_else(|| function.to_owned(), |s| s.diagnostic); + format!( + "checked block collective in {}: {reason}; {diagnostic} MIR block {block}", + self.root + ) + } + fn source_block(&self, graph: &FunctionGraph, block: usize) -> usize { + graph.operations[block] + .last() + .and_then(|term| library::library_source_site(self.ctx, term.op)) + .map_or(block, |site| site.block) + } + #[allow(clippy::too_many_arguments)] + fn analyze( + &mut self, + function: &str, + arguments: Vec, + validate: bool, + entry_control: Control, + pending: Pending, + epochs: Epochs, + ) -> Result<(Fact, Pending, Epochs), String> { + if self.stack.iter().any(|f| f == function) { + return if validate { + Err(self.error(function, 0, "recursive helper participation is unsupported")) + } else { + Ok((Fact::Varying, pending, epochs)) + }; + } + let Some(graph) = self + .symbols + .get(function) + .and_then(|s| self.graph.functions.get(s)) + else { + return if validate { + Err(self.error( + function, + 0, + "call without MIR cannot prove collective participation", + )) + } else { + Ok((Fact::Varying, pending, epochs)) + }; + }; + if MirFuncOp::wrap(self.ctx, graph.op) + .unwrap() + .library_unchecked_indexing(self.ctx) + { + return Err(self.error( + function, + 0, + "checked block collectives do not support unchecked indexing; retain runtime bounds checks", + )); + } + if validate { + self.trace + .contexts + .insert(self.path.clone(), function.to_owned()); + } + let mut entry = State { + values: FxHashMap::default(), + slots: vec![Fact::Unset; graph.slots.len()], + borrowed: graph.escaped_slots.clone(), + pending, + epochs, + }; + for (index, fact) in arguments.into_iter().enumerate() { + entry + .values + .insert(graph.blocks[0].deref(self.ctx).get_argument(index), fact); + } + let mut incoming = vec![None; graph.blocks.len()]; + incoming[0] = Some(entry); + let mut controls = vec![entry_control; graph.blocks.len()]; + let live = reachable(&graph.edges, 0, None); + let limit = graph + .blocks + .len() + .saturating_mul( + graph.value_count + + graph.slots.len() + + self.plan.as_ref().map_or(0, |p| p.sites.len() * 4) + + 4, + ) + .saturating_mul(6) + .max(8); + self.stack.push(function.to_owned()); + let mut settled = false; + for _ in 0..limit { + let mut changed = false; + for &block in &live { + let Some(mut state) = incoming[block].clone() else { + continue; + }; + self.block(function, graph, block, &mut state, controls[block], false)?; + let term = graph.operations[block].last().unwrap().op; + let successors = if Operation::get_op::(term, self.ctx).is_some() { + let targets: Vec<_> = graph.edges[block] + .iter() + .zip(&graph.successor_ordinals[block]) + .map(|(&t, &i)| (t, i == 0)) + .collect(); + let branches = branch_controls( + state.fact(term.deref(self.ctx).get_operand(0)), + &targets, + controls[block], + ); + for &(target, restriction) in &branches { + for member in reachable(&graph.edges, target, graph.joins[block]) { + let restricted = controls[member].narrow(restriction.domain); + changed |= restricted != controls[member]; + controls[member] = restricted; + } + } + branches.into_iter().map(|(t, _)| t).collect::>() + } else { + graph.edges[block].clone() + }; + for (edge, &target) in graph.edges[block].iter().enumerate() { + if !successors.contains(&target) { + continue; + } + let mut outgoing = Cow::Borrowed(&state); + let obj = Operation::get_op_dyn(term, self.ctx); + if let Some(branch) = op_cast::(obj.as_ref()) { + for (arg, index) in branch + .successor_operand_range( + self.ctx, + graph.successor_ordinals[block][edge], + ) + .enumerate() + { + let value = term.deref(self.ctx).get_operand(index); + let mut selection = controls[block].domain; + // Reconvergence does not erase a value selection. A + // conditional branch may pass different values to + // the same destination block argument on its edges. + if Operation::get_op::(term, self.ctx).is_some() + && graph.edges[block].iter().enumerate().any( + |(other, &destination)| { + destination == target && { + let other_index = branch + .successor_operand_range( + self.ctx, + graph.successor_ordinals[block][other], + ) + .nth(arg) + .unwrap(); + term.deref(self.ctx).get_operand(other_index) != value + } + }, + ) + { + selection = selection.meet( + state + .fact(term.deref(self.ctx).get_operand(0)) + .precision() + .unwrap_or(Domain::LANE), + ); + } + let fact = state.fact(value).taint(selection); + outgoing.to_mut().values.insert( + graph.blocks[target].deref(self.ctx).get_argument(arg), + fact, + ); + } + } + match &mut incoming[target] { + Some(previous) => changed |= previous.merge(&outgoing), + slot @ None => { + *slot = Some(outgoing.into_owned()); + changed = true; + } + } + } + } + if !changed { + settled = true; + break; + } + } + if !settled { + self.stack.pop(); + return Err(self.error( + function, + 0, + "value analysis did not converge in its bounded subset", + )); + } + let mut returned = Fact::Unset; + let mut returned_pending = Pending::new(); + let mut returned_epochs = Epochs::default(); + for block in live { + let term = graph.operations[block].last().unwrap().op; + let is_return = Operation::get_op::(term, self.ctx).is_some(); + // Non-return effects already flow through the settled incoming states. + // Revisit all blocks only when validating their source obligations. + if !validate && !is_return { + continue; + } + let Some(mut state) = incoming[block].clone() else { + continue; + }; + self.block( + function, + graph, + block, + &mut state, + controls[block], + validate, + )?; + if validate + && graph.cyclic_branches.contains(&block) + && !state + .fact(term.deref(self.ctx).get_operand(0)) + .uniform_at(Domain::BLOCK) + { + self.stack.pop(); + return Err(self.error( + function, + self.source_block(graph, block), + "lane-varying cycle cannot prove collective participation", + )); + } + if is_return { + let value = if term.deref(self.ctx).get_num_operands() == 0 { + Fact::Fields(vec![]) + } else { + state.fact(term.deref(self.ctx).get_operand(0)) + }; + returned = returned.merge(&return_fact(value.taint(controls[block].domain))); + merge_pending(&mut returned_pending, &state.pending); + returned_epochs.merge(&state.epochs); + } + } + self.stack.pop(); + Ok((return_fact(returned), returned_pending, returned_epochs)) + } + #[allow(clippy::too_many_arguments)] + fn library( + &mut self, + function: &str, + op: Ptr, + state: &mut State, + control: Control, + validate: bool, + ) -> Result, String> { + let library = MirLibraryOp::new(op); + let role = *library.get_attr_role(self.ctx).unwrap(); + let call = (*library.get_attr_site(self.ctx).unwrap()).clone(); + let marker = String::from((*library.get_attr_library_callee(self.ctx).unwrap()).clone()); + let mut site = self.path.clone(); + site.push(call.block); + let operands: Vec<_> = op.deref(self.ctx).operands().collect(); + let facts: Vec<_> = operands.iter().map(|v| state.fact(*v)).collect(); + let resolved: Vec<_> = facts.iter().cloned().map(|f| call_fact(f, state)).collect(); + let scope = Domain::BLOCK; + let mut origin = None; + let mut policy = Policy::default(); + let mut result = None; + let mut storage_use = false; + match role { + Role::NativeOperandBoundary => return Ok(None), + Role::StorageNew(new) => { + policy = new; + result = Some(Fact::Descriptor(site.clone(), policy)); + } + Role::StorageReconfigure { new, .. } => { + if let Some((id, _)) = resolved[0].descriptor() { + if validate + && state + .epochs + .used + .get(&id) + .is_some_and(|used| used.iter().any(|old| *old != new)) + { + return Err(self.error( + function, + call.block, + "storage policy cannot change after use", + )); + } + origin = Some(id.clone()); + policy = new; + result = Some(Fact::Descriptor(id, policy)); + } else if validate { + return Err(self.error( + function, + call.block, + "storage reconfiguration has ambiguous descriptor origin", + )); + } else { + result = Some(Fact::Varying); + } + } + Role::StorageRelease => { + if let Some(reservation) = resolved[0].reservation() { + if let Some(region) = state.epochs.live.remove(&reservation) { + state.epochs.dirty.insert(region); + } else if validate && self.plan.is_some() { + return Err(self.error( + function, + call.block, + "storage release has no live reservation", + )); + } + } else if validate { + return Err(self.error( + function, + call.block, + "storage release has ambiguous reservation origin", + )); + } + } + Role::StorageReserve { .. } => { + storage_use = true; + result = Some(Fact::Reservation(site.clone())); + } + Role::Native(Native::CheckedBlockStorage { .. }) => { + storage_use = true; + } + Role::Native(_) => {} + } + let storage_operation = matches!( + role, + Role::StorageNew(_) + | Role::StorageReconfigure { .. } + | Role::StorageReserve { .. } + | Role::StorageRelease + ) || storage_use; + if storage_operation && validate && !control.admits(Domain::BLOCK) { + return Err(self.error( + function, + call.block, + "storage operation is reached through lane-varying control flow", + )); + } + if storage_use { + let (index, implicit) = match role { + Role::Native(Native::CheckedBlockStorage { storage, .. }) => { + (1, matches!(storage, LibraryStorageArg::Implicit)) + } + _ => (0, false), + }; + if let Some((id, p)) = resolved[index].descriptor() { + origin = Some(id); + policy = p; + } else if !implicit && validate { + return Err(self.error( + function, + call.block, + "storage use has no unambiguous descriptor origin", + )); + } + if let Some(id) = &origin { + state + .epochs + .used + .entry(id.clone()) + .or_default() + .insert(policy); + } + } + if validate { + self.trace.events.insert( + site.clone(), + Event { + call: call.clone(), + role, + marker, + origin, + policy, + }, + ); + } + if let Some(binding) = self.plan.as_ref().and_then(|p| p.sites.get(&site)).cloned() { + if validate && state.epochs.live.values().any(|r| *r == binding.region) { + return Err(format!( + "checked block collective in {}: storage use overlaps a live reservation at {site:?}", + self.root + )); + } + if state.epochs.dirty.contains(&binding.region) { + if binding.implicit { + self.plan + .as_mut() + .unwrap() + .sites + .get_mut(&site) + .unwrap() + .before = true; + complete(state); + } else if validate { + return Err(format!( + "checked block collective in {}: explicit storage reuse requires a full-block barrier at {site:?}", + self.root + )); + } + } + state.epochs.dirty.insert(binding.region.clone()); + if matches!(role, Role::StorageReserve { .. }) { + state.epochs.live.insert(site.clone(), binding.region); + } + } + let Role::Native(native) = role else { + return Ok(result); + }; + if validate { + if !checked(role) + || ((self.resource_root || self.storage_root) + && resource(role).is_none() + && !storage_operation) + { + return Err(self.error(function,call.block,if self.resource_root {"untracked and resource-checked library calls cannot be mixed in one checked root"} + else if self.storage_root {"untracked and planned-storage library calls cannot be mixed in one checked root"} + else {"raw and checked library calls cannot be mixed in one checked root"})); + } + if !control.admits(scope) { + return Err(self.error( + function, + call.block, + "operation is reached through lane-varying control flow", + )); + } + if let Some(index) = controls_index(native) { + let Fact::Fields(fields) = &facts[index] else { + return Err(self.error( + function, + call.block, + "uniform argument tuple has unsupported provenance", + )); + }; + if fields.iter().any(|f| !f.uniform_at(scope)) { + return Err(self.error( + function, + call.block, + "memory carrier or scalar control is not proved block-uniform", + )); + } + } + } + if let Some(access) = resource(role) { + let witness = call_fact(facts[0].clone(), state); + if let Some(origin) = witness.resource_origin() { + let write = access == LibraryAccess::Write; + let old = state.pending.get(origin).copied().unwrap_or(0); + if validate + && (!self.storage_root || self.plan.is_some()) + && (old & 2 != 0 || (write && old != 0)) + { + return Err(self.error( + function, + call.block, + "conflicting collective resource access requires a full-block barrier", + )); + } + *state.pending.entry(origin.to_owned()).or_default() |= if write { 2 } else { 1 }; + } else if validate { + return Err(self.error( + function, + call.block, + "resource witness has no unambiguous kernel allocation origin", + )); + } + } + if self + .plan + .as_ref() + .and_then(|p| p.sites.get(&site)) + .is_some_and(|b| b.after) + { + complete(state); + } + facts.iter().for_each(|fact| state.expose(fact)); + invalidate_borrows(state); + Ok(None) + } + #[allow(clippy::too_many_arguments)] + fn block( + &mut self, + function: &str, + graph: &FunctionGraph, + block: usize, + state: &mut State, + control: Control, + validate: bool, + ) -> Result<(), String> { + let mut aborted = false; + let mut facts = Vec::new(); + for prepared in &graph.operations[block] { + let op = prepared.op; + let operands = &prepared.operands; + let kind = prepared.kind; + if kind == TransferKind::Library { + if let Some(fact) = self.library(function, op, state, control, validate)? { + for &result in &prepared.results { + state + .values + .insert(result, fact.clone().taint(control.domain)); + } + } + continue; + } + facts.clear(); + facts.extend(operands.iter().map(|&v| state.fact(v))); + if kind == TransferKind::Source { + let source = MirLibrarySourceOp::new(op); + let effect = source.get_attr_library_source_effect(self.ctx).unwrap(); + let fact = match effect.kind { + SourceKind::BlockSync => { + if validate && !control.admits(Domain::BLOCK) { + return Err(self.error( + function, + effect.site.block, + "block barrier is reached through lane-varying control flow", + )); + } + complete(state); + Fact::Fields(vec![]) + } + SourceKind::BlockQuery => Fact::Uniform(Domain::BLOCK), + SourceKind::LaneQuery => Fact::Uniform(Domain::LANE), + SourceKind::UnitHint => Fact::Fields(vec![]), + SourceKind::ScalarIntrinsic | SourceKind::PointerOffset => { + facts.first().cloned().unwrap_or(Fact::Varying) + } + SourceKind::DynamicShared => Fact::Varying, + SourceKind::Abort => { + aborted = true; + Fact::Varying + } + SourceKind::UnsupportedIntrinsic + | SourceKind::UnresolvedCall + | SourceKind::NontrivialDrop + | SourceKind::InlineAssembly => { + if validate { + return Err(self.error(function,effect.site.block,match effect.kind { + SourceKind::UnsupportedIntrinsic=>"intrinsic is outside the supported checked block subset", + SourceKind::UnresolvedCall=>"unresolved call cannot prove collective participation", + _=>"inline assembly or nontrivial destructor participation is unsupported", + })); + } + invalidate_borrows(state); + Fact::Varying + } + }; + if let Some(&value) = operands.first() { + state.values.insert(value, fact.taint(control.domain)); + } + continue; + } + if kind == TransferKind::WriteCheck { + let write = MirLibraryWriteCheckOp::new(op); + if validate && (self.resource_root || self.storage_root) { + let source = write.get_attr_library_write_source(self.ctx).unwrap(); + match source.kind { + LibraryWriteKind::Bulk => { + return Err(self.error( + function, + source.site.block, + "bulk memory mutation is unsupported in resource-checked roots", + )); + } + LibraryWriteKind::Indirect { nominal_protected } + if nominal_protected + || facts + .iter() + .any(|f| call_fact(f.clone(), state).has_resource()) => + { + return Err(self.error(function,source.site.block,"mutation through a reference to a resource descriptor is unsupported; forward or move the carrier")); + } + _ => {} + } + } + continue; + } + if kind == TransferKind::Call { + let call = MirCallOp::new(op); + let result = if call.get_attr_library_expanded_source(self.ctx).is_some() { + scalar_precision(facts.iter().cloned()) + } else if let Some(site) = call.get_attr_library_source_site(self.ctx) { + let site = (*site).clone(); + if let Some(target) = call.get_attr_library_call_target(self.ctx) { + if validate && target.kind != LibraryCallKind::Direct { + return Err(self.error( + function, + site.block, + "indirect or destructor collective context is unsupported", + )); + } + let arguments = + facts.iter().cloned().map(|f| call_fact(f, state)).collect(); + facts.iter().for_each(|f| state.expose(f)); + self.path.push(site.block); + let analyzed = self.analyze( + &target.export, + arguments, + validate, + control, + state.pending.clone(), + state.epochs.clone(), + ); + self.path.pop(); + let (result, pending, epochs) = analyzed?; + state.pending = pending; + state.epochs = epochs; + invalidate_borrows(state); + result + } else { + if validate { + return Err(self.error( + function, + site.block, + "unresolved call cannot prove collective participation", + )); + } + invalidate_borrows(state); + Fact::Varying + } + } else { + Fact::Varying + }; + for &value in &prepared.results { + state + .values + .insert(value, result.clone().taint(control.domain)); + } + continue; + } + if kind == TransferKind::Store { + let store_op = MirStoreOp::new(op); + store( + state.fact(store_op.address_opd(self.ctx)), + state + .fact(store_op.value_opd(self.ctx)) + .taint(control.domain), + state, + ); + continue; + } + if kind == TransferKind::Bulk { + // A transfer may span several fields or elements. Invalidate + // escaped allocations instead of just the destination field. + invalidate_borrows(state); + continue; + } + if kind == TransferKind::SetDiscriminant { + if let Some(address) = facts.first() { + store(address.clone(), Fact::Varying, state); + } + continue; + } + if validate && kind == TransferKind::Unreachable && !aborted { + return Err(self.error( + function, + self.source_block(graph, block), + "unreachable terminator is not a proved whole-kernel abort", + )); + } + if prepared.results.is_empty() { + continue; + } + let result = prepared.results[0]; + let fact = if let Some(&slot) = graph.slots.get(&op) { + Fact::Local(slot, vec![]) + } else if kind == TransferKind::Load { + let load = MirLoadOp::new(op); + dereference(state.fact(load.address_opd(self.ctx)), state) + } else if kind == TransferKind::Ref { + Fact::Referent(Box::new(facts[0].clone())) + } else if kind == TransferKind::FieldAddr { + let field = MirFieldAddrOp::new(op); + let index = field.get_attr_field_index(self.ctx).unwrap().0 as usize; + match &facts[0] { + Fact::Local(slot, path) => { + let mut path = path.clone(); + path.push(index); + Fact::Local(*slot, path) + } + Fact::Referent(f) => Fact::Referent(Box::new(f.field(index))), + Fact::Resource(..) | Fact::Descriptor(..) | Fact::Reservation(..) => { + Fact::Referent(Box::new(facts[0].clone())) + } + fact => Fact::Referent(Box::new(fact.field(index))), + } + } else if kind == TransferKind::ExtractField { + let field = MirExtractFieldOp::new(op); + facts[0].field(field.get_attr_index(self.ctx).unwrap().0 as usize) + } else if kind == TransferKind::InsertField { + let field = MirInsertFieldOp::new(op); + let mut base = facts[0].clone(); + let index = field.get_attr_insert_index(self.ctx).unwrap().0 as usize; + if !matches!(base, Fact::Fields(_)) { + base = aggregate_bottom(self.ctx, result); + } + write_fields(&mut base, &[index], facts[1].clone()); + base + } else if kind == TransferKind::ConstructEnum { + Fact::Enum { + tag_uniform: Domain::BLOCK, + fields: facts.clone(), + } + } else if kind == TransferKind::GetDiscriminant { + match &facts[0] { + Fact::Enum { tag_uniform, .. } => Fact::Uniform(*tag_uniform), + fact => scalar_precision(std::iter::once(fact.clone())), + } + } else if kind == TransferKind::EnumPayload { + let payload = MirEnumPayloadOp::new(op); + facts[0].variant_field( + payload.get_attr_payload_field_index(self.ctx).unwrap().0 as usize, + ) + } else if kind == TransferKind::Aggregate { + Fact::Fields(facts.clone()) + } else if kind == TransferKind::Constant { + Fact::Uniform(Domain::BLOCK) + } else if kind == TransferKind::Undef { + aggregate_bottom(self.ctx, result) + } else if kind == TransferKind::Cast { + let fact = facts[0].clone(); + if fact.has_local() { + let source_ty = operands[0].get_type(self.ctx); + let target_ty = result.get_type(self.ctx); + let same_pointee = matches!( + ( + source_ty.deref(self.ctx).downcast_ref::(), + target_ty.deref(self.ctx).downcast_ref::(), + ), + (Some(source), Some(target)) if source.pointee == target.pointee + ); + if source_ty == target_ty || same_pointee { + fact + } else { + // A reinterpreted pointer no longer names the whole slot or field. + state.expose(&fact); + Fact::Varying + } + } else { + fact + } + } else if matches!( + kind, + TransferKind::PointerOffset | TransferKind::ArrayElementAddr + ) { + state.expose(&facts[0]); + if let Fact::Local(slot, path) = &facts[0] { + if kind == TransferKind::PointerOffset + && integer(self.ctx, operands[1]) == Some(0) + { + Fact::Local(*slot, path.clone()) + } else { + Fact::Varying + } + } else { + match (facts[0].precision(), facts[1].precision()) { + (Some(a), Some(b)) => facts[0].clone().taint(a.meet(b)), + _ => Fact::Varying, + } + } + } else if kind == TransferKind::ExtractArrayElement { + integer(self.ctx, operands[1]).map_or(Fact::Varying, |i| facts[0].field(i as usize)) + } else if kind == TransferKind::CheckedArithmetic { + let f = scalar_precision(facts.iter().cloned()); + Fact::Fields(vec![f.clone(), f]) + } else if matches!( + kind, + TransferKind::Scalar | TransferKind::Eq | TransferKind::Ne | TransferKind::Not + ) { + scalar_precision(facts.iter().cloned()) + } else { + Fact::Varying + }; + state.values.insert(result, fact.taint(control.domain)); + for &value in prepared.results.iter().skip(1) { + state.values.insert(value, Fact::Varying); + } + } + Ok(()) + } +} +fn aggregate_bottom(ctx: &Context, value: Value) -> Fact { + let ty = value.get_type(ctx); + let ty = ty.deref(ctx); + if let Some(tuple) = ty.downcast_ref::() { + Fact::Fields(vec![Fact::Unset; tuple.get_types().len()]) + } else if let Some(structure) = ty.downcast_ref::() { + Fact::Fields(vec![Fact::Unset; structure.field_count()]) + } else { + Fact::Varying + } +} + +/// Run the one stateful collective analysis on the actual prepared module. +/// +/// Original roles retain root eligibility across preparation. Only structural +/// CFG/SSA information is cached; contextual values, slots and pending resource +/// accesses are evaluated on that same prepared IR before provider execution. +#[allow(clippy::too_many_arguments)] +pub fn analyze_libraries( + module: Ptr, + ctx: &Context, + analyses: &mut AnalysisManager, + rooted_functions: &BTreeMap>, + symbols: &BTreeMap, + original_roles: &BTreeMap, + admitted_shapes: &BTreeMap, + mut plans: Option<&mut BTreeMap>, +) -> pliron::result::Result { + let graph = analyses.get_analysis::(module, ctx)?; + let mut validation = Validation::default(); + for (root, reached) in rooted_functions { + let roles: Vec<_> = original_roles + .iter() + .filter(|(site, _)| reached.contains(&site.caller)) + .map(|(_, role)| *role) + .collect(); + if !roles.iter().any(|&r| checked(r) || planned_storage(r)) { + continue; + } + let Some(&shape) = admitted_shapes.get(root) else { + return pliron::verify_err!( + module.deref(ctx).loc(), + "checked collective root {root} has no admitted MIR geometry" + ); + }; + let resource_root = roles.iter().any(|&r| resource(r).is_some()); + let storage_root = roles.iter().any(|&r| planned_storage(r)); + let Some(function) = symbols.get(root).and_then(|s| graph.functions.get(s)) else { + return pliron::verify_err!( + module.deref(ctx).loc(), + "checked root {root} has no prepared MIR body" + ); + }; + let arguments = (0..function.blocks[0].deref(ctx).get_num_arguments()) + .map(|i| { + match MirFuncOp::wrap(ctx, function.op) + .unwrap() + .library_root_argument(ctx, i) + { + Some(LibraryRootArgumentClass::Scalar) => Fact::Uniform(Domain::BLOCK), + Some(LibraryRootArgumentClass::ResourceCandidate) + if resource_root || storage_root => + { + Fact::Resource(format!("{root}:{}", i + 1), Domain::BLOCK) + } + Some( + LibraryRootArgumentClass::ResourceCandidate + | LibraryRootArgumentClass::Kernel, + ) => Fact::Kernel(format!("{root}:{}", i + 1), Domain::BLOCK), + Some(LibraryRootArgumentClass::StorageCandidate) | None => Fact::Varying, + } + }) + .collect(); + let mut trace = Trace::default(); + let mut analyzer = Analyzer { + ctx, + graph: &graph, + symbols, + root, + resource_root, + storage_root, + stack: vec![], + path: vec![], + trace: &mut trace, + plan: plans.as_mut().and_then(|plans| plans.get_mut(root)), + }; + if let Err(error) = analyzer.analyze( + root, + arguments, + true, + Control::FULL, + Pending::new(), + Epochs::default(), + ) { + return pliron::verify_err!(function.op.deref(ctx).loc(), "{error}"); + } + validation.shapes.insert(root.clone(), shape); + validation.traces.insert(root.clone(), trace); + } + Ok(validation) +} + +#[cfg(test)] +#[path = "stateful_tests.rs"] +mod tests; diff --git a/cuda-oxide/crates/mir-transforms/src/library/stateful_tests.rs b/cuda-oxide/crates/mir-transforms/src/library/stateful_tests.rs new file mode 100644 index 0000000000..3026cbff42 --- /dev/null +++ b/cuda-oxide/crates/mir-transforms/src/library/stateful_tests.rs @@ -0,0 +1,1543 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use super::*; +use dialect_mir::{ + library::{LibraryFunctionSourceAttr, LibraryScalar, LibrarySourceEffectAttr, NativeSpec}, + types::{MirPointerKind, MirPtrType, MirTupleType}, +}; +use pliron::{ + builtin::{ + attributes::TypeAttr, + op_interfaces::{SingleBlockRegionInterface, SymbolOpInterface}, + ops::ModuleOp, + types::{FunctionType, IntegerType, Signedness}, + }, + op::Op, + r#type::TypeHandle, +}; + +fn function(ctx: &mut Context) -> (Ptr, Ptr, Ptr) { + dialect_mir::register(ctx); + let module = ModuleOp::new(ctx, "stateful".try_into().unwrap()); + let signature = FunctionType::get(ctx, vec![], vec![]); + let function = Operation::new( + ctx, + MirFuncOp::get_concrete_op_info(), + vec![], + vec![], + vec![], + 1, + ); + let f = MirFuncOp::new(ctx, function, TypeAttr::new(signature.into())); + f.set_symbol_name(ctx, "root".try_into().unwrap()); + f.set_library_function_source( + ctx, + LibraryFunctionSourceAttr { + export: "root".into(), + mangled: "root_marker".into(), + diagnostic: "root".into(), + checked_ancestor: true, + }, + ); + module.append_operation(ctx, function, 0); + let block = BasicBlock::new(ctx, None, vec![]); + block.insert_at_back(function.deref(ctx).get_region(0), ctx); + (module.get_operation(), function, block) +} +fn emit( + ctx: &mut Context, + block: Ptr, + types: Vec, + operands: Vec, +) -> Ptr { + let op = Operation::new(ctx, O::get_concrete_op_info(), types, operands, vec![], 0); + op.insert_at_back(block, ctx); + op +} +fn site(block: usize) -> LibraryCallSite { + LibraryCallSite { + caller: "root".into(), + block, + } +} +fn value(ctx: &mut Context, block: Ptr, ty: TypeHandle) -> Value { + emit::(ctx, block, vec![ty], vec![]) + .deref(ctx) + .get_result(0) +} +#[allow(clippy::disallowed_methods)] +fn native(ctx: &mut Context, block: Ptr, control: Value, index: usize) -> Role { + let scalar: TypeHandle = IntegerType::get(ctx, 32, Signedness::Signed).into(); + let input: TypeHandle = + MirPtrType::get_generic_with_kind(ctx, scalar, false, MirPointerKind::RawConst).into(); + let output: TypeHandle = + MirPtrType::get_generic_with_kind(ctx, scalar, true, MirPointerKind::RawMut).into(); + let memory = value(ctx, block, input); + MirLibrarySourceOp::build( + ctx, + LibrarySourceEffectAttr { + site: site(index), + kind: SourceKind::BlockQuery, + }, + Some(memory), + ) + .insert_at_back(block, ctx); + let tuple: TypeHandle = MirTupleType::get(ctx, vec![input, scalar]).into(); + let controls = emit::(ctx, block, vec![tuple], vec![memory, control]) + .deref(ctx) + .get_result(0); + let payload: TypeHandle = MirTupleType::get(ctx, vec![output]).into(); + let payload_value = value(ctx, block, payload); + let signature = FunctionType::get(ctx, vec![tuple, payload], vec![]); + let role = Role::Native(Native::CheckedBlock { + spec: NativeSpec { + scalar: LibraryScalar::Signed(32), + items: 1, + }, + }); + MirLibraryOp::build_call( + ctx, + role, + site(index), + "checked_marker".into(), + signature, + vec![controls, payload_value], + ) + .insert_at_back(block, ctx); + role +} +fn source_value(ctx: &mut Context, block: Ptr, kind: SourceKind, slot: bool) -> Value { + let scalar: TypeHandle = IntegerType::get(ctx, 32, Signedness::Signed).into(); + let result = value(ctx, block, scalar); + MirLibrarySourceOp::build( + ctx, + LibrarySourceEffectAttr { + site: site(0), + kind, + }, + Some(result), + ) + .insert_at_back(block, ctx); + if slot { + let pointer = MirPtrType::get_generic(ctx, scalar, true).into(); + let address = emit::(ctx, block, vec![pointer], vec![]) + .deref(ctx) + .get_result(0); + emit::(ctx, block, vec![], vec![address, result]); + emit::(ctx, block, vec![scalar], vec![address]) + .deref(ctx) + .get_result(0) + } else { + result + } +} +fn analyze( + ctx: &Context, + module: Ptr, + manager: &mut AnalysisManager, + roles: &BTreeMap, +) -> pliron::result::Result { + analyze_libraries( + module, + ctx, + manager, + &BTreeMap::from([("root".into(), BTreeSet::from(["root".into()]))]), + &BTreeMap::from([("root".into(), "root".into())]), + roles, + &BTreeMap::from([("root".into(), [64, 1, 1])]), + None, + ) +} + +#[test] +#[allow(clippy::disallowed_methods)] +fn scalar_callbacks_remain_raw_and_cannot_join_checked_roots() { + for checked_root in [false, true] { + let ctx = &mut Context::new(); + let (module, _, block) = function(ctx); + let scalar: TypeHandle = IntegerType::get(ctx, 32, Signedness::Signed).into(); + let pointer = + MirPtrType::get_generic_with_kind(ctx, scalar, true, MirPointerKind::RawMut).into(); + let tuple = MirTupleType::get(ctx, vec![pointer, pointer]).into(); + let environment = + MirPtrType::get_generic_with_kind(ctx, scalar, false, MirPointerKind::SharedRef).into(); + let operands = vec![value(ctx, block, tuple), value(ctx, block, environment)]; + let signature = FunctionType::get(ctx, vec![tuple, environment], vec![]); + let role = Role::Native(Native::Callback { + spec: NativeSpec { + scalar: LibraryScalar::Signed(32), + items: 1, + }, + block_threads: 64, + }); + MirLibraryOp::build_call( + ctx, + role, + site(1), + "callback_marker".into(), + signature, + operands, + ) + .insert_at_back(block, ctx); + let mut roles = BTreeMap::from([(site(1), role)]); + if checked_root { + let control = source_value(ctx, block, SourceKind::BlockQuery, false); + roles.insert(site(2), native(ctx, block, control, 2)); + } + emit::(ctx, block, vec![], vec![]); + module.deref(ctx).verify(ctx).unwrap(); + let result = analyze(ctx, module, &mut AnalysisManager::default(), &roles); + if checked_root { + assert!( + result + .unwrap_err() + .to_string() + .contains("raw and checked library calls") + ); + } else { + assert!(result.unwrap().shapes.is_empty()); + } + } +} + +#[test] +fn query_result_before_destination_store_has_the_same_ssa_and_full_slot_proof() { + for kind in [SourceKind::BlockQuery, SourceKind::LaneQuery] { + let ctx = &mut Context::new(); + let (module, _, block) = function(ctx); + let control = source_value(ctx, block, kind, true); + let role = native(ctx, block, control, 1); + emit::(ctx, block, vec![], vec![]); + module.deref(ctx).verify(ctx).unwrap(); + let roles = BTreeMap::from([(site(1), role)]); + let manager = &mut AnalysisManager::default(); + let full = analyze(ctx, module, manager, &roles); + assert_eq!(full.is_ok(), kind == SourceKind::BlockQuery); + if let Err(error) = full { + assert!(error.to_string().contains("not proved block-uniform")); + } + pliron::opts::mem2reg::mem2reg(module, ctx, manager).unwrap(); + let mut invalidated = pliron::pass::PassResult::default(); + invalidated.ir_changed = pliron::irbuild::IRStatus::Changed; + manager.retain_preserved(&invalidated); + module.deref(ctx).verify(ctx).unwrap(); + assert!( + !super::super::admission::operations(ctx, module) + .iter() + .any(|&op| Operation::get_op::(op, ctx).is_some()) + ); + let ssa = analyze(ctx, module, manager, &roles); + assert_eq!(ssa.is_ok(), kind == SourceKind::BlockQuery); + if let Err(error) = ssa { + assert!(error.to_string().contains("not proved block-uniform")); + } + } +} + +#[test] +fn separate_helper_returns_keep_selection_taint_after_promotion() { + use dialect_mir::library::{LibraryCallKind, LibraryCallTargetAttr}; + use pliron::builtin::{ + attributes::{OperandSegmentSizesAttr, StringAttr}, + op_interfaces::ATTR_KEY_OPERAND_SEGMENT_SIZES, + }; + + for kind in [SourceKind::LaneQuery, SourceKind::BlockQuery] { + let ctx = &mut Context::new(); + let (module, _, root) = function(ctx); + let scalar: TypeHandle = IntegerType::get(ctx, 32, Signedness::Signed).into(); + let bit: TypeHandle = IntegerType::get(ctx, 1, Signedness::Signless).into(); + let condition = value(ctx, root, bit); + MirLibrarySourceOp::build( + ctx, + LibrarySourceEffectAttr { + site: site(0), + kind, + }, + Some(condition), + ) + .insert_at_back(root, ctx); + let first = source_value(ctx, root, SourceKind::BlockQuery, false); + let second = source_value(ctx, root, SourceKind::BlockQuery, false); + assert_ne!(first, second); + + let signature = FunctionType::get(ctx, vec![bit, scalar, scalar], vec![scalar]); + let helper = Operation::new( + ctx, + MirFuncOp::get_concrete_op_info(), + vec![], + vec![], + vec![], + 1, + ); + let helper_op = MirFuncOp::new(ctx, helper, TypeAttr::new(signature.into())); + helper_op.set_symbol_name(ctx, "choose_return".try_into().unwrap()); + helper_op.set_library_function_source( + ctx, + LibraryFunctionSourceAttr { + export: "choose_return".into(), + mangled: "choose_return_marker".into(), + diagnostic: "choose_return".into(), + checked_ancestor: false, + }, + ); + Operation::get_op::(module, ctx) + .unwrap() + .append_operation(ctx, helper, 0); + let entry = BasicBlock::new(ctx, None, vec![bit, scalar, scalar]); + let left = BasicBlock::new(ctx, None, vec![]); + let right = BasicBlock::new(ctx, None, vec![]); + for block in [entry, left, right] { + block.insert_at_back(helper.deref(ctx).get_region(0), ctx); + } + let arguments: Vec<_> = (0..3).map(|i| entry.deref(ctx).get_argument(i)).collect(); + let pointer = MirPtrType::get_generic(ctx, scalar, true).into(); + let address = emit::(ctx, entry, vec![pointer], vec![]) + .deref(ctx) + .get_result(0); + let branch = Operation::new( + ctx, + MirCondBranchOp::get_concrete_op_info(), + vec![], + vec![arguments[0]], + vec![left, right], + 0, + ); + branch.deref_mut(ctx).attributes.set( + ATTR_KEY_OPERAND_SEGMENT_SIZES.clone(), + OperandSegmentSizesAttr(vec![1, 0, 0]), + ); + branch.insert_at_back(entry, ctx); + let mut returns = Vec::new(); + for (block, argument) in [(left, arguments[1]), (right, arguments[2])] { + // The importer materializes each source return through local _0. + emit::(ctx, block, vec![], vec![address, argument]); + let loaded = emit::(ctx, block, vec![scalar], vec![address]) + .deref(ctx) + .get_result(0); + returns.push(emit::(ctx, block, vec![], vec![loaded])); + } + let call = emit::(ctx, root, vec![scalar], vec![condition, first, second]); + let call_op = MirCallOp::new(call); + call_op.set_attr_callee(ctx, StringAttr::new("choose_return".into())); + call_op.set_attr_library_source_site(ctx, site(0)); + call_op.set_attr_library_call_target( + ctx, + LibraryCallTargetAttr { + kind: LibraryCallKind::Direct, + export: "choose_return".into(), + marker: "choose_return_marker".into(), + }, + ); + let control = call.deref(ctx).get_result(0); + let role = native(ctx, root, control, 1); + emit::(ctx, root, vec![], vec![]); + let roles = BTreeMap::from([(site(1), role)]); + let manager = &mut AnalysisManager::default(); + for promoted in [false, true] { + if promoted { + pliron::opts::mem2reg::mem2reg(module, ctx, manager).unwrap(); + let mut invalidated = pliron::pass::PassResult::default(); + invalidated.ir_changed = pliron::irbuild::IRStatus::Changed; + manager.retain_preserved(&invalidated); + assert!( + !super::super::admission::operations(ctx, module) + .iter() + .any(|&op| Operation::get_op::(op, ctx).is_some()) + ); + for (&ret, &argument) in returns.iter().zip(&arguments[1..]) { + assert!(Operation::get_op::(ret, ctx).is_some()); + assert_eq!(ret.deref(ctx).get_operand(0), argument); + } + } + module.deref(ctx).verify(ctx).unwrap(); + let result = analyze_libraries( + module, + ctx, + manager, + &BTreeMap::from([( + "root".into(), + BTreeSet::from(["root".into(), "choose_return".into()]), + )]), + &BTreeMap::from([ + ("root".into(), "root".into()), + ("choose_return".into(), "choose_return".into()), + ]), + &roles, + &BTreeMap::from([("root".into(), [64, 1, 1])]), + None, + ); + assert_eq!( + result.is_ok(), + kind == SourceKind::BlockQuery, + "promoted={promoted}" + ); + if let Err(error) = result { + assert!(error.to_string().contains("not proved block-uniform")); + } + } + } +} + +#[test] +fn same_successor_arguments_keep_their_lane_varying_selection() { + use pliron::builtin::{ + attributes::OperandSegmentSizesAttr, op_interfaces::ATTR_KEY_OPERAND_SEGMENT_SIZES, + }; + for identical in [false, true] { + let ctx = &mut Context::new(); + let (module, f, entry) = function(ctx); + let bit: TypeHandle = IntegerType::get(ctx, 1, Signedness::Signless).into(); + let condition = value(ctx, entry, bit); + MirLibrarySourceOp::build( + ctx, + LibrarySourceEffectAttr { + site: site(0), + kind: SourceKind::LaneQuery, + }, + Some(condition), + ) + .insert_at_back(entry, ctx); + let a = source_value(ctx, entry, SourceKind::BlockQuery, false); + let b = if identical { + a + } else { + source_value(ctx, entry, SourceKind::BlockQuery, false) + }; + let join = BasicBlock::new(ctx, None, vec![a.get_type(ctx)]); + join.insert_at_back(f.deref(ctx).get_region(0), ctx); + let branch = Operation::new( + ctx, + MirCondBranchOp::get_concrete_op_info(), + vec![], + vec![condition, a, b], + vec![join, join], + 0, + ); + branch.deref_mut(ctx).attributes.set( + ATTR_KEY_OPERAND_SEGMENT_SIZES.clone(), + OperandSegmentSizesAttr(vec![1, 1, 1]), + ); + branch.insert_at_back(entry, ctx); + let argument = join.deref(ctx).get_argument(0); + let role = native(ctx, join, argument, 4); + emit::(ctx, join, vec![], vec![]); + module.deref(ctx).verify(ctx).unwrap(); + let result = analyze( + ctx, + module, + &mut AnalysisManager::default(), + &BTreeMap::from([(site(4), role)]), + ); + assert_eq!(result.is_ok(), identical); + if let Err(error) = result { + assert!(error.to_string().contains("not proved block-uniform")); + } + } +} + +fn test_constant(ctx: &mut Context, block: Ptr, width: u32, n: u64) -> Value { + use pliron::{builtin::attributes::IntegerAttr, utils::apint::APInt}; + let ty = IntegerType::get(ctx, width, Signedness::Unsigned); + let op = emit::(ctx, block, vec![ty.into()], vec![]); + MirConstantOp::new(op).set_attr_value( + ctx, + IntegerAttr::new( + ty, + APInt::from_u64(n, std::num::NonZeroUsize::new(width as usize).unwrap()), + ), + ); + op.deref(ctx).get_result(0) +} + +#[allow(clippy::disallowed_methods)] +fn test_slot(ctx: &mut Context, block: Ptr, initial: Value) -> Value { + let pointer = MirPtrType::get_generic(ctx, initial.get_type(ctx), true).into(); + let address = emit::(ctx, block, vec![pointer], vec![]) + .deref(ctx) + .get_result(0); + emit::(ctx, block, vec![], vec![address, initial]); + address +} + +fn test_helper( + ctx: &mut Context, + module: Ptr, + inputs: Vec, + outputs: Vec, +) -> Ptr { + let signature = FunctionType::get(ctx, inputs.clone(), outputs); + let op = Operation::new( + ctx, + MirFuncOp::get_concrete_op_info(), + vec![], + vec![], + vec![], + 1, + ); + let f = MirFuncOp::new(ctx, op, TypeAttr::new(signature.into())); + f.set_symbol_name(ctx, "helper".try_into().unwrap()); + f.set_library_function_source( + ctx, + LibraryFunctionSourceAttr { + export: "helper".into(), + mangled: "helper_marker".into(), + diagnostic: "helper".into(), + checked_ancestor: false, + }, + ); + Operation::get_op::(module, ctx) + .unwrap() + .append_operation(ctx, op, 0); + let block = BasicBlock::new(ctx, None, inputs); + block.insert_at_back(op.deref(ctx).get_region(0), ctx); + block +} + +fn test_call( + ctx: &mut Context, + block: Ptr, + inputs: Vec, + outputs: Vec, +) -> Ptr { + use dialect_mir::library::LibraryCallTargetAttr; + use pliron::builtin::attributes::StringAttr; + let op = emit::(ctx, block, outputs, inputs); + let call = MirCallOp::new(op); + call.set_attr_callee(ctx, StringAttr::new("helper".into())); + call.set_attr_library_source_site(ctx, site(0)); + call.set_attr_library_call_target( + ctx, + LibraryCallTargetAttr { + kind: LibraryCallKind::Direct, + export: "helper".into(), + marker: "helper_marker".into(), + }, + ); + op +} + +fn slot_and_promoted_verdicts( + ctx: &mut Context, + module: Ptr, + roles: &BTreeMap, +) -> Vec> { + let manager = &mut AnalysisManager::default(); + [false, true] + .into_iter() + .map(|promoted| { + if promoted { + pliron::opts::mem2reg::mem2reg(module, ctx, manager).unwrap(); + let mut changed = pliron::pass::PassResult::default(); + changed.ir_changed = pliron::irbuild::IRStatus::Changed; + manager.retain_preserved(&changed); + } + module.deref(ctx).verify(ctx).unwrap(); + analyze_libraries( + module, + ctx, + manager, + &BTreeMap::from([( + "root".into(), + BTreeSet::from(["root".into(), "helper".into()]), + )]), + &BTreeMap::from([ + ("root".into(), "root".into()), + ("helper".into(), "helper".into()), + ]), + roles, + &BTreeMap::from([("root".into(), [64, 1, 1])]), + None, + ) + .map(|_| ()) + .map_err(|error| error.to_string()) + }) + .collect() +} + +#[test] +fn engine_review_merged_slot_alias_invalidates_only_exposed_controls() { + use pliron::builtin::{ + attributes::OperandSegmentSizesAttr, op_interfaces::ATTR_KEY_OPERAND_SEGMENT_SIZES, + }; + for read_escaped in [false, true] { + let ctx = &mut Context::new(); + let (module, f, entry) = function(ctx); + let uniform = source_value(ctx, entry, SourceKind::BlockQuery, false); + let lane = source_value(ctx, entry, SourceKind::LaneQuery, false); + let a = test_slot(ctx, entry, uniform); + let b = test_slot(ctx, entry, uniform); + let untouched = test_slot(ctx, entry, uniform); + // A proof-only address observation must not make ordinary locals borrowed. + MirLibraryWriteCheckOp::build( + ctx, + library::LibraryWriteSourceAttr { + site: site(0), + statement: 0, + kind: LibraryWriteKind::Indirect { + nominal_protected: false, + }, + }, + vec![untouched], + ) + .insert_at_back(entry, ctx); + let bit = IntegerType::get(ctx, 1, Signedness::Signless).into(); + let condition = value(ctx, entry, bit); + MirLibrarySourceOp::build( + ctx, + LibrarySourceEffectAttr { + site: site(0), + kind: SourceKind::BlockQuery, + }, + Some(condition), + ) + .insert_at_back(entry, ctx); + let left = BasicBlock::new(ctx, None, vec![]); + let right = BasicBlock::new(ctx, None, vec![]); + let join = BasicBlock::new(ctx, None, vec![a.get_type(ctx)]); + for block in [left, right, join] { + block.insert_at_back(f.deref(ctx).get_region(0), ctx); + } + let branch = Operation::new( + ctx, + MirCondBranchOp::get_concrete_op_info(), + vec![], + vec![condition], + vec![left, right], + 0, + ); + branch.deref_mut(ctx).attributes.set( + ATTR_KEY_OPERAND_SEGMENT_SIZES.clone(), + OperandSegmentSizesAttr(vec![1, 0, 0]), + ); + branch.insert_at_back(entry, ctx); + for (block, pointer) in [(left, a), (right, b)] { + Operation::new( + ctx, + MirGotoOp::get_concrete_op_info(), + vec![], + vec![pointer], + vec![join], + 0, + ) + .insert_at_back(block, ctx); + } + let selected = join.deref(ctx).get_argument(0); + emit::(ctx, join, vec![], vec![selected, lane]); + let address = if read_escaped { a } else { untouched }; + let control = emit::(ctx, join, vec![uniform.get_type(ctx)], vec![address]) + .deref(ctx) + .get_result(0); + let role = native(ctx, join, control, 4); + emit::(ctx, join, vec![], vec![]); + let verdicts = slot_and_promoted_verdicts(ctx, module, &BTreeMap::from([(site(4), role)])); + assert_eq!( + verdicts.iter().map(Result::is_ok).collect::>(), + vec![!read_escaped; 2], + "{verdicts:?}" + ); + if read_escaped { + assert!( + verdicts + .iter() + .all(|v| v.as_ref().unwrap_err().contains("not proved block-uniform")) + ); + } + } +} + +#[test] +fn engine_review_aggregate_slot_alias_survives_helper_invalidation() { + use dialect_mir::attributes::FieldIndexAttr; + for read_escaped in [false, true] { + let ctx = &mut Context::new(); + let (module, _, root) = function(ctx); + let uniform = source_value(ctx, root, SourceKind::BlockQuery, false); + let lane = source_value(ctx, root, SourceKind::LaneQuery, false); + let scalar = uniform.get_type(ctx); + let address = test_slot(ctx, root, uniform); + let untouched = test_slot(ctx, root, uniform); + let tuple = MirTupleType::get(ctx, vec![address.get_type(ctx)]).into(); + let aggregate = emit::(ctx, root, vec![tuple], vec![address]) + .deref(ctx) + .get_result(0); + let captured = test_slot(ctx, root, aggregate); + let helper = test_helper(ctx, module, vec![captured.get_type(ctx), scalar], vec![]); + let capture = helper.deref(ctx).get_argument(0); + let loaded = emit::(ctx, helper, vec![tuple], vec![capture]) + .deref(ctx) + .get_result(0); + let field = + emit::(ctx, helper, vec![address.get_type(ctx)], vec![loaded]); + MirExtractFieldOp::new(field).set_attr_index(ctx, FieldIndexAttr(0)); + let field = field.deref(ctx).get_result(0); + let written = helper.deref(ctx).get_argument(1); + emit::(ctx, helper, vec![], vec![field, written]); + emit::(ctx, helper, vec![], vec![]); + test_call(ctx, root, vec![captured, lane], vec![]); + let control = emit::( + ctx, + root, + vec![scalar], + vec![if read_escaped { address } else { untouched }], + ) + .deref(ctx) + .get_result(0); + let role = native(ctx, root, control, 1); + emit::(ctx, root, vec![], vec![]); + let verdicts = slot_and_promoted_verdicts(ctx, module, &BTreeMap::from([(site(1), role)])); + assert_eq!( + verdicts.iter().map(Result::is_ok).collect::>(), + vec![!read_escaped; 2], + "{verdicts:?}" + ); + if read_escaped { + assert!( + verdicts + .iter() + .all(|v| v.as_ref().unwrap_err().contains("not proved block-uniform")) + ); + } + } +} + +#[test] +#[allow(clippy::disallowed_methods)] +fn engine_review_array_element_store_does_not_replace_whole_fact() { + use dialect_mir::{ + attributes::{FieldIndexAttr, MirCastKindAttr, MirPointerKindAuthorityAttr}, + types::MirArrayType, + }; + #[derive(Clone, Copy, Debug)] + enum CastPath { + Direct, + ZeroOffset, + Tuple, + } + let mut failures = Vec::new(); + for (overwritten, cast_path) in [ + (None, None), + (Some(1), None), + (Some(0), None), + (Some(0), Some(CastPath::Direct)), + (Some(0), Some(CastPath::ZeroOffset)), + (Some(0), Some(CastPath::Tuple)), + ] { + let ctx = &mut Context::new(); + let (module, _, root) = function(ctx); + let uniform = source_value(ctx, root, SourceKind::BlockQuery, false); + let item = if overwritten.is_some() { + source_value(ctx, root, SourceKind::LaneQuery, false) + } else { + uniform + }; + let scalar = item.get_type(ctx); + let array = MirArrayType::get(ctx, scalar, 4).into(); + let items = emit::(ctx, root, vec![array], vec![item; 4]) + .deref(ctx) + .get_result(0); + let address = test_slot(ctx, root, items); + // Pointee-preserving pointer-kind boundaries retain the same slot. + let raw_array = + MirPtrType::get_generic_with_kind(ctx, array, true, MirPointerKind::RawMut).into(); + let cast = emit::(ctx, root, vec![raw_array], vec![address]); + MirCastOp::new(cast).set_attr_cast_kind(ctx, MirCastKindAttr::PtrToPtr); + MirCastOp::new(cast) + .set_pointer_kind_authority(ctx, MirPointerKindAuthorityAttr::RawAddress); + let address = cast.deref(ctx).get_result(0); + if let Some(index) = overwritten { + let index = test_constant(ctx, root, 32, index); + let element = + MirPtrType::get_generic_with_kind(ctx, scalar, true, MirPointerKind::RawMut).into(); + let element = if matches!(cast_path, Some(CastPath::Tuple)) { + let source_tuple = MirTupleType::get(ctx, vec![raw_array]).into(); + let target_tuple = MirTupleType::get(ctx, vec![element]).into(); + let tuple = + emit::(ctx, root, vec![source_tuple], vec![address]) + .deref(ctx) + .get_result(0); + let cast = emit::(ctx, root, vec![target_tuple], vec![tuple]); + MirCastOp::new(cast).set_attr_cast_kind(ctx, MirCastKindAttr::Transmute); + MirCastOp::new(cast) + .set_pointer_kind_authority(ctx, MirPointerKindAuthorityAttr::RustCast); + let tuple = cast.deref(ctx).get_result(0); + let extracted = emit::(ctx, root, vec![element], vec![tuple]); + MirExtractFieldOp::new(extracted).set_attr_index(ctx, FieldIndexAttr(0)); + extracted.deref(ctx).get_result(0) + } else if let Some(path) = cast_path { + let cast = emit::(ctx, root, vec![element], vec![address]); + MirCastOp::new(cast) + .set_attr_cast_kind(ctx, MirCastKindAttr::PointerCoercionArrayToPointer); + MirCastOp::new(cast) + .set_pointer_kind_authority(ctx, MirPointerKindAuthorityAttr::RustCast); + let decayed = cast.deref(ctx).get_result(0); + if matches!(path, CastPath::ZeroOffset) { + emit::(ctx, root, vec![element], vec![decayed, index]) + .deref(ctx) + .get_result(0) + } else { + decayed + } + } else { + emit::(ctx, root, vec![element], vec![address, index]) + .deref(ctx) + .get_result(0) + }; + emit::(ctx, root, vec![], vec![element, uniform]); + } + let helper = test_helper(ctx, module, vec![array], vec![scalar]); + let index = test_constant(ctx, helper, 32, 1); + let argument = helper.deref(ctx).get_argument(0); + let extracted = + emit::(ctx, helper, vec![scalar], vec![argument, index]) + .deref(ctx) + .get_result(0); + emit::(ctx, helper, vec![], vec![extracted]); + let loaded = emit::(ctx, root, vec![array], vec![address]) + .deref(ctx) + .get_result(0); + let call = test_call(ctx, root, vec![loaded], vec![scalar]); + let result = call.deref(ctx).get_result(0); + let role = native(ctx, root, result, 1); + emit::(ctx, root, vec![], vec![]); + let verdicts = slot_and_promoted_verdicts(ctx, module, &BTreeMap::from([(site(1), role)])); + if verdicts.iter().map(Result::is_ok).collect::>() != vec![overwritten.is_none(); 2] + { + failures.push(format!( + "index={overwritten:?}, cast_path={cast_path:?}: {verdicts:?}" + )); + continue; + } + if overwritten.is_some() { + assert!( + verdicts + .iter() + .all(|v| v.as_ref().unwrap_err().contains("not proved block-uniform")) + ); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +#[test] +fn bulk_transfers_invalidate_escaped_slots_and_sibling_fields() { + #[derive(Clone, Copy, Debug)] + enum Destination { + Scalar, + AdjacentField, + Unrelated, + } + fn field( + ctx: &mut Context, + block: Ptr, + address: Value, + scalar: TypeHandle, + index: u32, + ) -> Value { + let pointer = MirPtrType::get_generic(ctx, scalar, true).into(); + let op = MirFieldAddrOp::build(ctx, address, pointer, index).unwrap(); + op.insert_at_back(block, ctx); + op.deref(ctx).get_result(0) + } + let mut failures = Vec::new(); + for moving in [false, true] { + for destination in [ + Destination::Scalar, + Destination::AdjacentField, + Destination::Unrelated, + ] { + let ctx = &mut Context::new(); + let (module, _, root) = function(ctx); + let uniform = source_value(ctx, root, SourceKind::BlockQuery, false); + let lane = source_value(ctx, root, SourceKind::LaneQuery, false); + let scalar = uniform.get_type(ctx); + let (dst, src, observed, count) = if matches!(destination, Destination::AdjacentField) { + let pair = MirTupleType::get(ctx, vec![scalar, scalar]).into(); + let initial = emit::(ctx, root, vec![pair], vec![uniform; 2]) + .deref(ctx) + .get_result(0); + let replacement = emit::(ctx, root, vec![pair], vec![lane; 2]) + .deref(ctx) + .get_result(0); + let dst = test_slot(ctx, root, initial); + let src = test_slot(ctx, root, replacement); + let observed = field(ctx, root, dst, scalar, 1); + // Two scalar elements overwrite both adjacent fields. Invalidating + // just the first field would leave the observed sibling stale. + ( + field(ctx, root, dst, scalar, 0), + field(ctx, root, src, scalar, 0), + observed, + 2, + ) + } else { + let dst = test_slot(ctx, root, uniform); + let src = test_slot(ctx, root, lane); + let observed = if matches!(destination, Destination::Unrelated) { + test_slot(ctx, root, uniform) + } else { + dst + }; + (dst, src, observed, 1) + }; + let count = test_constant(ctx, root, 64, count); + let transfer = if moving { + MirMemmoveOp::build(ctx, dst, src, count) + } else { + MirMemcpyOp::build(ctx, dst, src, count) + } + .unwrap(); + transfer.insert_at_back(root, ctx); + let control = emit::(ctx, root, vec![scalar], vec![observed]) + .deref(ctx) + .get_result(0); + let role = native(ctx, root, control, 1); + emit::(ctx, root, vec![], vec![]); + let verdicts = + slot_and_promoted_verdicts(ctx, module, &BTreeMap::from([(site(1), role)])); + let accepted = matches!(destination, Destination::Unrelated); + if verdicts.iter().map(Result::is_ok).collect::>() != vec![accepted; 2] { + failures.push(format!( + "moving={moving}, destination={destination:?}: {verdicts:?}" + )); + } else if !accepted { + assert!( + verdicts + .iter() + .all(|v| v.as_ref().unwrap_err().contains("not proved block-uniform")) + ); + } + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +#[test] +fn unchecked_indexing_is_rejected_in_roots_and_ordinary_helpers() { + for marked in [None, Some("root"), Some("helper")] { + let ctx = &mut Context::new(); + let (module, root_op, root) = function(ctx); + let control = source_value(ctx, root, SourceKind::BlockQuery, true); + let scalar = control.get_type(ctx); + let helper = test_helper(ctx, module, vec![scalar], vec![scalar]); + let argument = helper.deref(ctx).get_argument(0); + emit::(ctx, helper, vec![], vec![argument]); + if let Some(name) = marked { + let op = if name == "root" { + root_op + } else { + super::super::admission::operations(ctx, module) + .into_iter() + .find(|&op| { + Operation::get_op::(op, ctx) + .is_some_and(|f| f.get_symbol_name(ctx).to_string() == "helper") + }) + .unwrap() + }; + MirFuncOp::wrap(ctx, op) + .unwrap() + .set_library_unchecked_indexing(ctx, true); + } + let call = test_call(ctx, root, vec![control], vec![scalar]); + let result = call.deref(ctx).get_result(0); + let role = native(ctx, root, result, 1); + emit::(ctx, root, vec![], vec![]); + let verdicts = slot_and_promoted_verdicts(ctx, module, &BTreeMap::from([(site(1), role)])); + assert_eq!( + verdicts.iter().map(Result::is_ok).collect::>(), + vec![marked.is_none(); 2], + "marked={marked:?}: {verdicts:?}" + ); + if let Some(name) = marked { + for verdict in verdicts { + let diagnostic = verdict.unwrap_err(); + assert!(diagnostic.contains( + "checked block collectives do not support unchecked indexing; retain runtime bounds checks" + )); + assert!(diagnostic.contains(&format!("{name} MIR block 0"))); + } + } + } +} + +#[test] +#[allow(clippy::disallowed_methods)] +fn resource_access_requires_completion_and_retains_original_witness_origin() { + use dialect_mir::{library::LibraryWitness, types::MirDisjointSliceType}; + for completed in [false, true] { + let ctx = &mut Context::new(); + dialect_mir::register(ctx); + let module = ModuleOp::new(ctx, "resource".try_into().unwrap()); + let scalar: TypeHandle = IntegerType::get(ctx, 32, Signedness::Signed).into(); + let carrier: TypeHandle = MirDisjointSliceType::get(ctx, scalar).into(); + let signature = FunctionType::get(ctx, vec![carrier], vec![]); + let function = Operation::new( + ctx, + MirFuncOp::get_concrete_op_info(), + vec![], + vec![], + vec![], + 1, + ); + let f = MirFuncOp::new(ctx, function, TypeAttr::new(signature.into())); + f.set_symbol_name(ctx, "root".try_into().unwrap()); + f.set_library_root_argument(ctx, 0, LibraryRootArgumentClass::ResourceCandidate); + module.append_operation(ctx, function, 0); + let block = BasicBlock::new(ctx, None, vec![carrier]); + block.insert_at_back(function.deref(ctx).get_region(0), ctx); + let witness_ty = + MirPtrType::get_generic_with_kind(ctx, carrier, true, MirPointerKind::UniqueRef).into(); + let argument = block.deref(ctx).get_argument(0); + let witness = emit::(ctx, block, vec![witness_ty], vec![argument]); + MirRefOp::new(witness).set_mutable(ctx, true); + MirRefOp::new(witness).set_pointer_kind_authority( + ctx, + dialect_mir::attributes::MirPointerKindAuthorityAttr::Reborrow, + ); + let witness = witness.deref(ctx).get_result(0); + let pointer: TypeHandle = + MirPtrType::get_generic_with_kind(ctx, scalar, true, MirPointerKind::RawMut).into(); + let memory = value(ctx, block, pointer); + MirLibrarySourceOp::build( + ctx, + LibrarySourceEffectAttr { + site: site(0), + kind: SourceKind::BlockQuery, + }, + Some(memory), + ) + .insert_at_back(block, ctx); + let tuple: TypeHandle = MirTupleType::get(ctx, vec![pointer]).into(); + let uniform = emit::(ctx, block, vec![tuple], vec![memory]) + .deref(ctx) + .get_result(0); + let payload = value(ctx, block, tuple); + let call_ty = FunctionType::get(ctx, vec![witness_ty, tuple, tuple], vec![]); + let mut roles = BTreeMap::new(); + for (index, access) in [(1, LibraryAccess::Read), (2, LibraryAccess::Write)] { + if index == 2 && completed { + MirLibrarySourceOp::build( + ctx, + LibrarySourceEffectAttr { + site: site(3), + kind: SourceKind::BlockSync, + }, + None, + ) + .insert_at_back(block, ctx); + } + let role = Role::Native(Native::CheckedBlockResource { + spec: NativeSpec { + scalar: LibraryScalar::Signed(32), + items: 1, + }, + access, + witness: LibraryWitness::CollectiveCarrier(1), + }); + MirLibraryOp::build_call( + ctx, + role, + site(index), + "resource_marker".into(), + call_ty, + vec![witness, uniform, payload], + ) + .insert_at_back(block, ctx); + roles.insert(site(index), role); + } + emit::(ctx, block, vec![], vec![]); + let verdicts = slot_and_promoted_verdicts(ctx, module.get_operation(), &roles); + assert_eq!( + verdicts.iter().map(Result::is_ok).collect::>(), + vec![completed; 2], + "{verdicts:?}" + ); + if !completed { + assert!(verdicts.iter().all(|v| { + v.as_ref() + .unwrap_err() + .contains("requires a full-block barrier") + })); + } + } +} + +#[test] +fn varying_cycles_stop_at_reconvergence() { + use pliron::builtin::{ + attributes::OperandSegmentSizesAttr, op_interfaces::ATTR_KEY_OPERAND_SEGMENT_SIZES, + }; + + // The second cycle never returns to the varying branch itself. The last + // case reconverges its varying choice before a uniform outer backedge. + let direct = vec![vec![1], vec![2, 3], vec![1], vec![]]; + let nested = vec![vec![1], vec![2, 4], vec![3, 4], vec![2], vec![]]; + let reconvergent = vec![ + vec![1], + vec![2, 6], + vec![3, 4], + vec![5], + vec![5], + vec![1], + vec![], + ]; + let cases = [ + ("varying backedge", &direct, 1, SourceKind::LaneQuery, false), + ("uniform backedge", &direct, 1, SourceKind::BlockQuery, true), + ( + "varying entry to inner cycle", + &nested, + 1, + SourceKind::LaneQuery, + false, + ), + ( + "uniform entry to inner cycle", + &nested, + 1, + SourceKind::BlockQuery, + true, + ), + ( + "reconvergent inner choice", + &reconvergent, + 2, + SourceKind::LaneQuery, + true, + ), + ]; + let mut mismatches = Vec::new(); + for (name, edges, selected, kind, accepted) in cases { + let ctx = &mut Context::new(); + let (module, f, entry) = function(ctx); + let bit = IntegerType::get(ctx, 1, Signedness::Signless).into(); + let mut conditions = Vec::new(); + for source_kind in [kind, SourceKind::BlockQuery] { + let condition = value(ctx, entry, bit); + MirLibrarySourceOp::build( + ctx, + LibrarySourceEffectAttr { + site: site(0), + kind: source_kind, + }, + Some(condition), + ) + .insert_at_back(entry, ctx); + conditions.push(test_slot(ctx, entry, condition)); + } + let control = source_value(ctx, entry, SourceKind::BlockQuery, false); + let mut blocks = vec![entry]; + for _ in 1..edges.len() { + let block = BasicBlock::new(ctx, None, vec![]); + block.insert_at_back(f.deref(ctx).get_region(0), ctx); + blocks.push(block); + } + let last = edges.len() - 1; + let mut roles = BTreeMap::new(); + for (index, targets) in edges.iter().enumerate() { + let block = blocks[index]; + if targets.is_empty() { + roles.insert(site(last), native(ctx, block, control, last)); + emit::(ctx, block, vec![], vec![]); + } else if targets.len() == 1 { + Operation::new( + ctx, + MirGotoOp::get_concrete_op_info(), + vec![], + vec![], + vec![blocks[targets[0]]], + 0, + ) + .insert_at_back(block, ctx); + } else { + let address = conditions[usize::from(index != selected)]; + let condition = emit::(ctx, block, vec![bit], vec![address]) + .deref(ctx) + .get_result(0); + let branch = Operation::new( + ctx, + MirCondBranchOp::get_concrete_op_info(), + vec![], + vec![condition], + targets.iter().map(|&i| blocks[i]).collect(), + 0, + ); + branch.deref_mut(ctx).attributes.set( + ATTR_KEY_OPERAND_SEGMENT_SIZES.clone(), + OperandSegmentSizesAttr(vec![1, 0, 0]), + ); + branch.insert_at_back(block, ctx); + } + } + let verdicts = slot_and_promoted_verdicts(ctx, module, &roles); + println!("{name}: {verdicts:?}"); + for verdict in verdicts { + if verdict.is_ok() != accepted { + mismatches.push(format!("{name}: {verdict:?}")); + } + if let Err(error) = verdict { + assert!( + error.contains("lane-varying cycle cannot prove collective participation"), + "{name}: {error}" + ); + } + } + } + assert!(mismatches.is_empty(), "{}", mismatches.join("\n")); +} + +#[test] +#[allow(clippy::disallowed_methods)] +fn planned_storage_reuses_the_same_proof_for_completion_and_global_resource_conflicts() { + use dialect_mir::{library::LibraryWitness, types::MirDisjointSliceType}; + for (policy, completed, bytes, expected) in [ + (None, false, 16, None), + ( + None, + false, + 0, + Some("conflicting collective resource access"), + ), + ( + Some(Policy::default()), + false, + 16, + Some("explicit storage reuse"), + ), + (Some(Policy::default()), true, 16, None), + ( + Some(Policy { + auto_sync: true, + ..Default::default() + }), + false, + 16, + None, + ), + ] { + let ctx = &mut Context::new(); + dialect_mir::register(ctx); + let module = ModuleOp::new(ctx, "resource".try_into().unwrap()); + let scalar: TypeHandle = IntegerType::get(ctx, 32, Signedness::Signed).into(); + let carrier: TypeHandle = MirDisjointSliceType::get(ctx, scalar).into(); + let signature = FunctionType::get(ctx, vec![carrier], vec![]); + let function = Operation::new( + ctx, + MirFuncOp::get_concrete_op_info(), + vec![], + vec![], + vec![], + 1, + ); + let f = MirFuncOp::new(ctx, function, TypeAttr::new(signature.into())); + f.set_symbol_name(ctx, "root".try_into().unwrap()); + f.set_library_root_argument(ctx, 0, LibraryRootArgumentClass::ResourceCandidate); + module.append_operation(ctx, function, 0); + let block = BasicBlock::new(ctx, None, vec![carrier]); + block.insert_at_back(function.deref(ctx).get_region(0), ctx); + let witness_ty = + MirPtrType::get_generic_with_kind(ctx, carrier, true, MirPointerKind::UniqueRef).into(); + let argument = block.deref(ctx).get_argument(0); + let witness = emit::(ctx, block, vec![witness_ty], vec![argument]); + MirRefOp::new(witness).set_mutable(ctx, true); + MirRefOp::new(witness).set_pointer_kind_authority( + ctx, + dialect_mir::attributes::MirPointerKindAuthorityAttr::Reborrow, + ); + let witness = witness.deref(ctx).get_result(0); + let pointer: TypeHandle = + MirPtrType::get_generic_with_kind(ctx, scalar, true, MirPointerKind::RawMut).into(); + let memory = value(ctx, block, pointer); + MirLibrarySourceOp::build( + ctx, + LibrarySourceEffectAttr { + site: site(0), + kind: SourceKind::BlockQuery, + }, + Some(memory), + ) + .insert_at_back(block, ctx); + let tuple: TypeHandle = MirTupleType::get(ctx, vec![pointer]).into(); + let uniform = emit::(ctx, block, vec![tuple], vec![memory]) + .deref(ctx) + .get_result(0); + let payload = value(ctx, block, tuple); + let mut roles = BTreeMap::new(); + let (storage, storage_arg) = if let Some(policy) = policy { + let byte = IntegerType::get(ctx, 8, Signedness::Unsigned).into(); + let descriptor = dialect_mir::types::MirStructType::get( + ctx, + "descriptor".into(), + vec!["private".into()], + vec![byte], + ) + .into(); + let signature = FunctionType::get(ctx, vec![], vec![descriptor]); + let role = Role::StorageNew(policy); + let new = + MirLibraryOp::build_call(ctx, role, site(10), "new".into(), signature, vec![]); + new.insert_at_back(block, ctx); + roles.insert(site(10), role); + let original = new.deref(ctx).get_result(0); + let address = test_slot(ctx, block, original); + let loaded = emit::(ctx, block, vec![descriptor], vec![address]) + .deref(ctx) + .get_result(0); + let borrow = + MirPtrType::get_generic_with_kind(ctx, descriptor, true, MirPointerKind::UniqueRef) + .into(); + let reference = emit::(ctx, block, vec![borrow], vec![loaded]); + MirRefOp::new(reference).set_mutable(ctx, true); + MirRefOp::new(reference).set_pointer_kind_authority( + ctx, + dialect_mir::attributes::MirPointerKindAuthorityAttr::Reborrow, + ); + ( + reference.deref(ctx).get_result(0), + LibraryStorageArg::Explicit(policy), + ) + } else { + let empty = MirTupleType::get(ctx, vec![]).into(); + (value(ctx, block, empty), LibraryStorageArg::Implicit) + }; + let call_ty = FunctionType::get( + ctx, + vec![witness_ty, storage.get_type(ctx), tuple, tuple], + vec![], + ); + for (index, access) in [(1, LibraryAccess::Read), (2, LibraryAccess::Write)] { + if index == 2 && completed { + MirLibrarySourceOp::build( + ctx, + LibrarySourceEffectAttr { + site: site(3), + kind: SourceKind::BlockSync, + }, + None, + ) + .insert_at_back(block, ctx); + } + let role = Role::Native(Native::CheckedBlockStorage { + spec: NativeSpec { + scalar: LibraryScalar::Signed(32), + items: 1, + }, + access, + witness: LibraryWitness::CollectiveCarrier(1), + storage: storage_arg, + }); + MirLibraryOp::build_call( + ctx, + role, + site(index), + "resource_marker".into(), + call_ty, + vec![witness, storage, uniform, payload], + ) + .insert_at_back(block, ctx); + roles.insert(site(index), role); + } + emit::(ctx, block, vec![], vec![]); + let module = module.get_operation(); + module.deref(ctx).verify(ctx).unwrap(); + let manager = &mut AnalysisManager::default(); + for promoted in [false, true] { + if promoted { + pliron::opts::mem2reg::mem2reg(module, ctx, manager).unwrap(); + let mut changed = pliron::pass::PassResult::default(); + changed.ir_changed = pliron::irbuild::IRStatus::Changed; + manager.retain_preserved(&changed); + module.deref(ctx).verify(ctx).unwrap(); + } + let first = analyze(ctx, module, manager, &roles).unwrap(); + let natives = super::super::storage::RootedNativePlans::from([( + "root".into(), + [1, 2] + .into_iter() + .map(|index| { + let mut plan = dialect_mir::library::LibraryCallPlan::internal( + dialect_mir::library::LibraryOperation::Native, + "resource_marker".into(), + ); + plan.scratch_bytes = bytes; + plan.scratch_alignment = 4; + (site(index), plan) + }) + .collect(), + )]); + let mut plans = + super::super::storage::allocate_library_storage(&first, &natives).unwrap(); + assert!(manager.try_get_analysis::(module).is_some()); + let second = analyze_libraries( + module, + ctx, + manager, + &BTreeMap::from([("root".into(), BTreeSet::from(["root".into()]))]), + &BTreeMap::from([("root".into(), "root".into())]), + &roles, + &first.shapes, + Some(&mut plans), + ); + if let Some(expected) = expected { + assert!(second.unwrap_err().to_string().contains(expected)); + } else { + second.unwrap(); + assert_eq!(plans["root"].bytes, bytes); + assert_eq!(plans["root"].sites[&vec![2]].before, policy.is_none()); + assert_eq!( + plans["root"].sites[&vec![1]].after, + policy.is_some_and(|p| p.auto_sync) + ); + } + } + } +} + +#[test] +#[allow(clippy::disallowed_methods)] +fn reservations_require_release_and_completion_or_disjoint_exclusive_placement() { + use dialect_mir::{library::LibraryReserveLayout, types::MirStructType}; + // A barrier does not release a token, and releasing a token does not complete + // its reads. Exclusive sites instead receive distinct aligned regions. + for (release, completed, exclusive, capacity, expected) in [ + (false, false, false, 0, Some("overlaps a live reservation")), + (false, true, false, 0, Some("overlaps a live reservation")), + (true, false, false, 0, Some("explicit storage reuse")), + (true, true, false, 0, None), + (false, false, true, 0, None), + (false, false, true, 19, Some("including exclusive padding")), + ] { + let ctx = &mut Context::new(); + let (module, _, block) = function(ctx); + let byte = IntegerType::get(ctx, 8, Signedness::Unsigned).into(); + let scalar = IntegerType::get(ctx, 32, Signedness::Signed).into(); + let descriptor = + MirStructType::get(ctx, "descriptor".into(), vec!["private".into()], vec![byte]).into(); + let pointer = + MirPtrType::get_generic_with_kind(ctx, scalar, true, MirPointerKind::RawMut).into(); + let token_type = + MirStructType::get(ctx, "token".into(), vec!["private".into()], vec![pointer]).into(); + let policy = Policy { + exclusive, + capacity, + ..Default::default() + }; + let new_role = Role::StorageNew(policy); + let signature = FunctionType::get(ctx, vec![], vec![descriptor]); + let new = + MirLibraryOp::build_call(ctx, new_role, site(10), "new".into(), signature, vec![]); + new.insert_at_back(block, ctx); + let initial = new.deref(ctx).get_result(0); + let address = test_slot(ctx, block, initial); + let loaded = emit::(ctx, block, vec![descriptor], vec![address]) + .deref(ctx) + .get_result(0); + let borrow_type = + MirPtrType::get_generic_with_kind(ctx, descriptor, true, MirPointerKind::UniqueRef) + .into(); + let borrow = emit::(ctx, block, vec![borrow_type], vec![loaded]); + MirRefOp::new(borrow).set_mutable(ctx, true); + MirRefOp::new(borrow).set_pointer_kind_authority( + ctx, + dialect_mir::attributes::MirPointerKindAuthorityAttr::Reborrow, + ); + let borrow = borrow.deref(ctx).get_result(0); + let mut roles = BTreeMap::from([(site(10), new_role)]); + for index in [1, 2] { + let role = Role::StorageReserve { + policy, + layout: LibraryReserveLayout { + scalar: LibraryScalar::Signed(32), + count: 1, + bytes: 4, + alignment: if index == 1 { 4 } else { 16 }, + }, + }; + let signature = FunctionType::get(ctx, vec![borrow_type], vec![token_type]); + let reserve = MirLibraryOp::build_call( + ctx, + role, + site(index), + "reserve".into(), + signature, + vec![borrow], + ); + reserve.insert_at_back(block, ctx); + roles.insert(site(index), role); + if index == 1 { + if release { + let token = reserve.deref(ctx).get_result(0); + let address = test_slot(ctx, block, token); + let token = emit::(ctx, block, vec![token_type], vec![address]) + .deref(ctx) + .get_result(0); + let signature = FunctionType::get(ctx, vec![token_type], vec![]); + MirLibraryOp::build_call( + ctx, + Role::StorageRelease, + site(3), + "release".into(), + signature, + vec![token], + ) + .insert_at_back(block, ctx); + roles.insert(site(3), Role::StorageRelease); + } + if completed { + MirLibrarySourceOp::build( + ctx, + LibrarySourceEffectAttr { + site: site(4), + kind: SourceKind::BlockSync, + }, + None, + ) + .insert_at_back(block, ctx); + } + } + } + emit::(ctx, block, vec![], vec![]); + module.deref(ctx).verify(ctx).unwrap(); + let manager = &mut AnalysisManager::default(); + for promoted in [false, true] { + if promoted { + pliron::opts::mem2reg::mem2reg(module, ctx, manager).unwrap(); + let mut changed = pliron::pass::PassResult::default(); + changed.ir_changed = pliron::irbuild::IRStatus::Changed; + manager.retain_preserved(&changed); + module.deref(ctx).verify(ctx).unwrap(); + } + let first = analyze(ctx, module, manager, &roles).unwrap(); + assert_eq!(first.traces["root"].events[&vec![2]].origin, Some(vec![10])); + let plans = super::super::storage::allocate_library_storage(&first, &BTreeMap::new()); + if capacity != 0 { + assert!(plans.err().unwrap().contains(expected.unwrap())); + continue; + } + let mut plans = plans.unwrap(); + assert_eq!(plans["root"].bytes, if exclusive { 20 } else { 4 }); + assert_eq!( + plans["root"].sites[&vec![2]].offset, + if exclusive { 16 } else { 0 } + ); + let second = analyze_libraries( + module, + ctx, + manager, + &BTreeMap::from([("root".into(), BTreeSet::from(["root".into()]))]), + &BTreeMap::from([("root".into(), "root".into())]), + &roles, + &first.shapes, + Some(&mut plans), + ); + if let Some(expected) = expected { + assert!(second.unwrap_err().to_string().contains(expected)); + } else { + second.unwrap(); + } + } + } +} diff --git a/cuda-oxide/crates/mir-transforms/src/library/storage.rs b/cuda-oxide/crates/mir-transforms/src/library/storage.rs new file mode 100644 index 0000000000..f440379321 --- /dev/null +++ b/cuda-oxide/crates/mir-transforms/src/library/storage.rs @@ -0,0 +1,198 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Owned stateful traces and placement of validated native storage layouts. +use dialect_mir::library::{ + LibraryCallPlan, LibraryCallSite, LibraryNativeMetadata as Native, LibraryRoleAttr as Role, +}; +use std::collections::BTreeMap; + +/// The authenticated source policy, shared with the retained dialect contract. +pub type Policy = dialect_mir::library::LibraryStoragePolicy; +/// Original direct-call block path, followed by the source operation's block. +pub type Site = Vec; +/// Validated provider responses keyed by root and original source call site. +pub type RootedNativePlans = BTreeMap>; +/// One validated static storage/native use in a concrete helper context. +#[derive(Clone, Debug)] +pub struct Event { + pub call: LibraryCallSite, + pub role: Role, + pub marker: String, + pub origin: Option, + pub policy: Policy, +} +/// Ordered static contexts and uses from one layout-free proof. +#[derive(Clone, Debug, Default)] +pub struct Trace { + pub contexts: BTreeMap, + pub events: BTreeMap, +} +/// Owned output retained while the host provider runs. +#[derive(Clone, Debug, Default)] +pub struct Validation { + pub shapes: BTreeMap, + pub traces: BTreeMap, +} +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Binding { + pub region: Site, + pub offset: u32, + /// Full descriptor allocation, including explicit capacity, from this view. + pub declared_bytes: u32, + pub bytes: u32, + pub alignment: u32, + pub implicit: bool, + pub auto_sync: bool, + pub before: bool, + pub after: bool, +} +#[derive(Default)] +pub struct Plan { + pub sites: BTreeMap, + pub bytes: u32, + pub alignment: u32, +} + +pub fn align_up(value: u32, alignment: u32) -> Result { + value + .checked_add(alignment - 1) + .map(|n| n & !(alignment - 1)) + .ok_or_else(|| "storage layout exceeds u32".into()) +} + +/// Place every static use once. `layouts` comes only from validated provider +/// responses or actual Rust array layouts; no CUB algorithm names are inspected. +pub fn allocate(trace: &Trace, layouts: &BTreeMap) -> Result { + type Uses = Vec<(Site, u32, u32)>; + let mut descriptors: BTreeMap, (Policy, Uses)> = BTreeMap::new(); + for (site, event) in &trace.events { + let Some(&(bytes, align)) = layouts.get(site) else { + continue; + }; + if matches!(event.role, Role::StorageReserve { .. }) && event.policy.auto_sync { + return Err("storage reservations cannot use auto_sync".into()); + } + if bytes == 0 { + continue; + } + let (policy, uses) = descriptors + .entry(event.origin.clone()) + .or_insert_with(|| (event.policy, Vec::new())); + if *policy != event.policy { + return Err("one storage origin has incompatible policies after use".into()); + } + uses.push((site.clone(), bytes, align)); + } + let mut plan = Plan { + alignment: 1, + ..Default::default() + }; + for (origin, (policy, uses)) in descriptors { + let base_align = uses + .iter() + .map(|(_, _, a)| *a) + .max() + .unwrap_or(1) + .max(policy.alignment); + let mut end = 0; + let mut placed = Vec::new(); + for (site, bytes, align) in uses { + let offset = if policy.exclusive { + align_up(end, align)? + } else { + 0 + }; + end = end.max( + offset + .checked_add(bytes) + .ok_or("storage layout exceeds u32")?, + ); + placed.push((site, offset, bytes, align)); + } + if policy.capacity != 0 && policy.capacity < end { + return Err(format!( + "storage capacity {} is smaller than complete plan {end}, including exclusive padding", + policy.capacity + )); + } + let allocation = end.max(policy.capacity); + let base = align_up(plan.bytes, base_align)?; + plan.bytes = base + .checked_add(allocation) + .ok_or("storage layout exceeds u32")?; + plan.alignment = plan.alignment.max(base_align); + for (site, offset, bytes, align) in placed { + // Empty is the implicit pool. Explicit origins always contain a bb. + let mut region = origin.clone().unwrap_or_default(); + if policy.exclusive { + region.push(usize::MAX); + region.extend(&site); + } + plan.sites.insert( + site, + Binding { + region, + offset: base + offset, + declared_bytes: allocation - offset, + bytes, + alignment: align.max(if offset == 0 { base_align } else { 1 }), + implicit: origin.is_none(), + auto_sync: policy.auto_sync, + before: false, + after: policy.auto_sync, + }, + ); + } + } + Ok(plan) +} + +/// Whether source reachability activates the planned storage subset. +pub fn planned_storage(role: Role) -> bool { + matches!( + role, + Role::StorageNew(_) + | Role::StorageReconfigure { .. } + | Role::StorageReserve { .. } + | Role::StorageRelease + | Role::Native(Native::CheckedBlockStorage { .. }) + ) +} + +/// Join only validated provider layouts or authenticated reservation layouts. +pub fn allocate_library_storage( + validation: &Validation, + native_plans: &RootedNativePlans, +) -> Result, String> { + let mut plans = BTreeMap::new(); + for (root, trace) in &validation.traces { + if !trace + .events + .values() + .any(|event| planned_storage(event.role)) + { + continue; + } + let mut layouts = BTreeMap::new(); + for (site, event) in &trace.events { + match event.role { + Role::Native(Native::CheckedBlockStorage { .. }) => { + let plan = native_plans + .get(root) + .and_then(|calls| calls.get(&event.call)) + .ok_or("missing storage provider plan")?; + layouts.insert(site.clone(), (plan.scratch_bytes, plan.scratch_alignment)); + } + Role::StorageReserve { layout, .. } => { + layouts.insert(site.clone(), (layout.bytes, layout.alignment)); + } + _ => {} + } + } + plans.insert(root.clone(), allocate(trace, &layouts)?); + } + Ok(plans) +} diff --git a/cuda-oxide/crates/oxide-artifacts/README.md b/cuda-oxide/crates/oxide-artifacts/README.md index d51f218699..98b9c6598f 100644 --- a/cuda-oxide/crates/oxide-artifacts/README.md +++ b/cuda-oxide/crates/oxide-artifacts/README.md @@ -56,7 +56,7 @@ Artifact Blob +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | Payload Count | Entry Count | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Compile Options (v2) / Reserved Zero (v1) | + | Compile Options (v2/v3) / Reserved Zero (v1) | | | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | Bundle Name ... / @@ -74,14 +74,14 @@ Artifact Blob Header size is currently 32 bytes. The final field occupies bytes 24 through 31 (the half-open range `[24, 32)`) and is a 64-bit compile-options bitset in -version 2. Bit 0 disables FMA contraction; unknown option bits are rejected. +versions 2 and 3. Bit 0 disables FMA contraction; unknown option bits are rejected. Version-1 bundles define the same bytes as reserved zero and use the historical default policy, where FMA contraction is allowed. -Version 2 is the latest supported format. For backward compatibility, the -writer still emits version 1 when the compile-options bitset is zero. It emits -version 2 only when a bundle carries a non-default compile policy, ensuring -that an older reader rejects policy it would otherwise silently ignore. +Version 3 carries finalized kernel launch facts. Bundles without launch facts +retain their existing encoding: version 1 for default compile policy and version +2 for non-default compile policy. A reader that does not support version 3 +rejects those bundles before loading code with unknown launch requirements. ```text Payload Record (24 bytes) @@ -111,7 +111,7 @@ Payload kind values: - `0x0200`: cubin ```text -Entry Record (24 bytes) +Entry Record (24 bytes in v1/v2; 48 bytes in v3) 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 @@ -135,8 +135,31 @@ Entry kind values: - `2`: device function If bit 0 of `Flags` is set, `Metadata` is present and interpreted by the -consumer for that entry kind. No metadata interpretation is currently required -for PTX kernel loading. +consumer for that entry kind. This opaque metadata is independent of typed +launch facts. + +Version 3 adds the following fields to every entry record: + +| Byte offset | Field | Type | +| --- | --- | --- | +| 24 | Exact block X | u32 | +| 28 | Exact block Y | u32 | +| 32 | Exact block Z | u32 | +| 36 | Finalized dynamic shared-memory bytes per block | u32 | +| 40 | Dynamic shared-memory alignment | u32 | +| 44 | Reserved zero | u32 | + +Flag bit 1 indicates that launch facts are present; bit 2 indicates an exact +block shape and requires bit 1. Facts are valid only for kernel entries. Exact +block dimensions must be nonzero and alignment must be a nonzero power of two. +Unknown version-3 entry flags are rejected. Absent fields are written as zero. + +Each record describes its concrete entry symbol, including separate generic +monomorphizations. A zero-byte allocation is distinct from absent facts. The +byte count excludes static shared memory and driver-reserved memory. The +producer supplies a finalized allocation and a matching generated alignment; +the container validates their representation but does not derive requirements +from device code or prove that the producer's claims are true. ## Object Storage @@ -154,11 +177,11 @@ cubin from embedded NVVM IR/LTOIR before loading. ## Constraints -- The latest supported format version is `2`; default-policy bundles are still - emitted as version `1` for backward compatibility. +- The latest supported format version is `3`; bundles without launch facts + retain their version `1` or `2` encoding. - All numeric fields are little-endian. - The blob header is fixed at 32 bytes. -- Payload and entry records are fixed at 24 bytes each. +- Payload records are 24 bytes; entry records are 24 bytes in v1/v2 and 48 in v3. - String and payload offsets are 32-bit, so a single blob must fit in `u32::MAX` bytes. - String lengths and record counts are 16-bit. diff --git a/cuda-oxide/crates/oxide-artifacts/src/lib.rs b/cuda-oxide/crates/oxide-artifacts/src/lib.rs index 99ef56fea2..7f0fc9ae5d 100644 --- a/cuda-oxide/crates/oxide-artifacts/src/lib.rs +++ b/cuda-oxide/crates/oxide-artifacts/src/lib.rs @@ -15,12 +15,20 @@ pub const ARTIFACT_SECTION_NAME: &str = ".oxart"; #[cfg(feature = "object-write")] const ARTIFACT_ANCHOR_SECTION_NAME: &str = ".oxlink"; pub const ARTIFACT_MAGIC: [u8; 8] = *b"OXIDEART"; -pub const ARTIFACT_VERSION: u16 = 2; +pub const ARTIFACT_VERSION: u16 = 3; +const COMPILE_OPTIONS_ARTIFACT_VERSION: u16 = 2; const LEGACY_ARTIFACT_VERSION: u16 = 1; const HEADER_BYTES: usize = 32; const PAYLOAD_RECORD_BYTES: usize = 24; -const ENTRY_RECORD_BYTES: usize = 24; +const LEGACY_ENTRY_RECORD_BYTES: usize = 24; +const ENTRY_RECORD_BYTES: usize = 48; + +const ENTRY_METADATA_PRESENT: u16 = 1; +const ENTRY_LAUNCH_FACTS_PRESENT: u16 = 1 << 1; +const ENTRY_EXACT_BLOCK_PRESENT: u16 = 1 << 2; +const KNOWN_ENTRY_FLAGS: u16 = + ENTRY_METADATA_PRESENT | ENTRY_LAUNCH_FACTS_PRESENT | ENTRY_EXACT_BLOCK_PRESENT; const OPTION_NO_FMA_CONTRACTION: u64 = 1 << 0; const OPTION_DEBUG_LINE_TABLES: u64 = 1 << 1; @@ -223,11 +231,28 @@ impl<'a> ArtifactPayloadSpec<'a> { } } +/// Finalized launch requirements for one concrete kernel entry. +/// +/// These facts describe the allocation selected by the compiler. They do not +/// include static shared memory or driver-reserved memory. An absent record +/// means that the producer did not provide finalized requirements; it is +/// distinct from a record requiring zero dynamic bytes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct KernelLaunchFacts { + /// Required `(x, y, z)` block dimensions, when the kernel needs an exact shape. + pub exact_block: Option<(u32, u32, u32)>, + /// Finalized dynamic shared-memory allocation in bytes per block. + pub dynamic_shared_bytes: u32, + /// Alignment required by the generated dynamic shared-memory declaration. + pub dynamic_shared_alignment: u32, +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct ArtifactEntrySpec<'a> { pub symbol: &'a str, pub kind: ArtifactEntryKind, pub metadata: Option, + pub launch_facts: Option, } impl<'a> ArtifactEntrySpec<'a> { @@ -236,6 +261,7 @@ impl<'a> ArtifactEntrySpec<'a> { symbol, kind, metadata: None, + launch_facts: None, } } @@ -243,6 +269,11 @@ impl<'a> ArtifactEntrySpec<'a> { self.metadata = Some(metadata); self } + + pub const fn with_launch_facts(mut self, facts: KernelLaunchFacts) -> Self { + self.launch_facts = Some(facts); + self + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -293,6 +324,7 @@ pub struct ArtifactEntry<'a> { pub symbol: &'a str, pub kind: ArtifactEntryKind, pub metadata: Option, + pub launch_facts: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -329,6 +361,7 @@ pub struct OwnedArtifactEntry { pub symbol: String, pub kind: ArtifactEntryKind, pub metadata: Option, + pub launch_facts: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -375,6 +408,7 @@ impl<'a> From> for OwnedArtifactBundle { symbol: entry.symbol.to_string(), kind: entry.kind, metadata: entry.metadata, + launch_facts: entry.launch_facts, }) .collect(), } @@ -445,6 +479,15 @@ impl std::error::Error for ArtifactError {} pub fn build_artifact_blob(spec: &ArtifactBundleSpec<'_>) -> Result, ArtifactError> { validate_spec(spec)?; + let has_launch_facts = spec + .entries + .iter() + .any(|entry| entry.launch_facts.is_some()); + let entry_record_bytes = if has_launch_facts { + ENTRY_RECORD_BYTES + } else { + LEGACY_ENTRY_RECORD_BYTES + }; let mut out = vec![0; HEADER_BYTES]; push_bytes(&mut out, spec.name.as_bytes()); push_bytes(&mut out, spec.target.as_bytes()); @@ -452,7 +495,7 @@ pub fn build_artifact_blob(spec: &ArtifactBundleSpec<'_>) -> Result, Art let payload_record_start = out.len(); out.resize(out.len() + spec.payloads.len() * PAYLOAD_RECORD_BYTES, 0); let entry_record_start = out.len(); - out.resize(out.len() + spec.entries.len() * ENTRY_RECORD_BYTES, 0); + out.resize(out.len() + spec.entries.len() * entry_record_bytes, 0); for (index, payload) in spec.payloads.iter().enumerate() { let name_offset = checked_u32(out.len(), "payload name offset")?; @@ -484,9 +527,21 @@ pub fn build_artifact_blob(spec: &ArtifactBundleSpec<'_>) -> Result, Art push_bytes(&mut out, entry.symbol.as_bytes()); align_vec(&mut out, 8); - let record = entry_record_start + index * ENTRY_RECORD_BYTES; + let record = entry_record_start + index * entry_record_bytes; write_u16(&mut out, record, entry.kind.to_u16()); - write_u16(&mut out, record + 2, u16::from(entry.metadata.is_some())); + let mut flags = u16::from(entry.metadata.is_some()); + if let Some(facts) = entry.launch_facts { + flags |= ENTRY_LAUNCH_FACTS_PRESENT; + if let Some((x, y, z)) = facts.exact_block { + flags |= ENTRY_EXACT_BLOCK_PRESENT; + write_u32(&mut out, record + 24, x); + write_u32(&mut out, record + 28, y); + write_u32(&mut out, record + 32, z); + } + write_u32(&mut out, record + 36, facts.dynamic_shared_bytes); + write_u32(&mut out, record + 40, facts.dynamic_shared_alignment); + } + write_u16(&mut out, record + 2, flags); write_u64(&mut out, record + 4, entry.metadata.unwrap_or(0)); write_u32(&mut out, record + 12, symbol_offset); write_u16( @@ -498,13 +553,15 @@ pub fn build_artifact_blob(spec: &ArtifactBundleSpec<'_>) -> Result, Art let total_len = checked_u32(out.len(), "total length")?; out[0..8].copy_from_slice(&ARTIFACT_MAGIC); - // Keep default-policy bundles on v1 for backward compatibility. A bundle - // that carries required compile policy uses v2 so an older reader rejects - // it instead of silently ignoring the semantic flag. - let version = if spec.compile_options == ArtifactCompileOptions::new() { + // Preserve the legacy bytes when no launch facts are supplied. Required + // launch facts use v3 so older readers reject them instead of silently + // ignoring the kernel's resource requirements. + let version = if has_launch_facts { + ARTIFACT_VERSION + } else if spec.compile_options == ArtifactCompileOptions::new() { LEGACY_ARTIFACT_VERSION } else { - ARTIFACT_VERSION + COMPILE_OPTIONS_ARTIFACT_VERSION }; write_u16(&mut out, 8, version); write_u16(&mut out, 10, HEADER_BYTES as u16); @@ -550,7 +607,10 @@ pub fn parse_artifact_blob(bytes: &[u8]) -> Result, ArtifactE return Err(ArtifactError::BadMagic); } let version = read_u16(bytes, 8)?; - if !matches!(version, LEGACY_ARTIFACT_VERSION | ARTIFACT_VERSION) { + if !matches!( + version, + LEGACY_ARTIFACT_VERSION | COMPILE_OPTIONS_ARTIFACT_VERSION | ARTIFACT_VERSION + ) { return Err(ArtifactError::UnsupportedVersion(version)); } let header_len = read_u16(bytes, 10)? as usize; @@ -591,8 +651,13 @@ pub fn parse_artifact_blob(bytes: &[u8]) -> Result, ArtifactE .ok_or(ArtifactError::TooLarge("payload records"))?; require_len(bytes, cursor, "payload records")?; let entry_records = cursor; + let entry_record_bytes = if version == ARTIFACT_VERSION { + ENTRY_RECORD_BYTES + } else { + LEGACY_ENTRY_RECORD_BYTES + }; cursor = cursor - .checked_add(entry_count * ENTRY_RECORD_BYTES) + .checked_add(entry_count * entry_record_bytes) .ok_or(ArtifactError::TooLarge("entry records"))?; require_len(bytes, cursor, "entry records")?; @@ -617,12 +682,12 @@ pub fn parse_artifact_blob(bytes: &[u8]) -> Result, ArtifactE let mut entries = Vec::with_capacity(entry_count); for index in 0..entry_count { - let record = entry_records + index * ENTRY_RECORD_BYTES; + let record = entry_records + index * entry_record_bytes; let kind_raw = read_u16(bytes, record)?; let kind = ArtifactEntryKind::from_u16(kind_raw) .ok_or(ArtifactError::UnsupportedEntryKind(kind_raw))?; let flags = read_u16(bytes, record + 2)?; - let metadata = if flags & 1 != 0 { + let metadata = if flags & ENTRY_METADATA_PRESENT != 0 { Some(read_u64(bytes, record + 4)?) } else { None @@ -630,10 +695,40 @@ pub fn parse_artifact_blob(bytes: &[u8]) -> Result, ArtifactE let symbol_offset = read_u32(bytes, record + 12)? as usize; let symbol_len = read_u16(bytes, record + 16)? as usize; let symbol = read_str(bytes, symbol_offset, symbol_len, "entry symbol")?; + let launch_facts = if version == ARTIFACT_VERSION { + if flags & !KNOWN_ENTRY_FLAGS != 0 + || flags & ENTRY_EXACT_BLOCK_PRESENT != 0 && flags & ENTRY_LAUNCH_FACTS_PRESENT == 0 + { + return Err(ArtifactError::Malformed(format!( + "invalid launch-facts flags {flags:#x} for entry '{symbol}'" + ))); + } + if flags & ENTRY_LAUNCH_FACTS_PRESENT != 0 { + Some(KernelLaunchFacts { + exact_block: if flags & ENTRY_EXACT_BLOCK_PRESENT != 0 { + Some(( + read_u32(bytes, record + 24)?, + read_u32(bytes, record + 28)?, + read_u32(bytes, record + 32)?, + )) + } else { + None + }, + dynamic_shared_bytes: read_u32(bytes, record + 36)?, + dynamic_shared_alignment: read_u32(bytes, record + 40)?, + }) + } else { + None + } + } else { + None + }; + validate_launch_facts(symbol, kind, launch_facts)?; entries.push(ArtifactEntry { symbol, kind, metadata, + launch_facts, }); } @@ -880,10 +975,39 @@ fn validate_spec(spec: &ArtifactBundleSpec<'_>) -> Result<(), ArtifactError> { if entry.symbol.is_empty() { return Err(ArtifactError::EmptyEntrySymbol); } + validate_launch_facts(entry.symbol, entry.kind, entry.launch_facts)?; } Ok(()) } +fn validate_launch_facts( + symbol: &str, + kind: ArtifactEntryKind, + facts: Option, +) -> Result<(), ArtifactError> { + let Some(facts) = facts else { + return Ok(()); + }; + let reason = if kind != ArtifactEntryKind::Kernel { + Some("launch facts require a kernel entry") + } else if facts + .exact_block + .is_some_and(|(x, y, z)| x == 0 || y == 0 || z == 0) + { + Some("exact block dimensions must be nonzero") + } else if !facts.dynamic_shared_alignment.is_power_of_two() { + Some("dynamic shared-memory alignment must be a nonzero power of two") + } else { + None + }; + match reason { + Some(reason) => Err(ArtifactError::Malformed(format!( + "invalid launch facts for entry '{symbol}': {reason}" + ))), + None => Ok(()), + } +} + fn checked_u16(value: usize, field: &'static str) -> Result { u16::try_from(value).map_err(|_| ArtifactError::TooLarge(field)) } @@ -1005,6 +1129,160 @@ mod tests { ); } + #[test] + fn concrete_kernel_launch_facts_survive_borrowed_and_owned_bundles() { + let first = KernelLaunchFacts { + exact_block: Some((32, 1, 1)), + dynamic_shared_bytes: 128, + dynamic_shared_alignment: 16, + }; + let second = KernelLaunchFacts { + exact_block: Some((8, 4, 1)), + dynamic_shared_bytes: 512, + dynamic_shared_alignment: 64, + }; + let empty = KernelLaunchFacts { + exact_block: None, + dynamic_shared_bytes: 0, + dynamic_shared_alignment: 1, + }; + let blob = build_artifact_blob( + &ArtifactBundleSpec::new("demo", "sm_90") + .with_compile_options(ArtifactCompileOptions::new().with_fma_contraction(false)) + .with_payload(ArtifactPayloadSpec::new( + ArtifactPayloadKind::Cubin, + "demo.cubin", + b"cubin", + )) + .with_entry( + ArtifactEntrySpec::new("apply_items_1", ArtifactEntryKind::Kernel) + .with_metadata(7) + .with_launch_facts(first), + ) + .with_entry( + ArtifactEntrySpec::new("apply_items_4", ArtifactEntryKind::Kernel) + .with_launch_facts(second), + ) + .with_entry( + ArtifactEntrySpec::new("no_scratch", ArtifactEntryKind::Kernel) + .with_launch_facts(empty), + ) + .with_entry(ArtifactEntrySpec::new( + "helper", + ArtifactEntryKind::DeviceFunction, + )), + ) + .unwrap(); + assert_eq!(read_u16(&blob, 8).unwrap(), ARTIFACT_VERSION); + let borrowed = parse_artifact_blob(&blob).unwrap(); + assert!(!borrowed.compile_options.fma_contraction_enabled()); + assert_eq!(borrowed.entry("apply_items_1").unwrap().metadata, Some(7)); + assert_eq!( + borrowed.entry("apply_items_1").unwrap().launch_facts, + Some(first) + ); + assert_eq!( + borrowed.entry("apply_items_4").unwrap().launch_facts, + Some(second) + ); + assert_eq!( + borrowed.entry("no_scratch").unwrap().launch_facts, + Some(empty) + ); + assert_eq!(borrowed.entry("helper").unwrap().launch_facts, None); + let owned = OwnedArtifactBundle::from(borrowed.clone()); + for entry in borrowed.entries { + assert_eq!( + owned.entry(entry.symbol).unwrap().launch_facts, + entry.launch_facts + ); + } + } + + #[test] + fn launch_facts_reject_invalid_shape_alignment_and_entry_kind() { + let valid = KernelLaunchFacts { + exact_block: Some((32, 1, 1)), + dynamic_shared_bytes: 128, + dynamic_shared_alignment: 16, + }; + let spec = |kind, facts| { + ArtifactBundleSpec::new("demo", "sm_90") + .with_payload(ArtifactPayloadSpec::new( + ArtifactPayloadKind::Ptx, + "demo.ptx", + b"ptx", + )) + .with_entry(ArtifactEntrySpec::new("hello", kind).with_launch_facts(facts)) + }; + for (kind, facts) in [ + (ArtifactEntryKind::DeviceFunction, valid), + ( + ArtifactEntryKind::Kernel, + KernelLaunchFacts { + exact_block: Some((32, 0, 1)), + ..valid + }, + ), + ( + ArtifactEntryKind::Kernel, + KernelLaunchFacts { + dynamic_shared_alignment: 0, + ..valid + }, + ), + ( + ArtifactEntryKind::Kernel, + KernelLaunchFacts { + dynamic_shared_alignment: 3, + ..valid + }, + ), + ] { + assert!(matches!( + build_artifact_blob(&spec(kind, facts)), + Err(ArtifactError::Malformed(_)) + )); + } + + let blob = build_artifact_blob(&spec(ArtifactEntryKind::Kernel, valid)).unwrap(); + let record = sample_payload_record_start() + PAYLOAD_RECORD_BYTES; + for (offset, value) in [(28, 0), (40, 3)] { + let mut corrupt = blob.clone(); + write_u32(&mut corrupt, record + offset, value); + assert!(matches!( + parse_artifact_blob(&corrupt), + Err(ArtifactError::Malformed(_)) + )); + } + let mut wrong_kind = blob.clone(); + write_u16( + &mut wrong_kind, + record, + ArtifactEntryKind::DeviceFunction.to_u16(), + ); + assert!(matches!( + parse_artifact_blob(&wrong_kind), + Err(ArtifactError::Malformed(_)) + )); + for flags in [ENTRY_EXACT_BLOCK_PRESENT, KNOWN_ENTRY_FLAGS | (1 << 3)] { + let mut corrupt = blob.clone(); + write_u16(&mut corrupt, record + 2, flags); + assert!(matches!( + parse_artifact_blob(&corrupt), + Err(ArtifactError::Malformed(_)) + )); + } + + let mut truncated = blob[..record + ENTRY_RECORD_BYTES - 1].to_vec(); + let len = truncated.len() as u32; + write_u32(&mut truncated, 12, len); + assert_eq!( + parse_artifact_blob(&truncated), + Err(ArtifactError::Truncated("entry records")) + ); + } + #[test] fn artifact_blob_round_trips_non_ptx_payload_kinds() { let blob = build_artifact_blob( @@ -1027,7 +1305,10 @@ mod tests { )), ) .unwrap(); - assert_eq!(read_u16(&blob, 8).unwrap(), ARTIFACT_VERSION); + assert_eq!( + read_u16(&blob, 8).unwrap(), + COMPILE_OPTIONS_ARTIFACT_VERSION + ); let bundles = parse_artifact_section(&blob).unwrap(); assert_eq!(bundles.len(), 1); @@ -1065,7 +1346,7 @@ mod tests { #[test] fn version_2_rejects_unknown_compile_option_bits() { let mut blob = sample_blob(); - write_u16(&mut blob, 8, ARTIFACT_VERSION); + write_u16(&mut blob, 8, COMPILE_OPTIONS_ARTIFACT_VERSION); write_u64(&mut blob, 24, 1 << 63); assert!(matches!( @@ -1110,7 +1391,10 @@ mod tests { ) .unwrap(); - assert_eq!(read_u16(&blob, 8).unwrap(), ARTIFACT_VERSION); + assert_eq!( + read_u16(&blob, 8).unwrap(), + COMPILE_OPTIONS_ARTIFACT_VERSION + ); assert_eq!( parse_artifact_blob(&blob).unwrap().compile_options, expected @@ -1121,7 +1405,7 @@ mod tests { #[test] fn version_2_rejects_conflicting_debug_bits() { let mut blob = sample_blob(); - write_u16(&mut blob, 8, ARTIFACT_VERSION); + write_u16(&mut blob, 8, COMPILE_OPTIONS_ARTIFACT_VERSION); write_u64(&mut blob, 24, OPTION_DEBUG_MASK); assert!(matches!( diff --git a/cuda-oxide/crates/reserved-oxide-symbols/src/lib.rs b/cuda-oxide/crates/reserved-oxide-symbols/src/lib.rs index 010bc715d2..0c4e417aeb 100644 --- a/cuda-oxide/crates/reserved-oxide-symbols/src/lib.rs +++ b/cuda-oxide/crates/reserved-oxide-symbols/src/lib.rs @@ -72,6 +72,37 @@ pub const MATERIALIZER_PROVENANCE_ENV: &str = "CUDA_OXIDE_INTERNAL_MATERIALIZER_ /// code remains keyed by the content-derived provenance digest. pub const MATERIALIZER_HANDSHAKE_ENV: &str = "CUDA_OXIDE_INTERNAL_MATERIALIZER_HANDSHAKE"; +/// Ordered, content-bound external LTO-IR files supplied by cargo-oxide. +pub const EXTERNAL_LTOIR_ENV: &str = "CUDA_OXIDE_INTERNAL_EXTERNAL_LTOIR_V1"; + +/// Definition name in `cuda_device::library`; consumers resolve its actual DefId. +pub const LIBRARY_CALL_V1: &str = "cuda_oxide_library_v1_246e25db_call"; +/// Compiler-rooted synchronous scalar callback marker. +pub const LIBRARY_CALLBACK_CALL_V1: &str = "cuda_oxide_library_v1_246e25db_call_with_callback"; + +/// Full-block marker whose controls and participation require compiler proof. +pub const LIBRARY_CHECKED_BLOCK_CALL_V1: &str = "cuda_oxide_library_v1_246e25db_checked_block_call"; +/// Full-block marker retaining a typed allocation witness and access mode. +pub const LIBRARY_CHECKED_BLOCK_RESOURCE_CALL_V1: &str = + "cuda_oxide_library_v1_246e25db_checked_block_resource_call"; + +/// Internal full-block storage marker. +pub const LIBRARY_CHECKED_BLOCK_STORAGE_CALL_V1: &str = + "cuda_oxide_library_v1_246e25db_checked_block_storage_call"; +/// Internal full-block storage marker. +pub const LIBRARY_STORAGE_NEW_V1: &str = "cuda_oxide_library_v1_246e25db_storage_new"; +/// Internal full-block storage marker. +pub const LIBRARY_STORAGE_RECONFIGURE_V1: &str = + "cuda_oxide_library_v1_246e25db_storage_reconfigure"; +/// Internal full-block storage marker. +pub const LIBRARY_STORAGE_RESERVE_V1: &str = "cuda_oxide_library_v1_246e25db_storage_reserve"; +/// Internal full-block storage marker. +pub const LIBRARY_STORAGE_RELEASE_V1: &str = "cuda_oxide_library_v1_246e25db_storage_release"; + +/// Final emitted identity for a helper specialized to one checked kernel root. +/// Embedded Rust symbols may contain device prefixes and must not be stripped. +pub const LIBRARY_CONTEXT_PREFIX: &str = "cuda_oxide_library_context_246e25db_"; + /// Optional comma-separated filter selecting crates that may own device code. pub const DEVICE_CODEGEN_CRATE_ENV: &str = "CUDA_OXIDE_DEVICE_CODEGEN_CRATE"; @@ -143,6 +174,11 @@ pub const LEGACY_DEVICE_PREFIX: &str = "cuda_oxide_device_246e25db_"; /// linker resolves against the original (unprefixed) external symbol. pub const DEVICE_EXTERN_PREFIX: &str = "cuda_oxide_device_extern_246e25db_"; +/// Doc-alias marker retained after `#[convergent]` macro expansion. +/// Unlike doc comments, aliases survive dependency metadata even for private +/// externs reached through a downstream generic instantiation. +pub const DEVICE_EXTERN_CONVERGENT_MARKER: &str = "cuda_oxide_device_extern_convergent_246e25db"; + /// Prefix added to closure-monomorphization helper functions generated /// by `#[kernel]` for kernels that take a closure parameter. /// diff --git a/cuda-oxide/crates/rustc-codegen-cuda/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/Cargo.lock index 512dd99178..30ebe3f861 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/Cargo.lock @@ -548,6 +548,7 @@ dependencies = [ "linkme", "llvm-export", "mir-lower", + "mir-transforms", "once_cell", "oxide-artifacts", "pliron", @@ -578,6 +579,8 @@ name = "mir-transforms" version = "0.2.1" dependencies = [ "dialect-mir", + "dialect-nvvm", + "oxide-artifacts", "pliron", "reserved-oxide-symbols", "rustc-hash", @@ -627,8 +630,6 @@ checksum = "0b61fbf7ec37342ce171d9f5e3b82be74bda6c4e84e3b28338a03d553913e6aa" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] @@ -815,6 +816,7 @@ dependencies = [ "pliron", "pliron-derive", "reserved-oxide-symbols", + "serde", "serde_json", "thiserror", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/Cargo.toml b/cuda-oxide/crates/rustc-codegen-cuda/Cargo.toml index c20211c939..d1c3fb3598 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/Cargo.toml +++ b/cuda-oxide/crates/rustc-codegen-cuda/Cargo.toml @@ -26,7 +26,7 @@ dialect-nvvm = { path = "../dialect-nvvm" } reserved-oxide-symbols = { path = "../reserved-oxide-symbols" } # `#[tile_export]` descriptions, read back out of the metadata functions' MIR. oxide-tile-export = { path = "../oxide-tile-export" } -oxide-artifacts = { version = "0.2.1", features = ["object-write"] } +oxide-artifacts = { version = "0.2.1", path = "../oxide-artifacts", features = ["object-write"] } # Shared driver-independent libNVVM + nvJitLink finalization policy. This # crate loads CUDA compiler libraries at run time and never links libcuda. cuda-artifact-finalizer = { path = "../cuda-artifact-finalizer" } @@ -42,6 +42,7 @@ combine = "4.6.8" # must match workspace; Pliron needs SourcePosition: H once_cell = "1.18" # must match workspace thiserror = "2" # must match workspace serde_json = "1" # must match workspace +serde = { version = "1", features = ["derive"] } # Note: rustc internal crates are NOT listed here. # They're accessed via `extern crate` with #![feature(rustc_private)] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/abi_hmm/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/abi_hmm/Cargo.lock index 894c5bdc74..b61cbb25eb 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/abi_hmm/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/abi_hmm/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/addressof_sharedarray/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/addressof_sharedarray/Cargo.lock index aec55dddd1..e5ee0008af 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/addressof_sharedarray/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/addressof_sharedarray/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/aligned_field_loads/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/aligned_field_loads/Cargo.lock index 66d6f345c4..921af297cc 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/aligned_field_loads/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/aligned_field_loads/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/aligned_field_stores/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/aligned_field_stores/Cargo.lock index d3a69ad842..5e318116b9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/aligned_field_stores/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/aligned_field_stores/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/array_constants/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/array_constants/Cargo.lock index 2ace543695..9010ea761f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/array_constants/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/array_constants/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/array_for_loop/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/array_for_loop/Cargo.lock index 50e069c917..bd29bc2233 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/array_for_loop/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/array_for_loop/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/array_index/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/array_index/Cargo.lock index 9aacc03b6a..0df32917f3 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/array_index/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/array_index/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/array_init/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/array_init/Cargo.lock index 23873c746e..0be3348c82 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/array_init/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/array_init/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/asin_acos_smoke/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/asin_acos_smoke/Cargo.lock index 11978cf8c3..8c98074f13 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/asin_acos_smoke/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/asin_acos_smoke/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/assume_init_uninhabited_trap/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/assume_init_uninhabited_trap/Cargo.lock index 57c5ac3626..0e6d47b26e 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/assume_init_uninhabited_trap/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/assume_init_uninhabited_trap/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/async_mlp/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/async_mlp/Cargo.lock index 61356c7cf0..e984e1885d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/async_mlp/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/async_mlp/Cargo.lock @@ -222,6 +222,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -560,8 +562,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/async_vecadd/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/async_vecadd/Cargo.lock index 0f89847696..0e6aaa990b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/async_vecadd/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/async_vecadd/Cargo.lock @@ -222,6 +222,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -560,8 +562,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/atomic_f16/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/atomic_f16/Cargo.lock index 2eedfa7f06..4b444b32b3 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/atomic_f16/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/atomic_f16/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/atomics/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/atomics/Cargo.lock index e62f2b5116..cbaf8353f0 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/atomics/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/atomics/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/barrier/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/barrier/Cargo.lock index 167540b149..dfa6ec6f08 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/barrier/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/barrier/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/bf16x2_arith/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/bf16x2_arith/Cargo.lock index 21784a78d9..64f35c4938 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/bf16x2_arith/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/bf16x2_arith/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/bf16x2_fma/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/bf16x2_fma/Cargo.lock index 02c07a415e..e73b06b0fd 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/bf16x2_fma/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/bf16x2_fma/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/bool_phi_cmp/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/bool_phi_cmp/Cargo.lock index dd5b1d27f2..9c01cf23e8 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/bool_phi_cmp/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/bool_phi_cmp/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/carrying_mul_add/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/carrying_mul_add/Cargo.lock index 4029312b7b..468dd3ee25 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/carrying_mul_add/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/carrying_mul_add/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cast_chunks/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cast_chunks/Cargo.lock index 23b2870e9f..03f8bb969e 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cast_chunks/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cast_chunks/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cast_tests/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cast_tests/Cargo.lock index 3b2f05431c..1b718bc114 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cast_tests/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cast_tests/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cbrt_smoke/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cbrt_smoke/Cargo.lock index ed4913c237..39b1636454 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cbrt_smoke/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cbrt_smoke/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/checked_arith/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/checked_arith/Cargo.lock index 68c6a15821..617f7f152b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/checked_arith/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/checked_arith/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/clc/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/clc/Cargo.lock index e47f313cfc..5d86097f9e 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/clc/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/clc/Cargo.lock @@ -199,6 +199,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -420,8 +422,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cluster/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cluster/Cargo.lock index 5d785dbfe8..6678cfdb87 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cluster/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cluster/Cargo.lock @@ -199,6 +199,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -420,8 +422,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/compiled_launch_facts/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/compiled_launch_facts/Cargo.lock new file mode 100644 index 0000000000..c1dd14428b --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/compiled_launch_facts/Cargo.lock @@ -0,0 +1,966 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn", + "which", +] + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "compiled_launch_facts" +version = "0.1.0" +dependencies = [ + "cuda-async", + "cuda-core", + "cuda-device", + "cuda-host", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "cuda-artifact-finalizer" +version = "0.2.1" +dependencies = [ + "libnvvm-sys", + "nvjitlink-sys", + "serde", + "sha2", + "thiserror 2.0.18", +] + +[[package]] +name = "cuda-async" +version = "0.5.0" +dependencies = [ + "anyhow", + "cuda-bindings", + "cuda-core", + "dashmap", + "futures", + "half", + "once_cell", + "rustc-hash 2.1.2", + "thiserror 1.0.69", +] + +[[package]] +name = "cuda-bindings" +version = "0.5.0" +dependencies = [ + "bindgen", + "libloading", + "prettyplease", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cuda-core" +version = "0.5.0" +dependencies = [ + "anyhow", + "cuda-bindings", + "cuda-core-derive", + "half", + "oxide-artifacts", +] + +[[package]] +name = "cuda-core-derive" +version = "0.5.0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cuda-device" +version = "0.2.1" +dependencies = [ + "cuda-macros", +] + +[[package]] +name = "cuda-host" +version = "0.2.1" +dependencies = [ + "cuda-artifact-finalizer", + "cuda-async", + "cuda-core", + "cuda-device", + "cuda-macros", + "half", + "libc", + "oxide-artifacts", + "ptx-parse", + "sha2", + "thiserror 2.0.18", +] + +[[package]] +name = "cuda-macros" +version = "0.2.1" +dependencies = [ + "oxide-tile-export", + "proc-macro2", + "quote", + "reserved-oxide-symbols", + "serde", + "serde_json", + "syn", +] + +[[package]] +name = "cuda-target-spec" +version = "0.1.0" + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libnvvm-sys" +version = "0.2.1" +dependencies = [ + "cuda-target-spec", + "libloading", + "thiserror 2.0.18", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nvjitlink-sys" +version = "0.2.1" +dependencies = [ + "libloading", + "thiserror 2.0.18", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "crc32fast", + "hashbrown 0.15.5", + "indexmap", + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oxide-artifacts" +version = "0.2.1" +dependencies = [ + "object", +] + +[[package]] +name = "oxide-tile-export" +version = "0.2.1" +dependencies = [ + "serde", + "serde_json", + "thiserror 2.0.18", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "ptx-parse" +version = "0.1.0" + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reserved-oxide-symbols" +version = "0.2.1" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/compiled_launch_facts/Cargo.toml b/cuda-oxide/crates/rustc-codegen-cuda/examples/compiled_launch_facts/Cargo.toml new file mode 100644 index 0000000000..5d4e02b884 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/compiled_launch_facts/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "compiled_launch_facts" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" + +[workspace] + +[features] +generic = [] +invalid-raw = [] + +[dependencies] +cuda-device = { path = "../../../cuda-device" } +cuda-host = { path = "../../../cuda-host", features = ["async"] } +cuda-core = { path = "../../../../../cuda-core" } +cuda-async = { path = "../../../../../cuda-async" } diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/compiled_launch_facts/README.md b/cuda-oxide/crates/rustc-codegen-cuda/examples/compiled_launch_facts/README.md new file mode 100644 index 0000000000..21d9b3e2f9 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/compiled_launch_facts/README.md @@ -0,0 +1,36 @@ +# Compiled launch facts + +This example uses `dynamic_shared = inferred` with unsafe fixed views of the +CUDA dynamic shared-memory base. `Scratch<1>` and `Scratch<4>` have different +compiler layouts, reached through the same generic helper. A second view aliases +the base and raises its alignment to 128 bytes. The views combine by maximum +extent and alignment; they do not allocate separate regions. The helper writes +one slot per lane and explicitly synchronizes before reading neighboring slots. + +The generated prepared launch methods read the facts from the selected embedded +artifact. `LaunchConfig2D::auto_shared` selects 128 or 512 bytes. Preparation +rejects an explicit 511-byte allocation for the larger kernel and rejects a +32-by-1 block even though it has the same thread count as the required 8-by-4 +block. The example also checks known-zero scratch and returns an owned output +buffer only after asynchronous launch completion. + +From `cuda-oxide/`, run with an architecture supported by your device: + +```sh +cargo oxide run compiled_launch_facts --arch sm_120 +cargo oxide run compiled_launch_facts --arch sm_120 --features generic +cargo oxide run compiled_launch_facts --arch sm_120 --materialize-cubin +cargo oxide run compiled_launch_facts --arch sm_120 --features generic --materialize-cubin +scripts/check-compiled-launch-facts.sh sm_120 +``` + +The script also checks that an inferred kernel containing an unconstrained raw +dynamic view fails to compile. Generic kernels load either the merged PTX set or a single materialized native +artifact containing their concrete specializations. Multiple native artifacts or +mixed native/PTX bundles are rejected to preserve one module's device globals. Explicit raw pointer access, +initialization, bounds, aliasing, and synchronization remain the responsibility +of the unsafe device code. No synchronization is inserted by `get_fixed`. +Inference follows Rust views through direct calls. Dynamic-base accesses in +external code, indirect calls or inline assembly must fit a fixed view declared +on that path; they are not analyzed. Without declared views, inference records +zero bytes. diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/compiled_launch_facts/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/compiled_launch_facts/src/main.rs new file mode 100644 index 0000000000..fcd6049324 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/compiled_launch_facts/src/main.rs @@ -0,0 +1,162 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use cuda_device::{DisjointSlice, kernel, launch_contract, shared::DynamicSharedArray, thread}; +use cuda_host::cuda_module; + +#[repr(C, align(64))] +struct Scratch { + slots: [[u32; 32]; ITEMS], +} + +#[cuda_module] +mod kernels { + use super::*; + + #[inline(never)] + fn exchange() -> u32 { + let lane = (thread::threadIdx_y() * 8 + thread::threadIdx_x()) as usize; + // Both views alias the same base. Every lane writes its own slots; + // the explicit barrier completes those writes before neighbor reads. + unsafe { + let scratch = DynamicSharedArray::, 16>::get_fixed().cast::(); + let alias = DynamicSharedArray::<[u32; 32], 128>::get_fixed().cast::(); + for item in 0..ITEMS { + scratch + .add(item * 32 + lane) + .write_volatile((item * 100 + lane) as u32); + } + thread::sync_threads(); + let neighbor = (lane + 1) % 32; + scratch.add((ITEMS - 1) * 32 + neighbor).read_volatile() + + alias.add(neighbor).read_volatile() + } + } + + #[kernel] + #[launch_contract(domain = 2, block = (8, 4, 1), dynamic_shared = inferred)] + pub fn fixed_one(mut output: DisjointSlice>) { + let value = exchange::<1>(); + if let Some(index) = thread::index_2d::<8>() + && let Some(out) = output.get_mut(index) + { + *out = value; + } + } + + #[kernel] + #[launch_contract(domain = 2, block = (8, 4, 1), dynamic_shared = inferred)] + pub fn fixed_four(mut output: DisjointSlice>) { + let value = exchange::<4>(); + if let Some(index) = thread::index_2d::<8>() + && let Some(out) = output.get_mut(index) + { + *out = value; + } + } + + #[kernel] + #[launch_contract(domain = 2, block = (8, 4, 1), dynamic_shared = inferred)] + pub fn zero() {} + + #[cfg(feature = "generic")] + #[launch_contract(domain = 2, block = (8, 4, 1), dynamic_shared = inferred)] + #[kernel] + pub fn generic(mut output: DisjointSlice>) { + let value = exchange::(); + if let Some(index) = thread::index_2d::<8>() + && let Some(out) = output.get_mut(index) + { + *out = value; + } + } + + #[cfg(feature = "invalid-raw")] + #[kernel] + #[launch_contract(domain = 2, block = (8, 4, 1), dynamic_shared = inferred)] + pub fn incomplete() { + unsafe { + DynamicSharedArray::<[u32; 32], 16>::get_fixed() + .cast::() + .write_volatile(1); + DynamicSharedArray::::get() + .add(64) + .write_volatile(2); + } + } +} + +fn verify(buffer: &cuda_core::DeviceBuffer, stream: &cuda_core::CudaStream, items: u32) { + let mut values = [0; 32]; + buffer.copy_to_host(stream, &mut values).unwrap(); + stream.synchronize().unwrap(); + for (lane, value) in values.into_iter().enumerate() { + assert_eq!(value, (items - 1) * 100 + 2 * ((lane as u32 + 1) % 32)); + } +} + +fn main() -> Result<(), Box> { + use cuda_async::simt::device_context::{init_device_contexts, with_cuda_context}; + use cuda_async::simt::device_operation::DeviceOperation; + use cuda_core::{DeviceBuffer, LaunchConfig2D}; + + init_device_contexts(0, 1)?; + let ctx = with_cuda_context(0, |ctx| ctx.clone())?; + let stream = ctx.default_stream(); + // SAFETY: the embedded module is compiled from these kernel declarations. + let module = unsafe { kernels::load_async(0)? }; + let config = LaunchConfig2D::auto_shared((1, 1), (8, 4)); + let one = module.prepare_fixed_one(config)?; + let four = module.prepare_fixed_four(config)?; + let zero = module.prepare_zero(config)?; + assert_eq!(one.__raw_config().shared_mem_bytes, 128); + assert_eq!(four.__raw_config().shared_mem_bytes, 512); + assert_eq!(zero.__raw_config().shared_mem_bytes, 0); + for prepared_facts in [ + one.function().launch_facts(), + four.function().launch_facts(), + ] { + let facts = prepared_facts.unwrap(); + assert_eq!(facts.exact_block, Some((8, 4, 1))); + assert_eq!(facts.dynamic_shared_alignment, 128); + } + assert_eq!( + zero.function().launch_facts().unwrap().dynamic_shared_bytes, + 0 + ); + assert!(matches!( + module.prepare_fixed_four(LaunchConfig2D::auto_shared((1, 1), (32, 1))), + Err(cuda_core::LaunchContractError::BlockShapeMismatch { .. }) + )); + assert!(matches!( + module.prepare_fixed_four(LaunchConfig2D::new((1, 1), (8, 4), 511)), + Err(cuda_core::LaunchContractError::DynamicSharedMemoryExactMismatch { .. }) + )); + let explicit = module.prepare_fixed_four(LaunchConfig2D::new((1, 1), (8, 4), 512))?; + + let output = DeviceBuffer::from_host(&stream, &[0u32; 32])?; + let output = module.fixed_one_async_owned(&one, output).sync()?; + verify(&output, &stream, 1); + let output = module.fixed_four_async_owned(&explicit, output).sync()?; + verify(&output, &stream, 4); + let output = module.fixed_four_async_owned(&four, output).sync()?; + verify(&output, &stream, 4); + + #[cfg(feature = "generic")] + { + let one = module.prepare_generic::<1>(config)?; + let four = module.prepare_generic::<4>(config)?; + assert_eq!(one.__raw_config().shared_mem_bytes, 128); + assert_eq!(four.__raw_config().shared_mem_bytes, 512); + let output = module.generic_async_owned::<1, _>(&one, output).sync()?; + verify(&output, &stream, 1); + let output = module.generic_async_owned::<4, _>(&four, output).sync()?; + verify(&output, &stream, 4); + } + println!( + "PASS: compiled launch facts: helper layouts, exact shape, explicit/automatic bytes, zero, and owned async passed" + ); + Ok(()) +} diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/compiler_features/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/compiler_features/Cargo.lock index f74d2527f8..56be2f9157 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/compiler_features/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/compiler_features/Cargo.lock @@ -199,6 +199,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -420,8 +422,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/complex_mul/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/complex_mul/Cargo.lock index 8e9267dce6..b120f7d4ba 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/complex_mul/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/complex_mul/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_aggregate/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_aggregate/Cargo.lock index 1b55fe2102..11f4a52ad8 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_aggregate/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_aggregate/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_bool_dead_branch/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_bool_dead_branch/Cargo.lock index 1624a6b9f9..d0129db5b8 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_bool_dead_branch/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_bool_dead_branch/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_generic/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_generic/Cargo.lock index 59af5b4d4c..3722835bef 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_generic/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_generic/Cargo.lock @@ -199,6 +199,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -420,8 +422,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_table_lookup/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_table_lookup/Cargo.lock index 6378ac770e..b392b08c49 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_table_lookup/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_table_lookup/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_tuple_table_lookup/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_tuple_table_lookup/Cargo.lock index e5bb7b77b8..d8e511de7e 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/const_tuple_table_lookup/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/const_tuple_table_lookup/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_index_from_end/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_index_from_end/Cargo.lock index 4a5ddfcbfc..4c4cc03681 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_index_from_end/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_index_from_end/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory/Cargo.lock index ec1b311b82..d7a2fdc0fb 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_coeffs/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_coeffs/Cargo.lock index 254f6e1446..dd05a21911 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_coeffs/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_coeffs/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_simple/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_simple/Cargo.lock index 0290d6b02d..f1119f8ece 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_simple/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_simple/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_table/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_table/Cargo.lock index 5ee35f9f49..6361c87d93 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_table/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/constant_memory_table/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/coop_groups_demo/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/coop_groups_demo/Cargo.lock index 1929b90d68..0892c097ec 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/coop_groups_demo/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/coop_groups_demo/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/copy_aggregate_borrow/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/copy_aggregate_borrow/Cargo.lock index 861cd541fa..a6f836ffd2 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/copy_aggregate_borrow/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/copy_aggregate_borrow/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/copy_nonoverlapping/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/copy_nonoverlapping/Cargo.lock index e0ad2e575b..69fa7aa0e7 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/copy_nonoverlapping/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/copy_nonoverlapping/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/counted_barrier/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/counted_barrier/Cargo.lock index 670a99a5a0..5d37a68a90 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/counted_barrier/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/counted_barrier/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cp_async_small/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cp_async_small/Cargo.lock index e2b60b35e4..6ec9a630ff 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cp_async_small/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cp_async_small/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cp_async_zfill/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cp_async_zfill/Cargo.lock index 554696b9a9..1841b399ca 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cp_async_zfill/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cp_async_zfill/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cpp_consumes_rust_device/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cpp_consumes_rust_device/Cargo.lock index 65b1237134..08e3c5ad23 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cpp_consumes_rust_device/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cpp_consumes_rust_device/Cargo.lock @@ -197,6 +197,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -418,8 +420,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_embedded/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_embedded/Cargo.lock index 86fd1896bc..384be6d492 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_embedded/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_embedded/Cargo.lock @@ -199,6 +199,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -429,8 +431,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/Cargo.lock index 93b3a02f35..549a3770b7 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/Cargo.lock @@ -200,6 +200,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -430,8 +432,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/Cargo.toml b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/Cargo.toml index 4a75520499..b37ac28fc1 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/Cargo.toml +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/Cargo.toml @@ -17,3 +17,7 @@ cuda-device = { path = "../../../cuda-device" } cuda-host = { path = "../../../cuda-host" } cuda-core = { path = "../../../../../cuda-core" } ptx-parse = { path = "../../../ptx-parse" } + +[features] +# Adds a second emitting crate to exercise the native-loader diagnostic. +native-ambiguity = ["kernel-lib/native-ambiguity"] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/README.md b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/README.md index aff5decf2b..66d7916b4d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/README.md +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/README.md @@ -1,5 +1,10 @@ # Cross-Crate Kernel Example +The `native-ambiguity` feature adds a concrete library kernel. With +`--materialize-cubin`, this produces separate library and consumer CUBINs; +the generic loader reports their incompatible topology. The compiled launch +facts script exercises this expected rejection. + This example demonstrates **defining type- and const-generic kernels in a library crate** and instantiating them from a binary crate. @@ -27,6 +32,7 @@ cross_crate_kernel/ ```bash cargo oxide run cross_crate_kernel +cargo oxide run cross_crate_kernel --arch sm_120 --materialize-cubin ``` The built executable can compare its own lookup names against generated PTX diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/kernel-lib/Cargo.toml b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/kernel-lib/Cargo.toml index 3300772048..b694ed873c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/kernel-lib/Cargo.toml +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/kernel-lib/Cargo.toml @@ -9,3 +9,6 @@ license = "Apache-2.0" cuda-core = { path = "../../../../../../cuda-core" } cuda-device = { path = "../../../../cuda-device" } cuda-host = { path = "../../../../cuda-host" } + +[features] +native-ambiguity = [] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/kernel-lib/src/lib.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/kernel-lib/src/lib.rs index 06fb421ff4..edd30936fe 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/kernel-lib/src/lib.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/kernel-lib/src/lib.rs @@ -17,6 +17,11 @@ use cuda_device::{DisjointSlice, cuda_module, kernel, thread}; pub mod kernels { use super::*; + /// Emits a separate library artifact for the native ambiguity check. + #[cfg(feature = "native-ambiguity")] + #[kernel] + pub fn concrete() {} + /// Generic scale kernel - multiplies each element by a factor. /// /// This kernel is exported from the library and can be instantiated diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/src/main.rs index 36388bf446..944f0aeb2c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_kernel/src/main.rs @@ -114,11 +114,9 @@ fn main() { // The device code is embedded in this binary, so there is no loose `.ptx` // to locate or keep in step. // - // `kernels` is generic, and a generic module's generated loader merges the - // PTX bundles of every crate in the build instead of selecting one by name - // (it discards the crate-name hint outright). That merge is what makes the - // specializations below resolvable at all: they are monomorphized here, in - // the binary, from kernels declared in kernel-lib. + // These specializations belong to the consuming binary. Generic loading + // merges its PTX bundles or loads the sole complete materialized artifact, + // then resolves each exact specialization name in that one CUDA module. let module = kernels::load(&ctx).expect("Failed to load embedded CUDA module"); // Test data @@ -300,7 +298,7 @@ fn main() { println!("\nThis demonstrates:"); println!(" - Generic kernels can be defined in library crates"); println!(" - They are monomorphized when used in the application"); - println!(" - PTX is generated for all used instantiations"); + println!(" - Device code is generated for all used instantiations"); println!(" - Const values participate in cross-crate kernel identity"); println!(" - Device helper functions from libraries also work"); } diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_merged_symbols/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_merged_symbols/Cargo.lock index 65317d8b2c..015244beff 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_merged_symbols/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_merged_symbols/Cargo.lock @@ -199,6 +199,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -429,8 +431,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_recursion/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_recursion/Cargo.lock index 02fa70fa98..af5245187e 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_recursion/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cross_crate_recursion/Cargo.lock @@ -199,6 +199,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -420,8 +422,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_contract/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_contract/Cargo.lock index 4265107f33..d6e13e508a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_contract/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_contract/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -420,8 +422,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_in_lib/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_in_lib/Cargo.lock index e21415db54..f2e91266b5 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_in_lib/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_in_lib/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -429,8 +431,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_nested/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_nested/Cargo.lock index 9cffb5991d..2c608c0120 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_nested/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cuda_module_nested/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cutile_inter_kernel/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cutile_inter_kernel/Cargo.lock index 3469a4e1aa..a193b9e820 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cutile_inter_kernel/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cutile_inter_kernel/Cargo.lock @@ -713,8 +713,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object 0.36.7", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cutile_inter_kernel/simt/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cutile_inter_kernel/simt/Cargo.lock index 5b1b40426e..1c16db6d5b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cutile_inter_kernel/simt/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cutile_inter_kernel/simt/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cvt_f16x2/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cvt_f16x2/Cargo.lock index 310b4cd386..75090d956c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cvt_f16x2/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cvt_f16x2/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/cvt_packed/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/cvt_packed/Cargo.lock index a26fc79858..fa83571c17 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/cvt_packed/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/cvt_packed/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/debug-tests/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/debug-tests/Cargo.lock index 896adc4fe1..412c98d7da 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/debug-tests/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/debug-tests/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -420,8 +422,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/debug/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/debug/Cargo.lock index a9b382b282..b3f053a6bb 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/debug/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/debug/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -420,8 +422,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/debug_pointer_locals/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/debug_pointer_locals/Cargo.lock index c6fcff1155..a4bed0209c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/debug_pointer_locals/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/debug_pointer_locals/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_closures/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_closures/Cargo.lock index cd74327ba3..656a35c2c4 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_closures/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_closures/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.lock index 547e2d265c..b1d85005e6 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -196,6 +198,13 @@ dependencies = [ name = "cuda-target-spec" version = "0.1.0" +[[package]] +name = "device-ffi-collectives" +version = "0.1.0" +dependencies = [ + "cuda-device", +] + [[package]] name = "device_ffi_test" version = "0.1.0" @@ -203,6 +212,7 @@ dependencies = [ "cuda-core", "cuda-device", "cuda-host", + "device-ffi-collectives", ] [[package]] @@ -419,8 +429,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.toml b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.toml index 28ace2a1d4..3c7fef7608 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.toml +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/Cargo.toml @@ -11,3 +11,4 @@ license = "Apache-2.0" cuda-device = { path = "../../../cuda-device" } cuda-host = { path = "../../../cuda-host" } cuda-core = { path = "../../../../../cuda-core" } +device-ffi-collectives = { path = "convergence-helper" } diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/README.md b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/README.md index d02e6056cd..b71a620120 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/README.md +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/README.md @@ -125,16 +125,17 @@ device_ffi_test/ ## How It Works -### 1. Rust Kernel Code (`src/main.rs`) +### 1. Rust Kernel Code (simplified) ```rust -use cuda_device::{device, kernel}; +use cuda_device::{convergent, device, kernel}; // Declare external device functions from CUDA C++ -// Note: No NVVM attributes needed - see "Why No NVVM Attributes?" below +// Mark collectives before the Rust device module is optimized. #[device] unsafe extern "C" { fn magnitude_squared(x: f32, y: f32) -> f32; + #[convergent] fn warp_reduce_sum(val: f32) -> f32; } @@ -205,46 +206,27 @@ callers that deliberately use this mapping may allow that lint on the extern. --- -## Why No NVVM Attributes on Extern Declarations? +## Convergence on Extern Declarations -When using LTOIR linking, **NVVM attributes (`#[convergent]`, `#[pure]`, `#[readonly]`) -are not needed on Rust extern function declarations**. Here's why: +Mark external warp and block collective declarations with `#[convergent]`. +cuda-oxide emits LLVM's `convergent` attribute on each marked declaration before +libNVVM optimizes the Rust device module. Attributes on external definitions +arrive later during LTO and cannot protect earlier optimization. -### The LTOIR Already Has Proper Attributes +The annotation forbids transformations that add control dependencies to the +call. It does not make threads reach the call or guarantee reconvergence; +callers must satisfy the collective's warp or block participation requirements. +Reachable declarations of the same symbol must agree on the annotation. Ordinary +scalar helpers and storage-size queries do not need this annotation. -External CUDA C++ compiled with `nvcc -dc -dlto` produces LTOIR with complete -attribute specifications on the function **definitions**: +The `convergence-helper` dependency declares `warp_reduce_sum` privately and +calls it through a public generic device helper. `verify-code-shape.sh` checks +that this declaration retains convergence in the consuming module, alongside +local collective declarations and unmarked scalar declarations. -```llvm -; From cccl_wrappers_text.ltoir -attributes #0 = { convergent inlinehint mustprogress willreturn } -attributes #7 = { convergent nounwind } -``` - -### nvJitLink Uses Definition Attributes - -During LTO linking: -1. cuda-oxide LTOIR has **declarations** (no function body) -2. External LTOIR has **definitions** with proper attributes -3. nvJitLink resolves symbols and uses the **definition's attributes** for optimization - -### Attributes on Declarations Are Redundant - -Since the linker uses the definition's attributes: -- Attributes on declarations don't affect the final code -- The external library authors already set correct attributes -- User-specified attributes could be incorrect or misleading - -### What About cuda-oxide Built-in Intrinsics? - -For cuda-oxide's own intrinsics (like `sync_threads()`, `shuffle()`, etc.), the -compiler handles convergent semantics internally. Users don't need to annotate these. - -### Historical Note - -Earlier versions of this example used `#[convergent]`, `#[pure]`, and `#[readonly]` -attributes. These have been removed as they provided no benefit with LTOIR linking -and could mislead users into thinking they were necessary. +Built-in intrinsics such as `sync_threads()` and warp shuffles already carry +convergence information. `#[pure]` and `#[readonly]` do not currently emit LLVM +memory-effect attributes. --- diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/Cargo.toml b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/Cargo.toml new file mode 100644 index 0000000000..a5a148e40d --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/Cargo.toml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "device-ffi-collectives" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" +publish = false + +[dependencies] +cuda-device = { path = "../../../../cuda-device" } diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/src/lib.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/src/lib.rs new file mode 100644 index 0000000000..52c17256c0 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/convergence-helper/src/lib.rs @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#[cuda_device::device] +unsafe extern "C" { + // Deliberately private: the downstream generic instantiation must recover + // convergence from dependency metadata without an exported declaration. + #[cuda_device::convergent] + fn warp_reduce_sum(value: f32) -> f32; +} + +/// Reduce a warp and scale its sum in a downstream device instantiation. +/// +/// # Safety +/// +/// All lanes in a full warp must participate in the reduction. +#[cuda_device::device] +pub unsafe fn scaled_warp_sum(value: f32) -> f32 { + unsafe { warp_reduce_sum(value) * SCALE as f32 } +} diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/src/main.rs index 0deb545121..02b51e9cdb 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/src/main.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/src/main.rs @@ -20,16 +20,15 @@ //! cargo run --release //! ``` -use cuda_device::{device, gpu_printf, kernel, shared::DynamicSharedArray}; +use cuda_device::{convergent, device, gpu_printf, kernel, shared::DynamicSharedArray}; use cuda_host::cuda_module; +use device_ffi_collectives::scaled_warp_sum; // ============================================================================= // External device functions from external_device_funcs.cu // -// NOTE: No NVVM attributes (#[convergent], #[pure], #[readonly]) are needed here. -// The external LTOIR already contains proper attributes on the function definitions. -// When nvJitLink performs LTO linking, it uses the definition's attributes. -// See README.md "Why No NVVM Attributes?" for details. +// Collective declarations carry convergence before libNVVM optimization. +// See README.md "Convergence on Extern Declarations" for details. // ============================================================================= #[device] @@ -37,7 +36,7 @@ unsafe extern "C" { fn magnitude_squared(x: f32, y: f32) -> f32; fn fast_rsqrt(x: f32) -> f32; fn dot_product(a: *const f32, b: *const f32, n: i32) -> f32; - fn warp_reduce_sum(val: f32) -> f32; + #[convergent] fn warp_ballot(predicate: i32) -> u32; fn simple_add(a: f32, b: f32) -> f32; // `char` is a plain 32-bit device-extern slot. Rust still warns because C @@ -56,13 +55,16 @@ unsafe extern "C" { // CCCL (CUB) wrapper functions from cccl_wrappers.cu // // These wrap CUB template functions as extern "C" for FFI. -// The CCCL LTOIR already has proper convergent/nounwind attributes. +// Mark collectives so their participation requirements survive prelink optimization. // ============================================================================= #[device] unsafe extern "C" { + #[convergent] fn cub_block_reduce_sum_f32_256(input: f32, temp: *mut u8) -> f32; + #[convergent] fn cub_block_reduce_max_f32_256(input: f32, temp: *mut u8) -> f32; + #[convergent] fn cub_warp_reduce_sum_f32(input: f32) -> f32; fn cub_get_block_reduce_temp_size_256() -> i32; } @@ -88,7 +90,7 @@ mod kernels { let added = unsafe { simple_add(mag, 1.0) }; // Test convergent function (all threads participate) - let sum = unsafe { warp_reduce_sum(added) }; + let sum = unsafe { scaled_warp_sum::<1>(added) }; // Lane 0 writes result if tid.is_multiple_of(32) { diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/verify-code-shape.sh b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/verify-code-shape.sh index 66bd4fcaf1..e61bf5750d 100755 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/verify-code-shape.sh +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_ffi_test/verify-code-shape.sh @@ -4,12 +4,14 @@ # A runtime q -> Q check cannot distinguish a plain i32 ABI from the plausible # but wrong `zeroext i32` mapping: both reach the same PTX and hardware result. -# Pin the declaration and calls in the emitted LLVM IR before LTO. +# Pin the declaration and calls in the emitted LLVM IR before LTO. Also check +# that a private extern reached through a dependency's generic helper retains +# convergence, while the ordinary char declarations remain unmarked. set -euo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ll="${root}/device_ffi_test.ll" +ll="${1:-${root}/device_ffi_test.ll}" test -s "${ll}" @@ -46,4 +48,19 @@ if grep -Eq '\b(signext|zeroext)\b' <<<"${char_lines}"; then exit 1 fi -echo "device_ffi_test char ABI shape: PASS" +# Calls retain the existing conservative convergence group; only these ordinary +# declarations must be unmarked. The anchored declaration patterns above also +# reject an unexpected attribute trailer. +if grep -E '^declare .*@(char_to_upper|char_store)\(' "${ll}" | grep -qw convergent; then + echo "error: ordinary char declarations must not be convergent" >&2 + exit 1 +fi + +for symbol in warp_reduce_sum cub_warp_reduce_sum_f32; do + require_count "convergent ${symbol} declaration" 1 \ + "^declare float @${symbol}\(float\) convergent$" +done +require_count "convergent warp_ballot declaration" 1 \ + '^declare i32 @warp_ballot\(i32\) convergent$' + +echo "device_ffi_test char ABI and convergence shape: PASS (${ll})" diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_global/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_global/Cargo.lock index cb70f208f9..4a7e666ede 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_global/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_global/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_panic_trap/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_panic_trap/Cargo.lock index 940b241f11..f50248d787 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/device_panic_trap/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/device_panic_trap/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/disjoint_from_raw_parts/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/disjoint_from_raw_parts/Cargo.lock index 047a387564..8340bf2461 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/disjoint_from_raw_parts/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/disjoint_from_raw_parts/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/disjoint_slice_len/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/disjoint_slice_len/Cargo.lock index 6489cf91da..6cea94045c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/disjoint_slice_len/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/disjoint_slice_len/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/dotprod/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/dotprod/Cargo.lock index 476fdafb8c..8e3110d87d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/dotprod/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/dotprod/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/dpx_minmax_chains/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/dpx_minmax_chains/Cargo.lock index a813d9446e..add2787035 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/dpx_minmax_chains/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/dpx_minmax_chains/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/drop_glue/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/drop_glue/Cargo.lock index f5e4bf7171..4bcbb56860 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/drop_glue/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/drop_glue/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/dst_metadata/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/dst_metadata/Cargo.lock index 818e5fb563..d3a4fb76da 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/dst_metadata/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/dst_metadata/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/dynamic_smem/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/dynamic_smem/Cargo.lock index d7a2d13df3..e0d8226027 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/dynamic_smem/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/dynamic_smem/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/elect_leader/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/elect_leader/Cargo.lock index af1b7f3494..b880a5af0a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/elect_leader/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/elect_leader/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_abi_multi_payload/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_abi_multi_payload/Cargo.lock index cf9ff9bcf6..05b09c89a4 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_abi_multi_payload/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_abi_multi_payload/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_array_match/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_array_match/Cargo.lock index bfd23bfe27..d03c010422 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_array_match/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_array_match/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_constant_provenance/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_constant_provenance/Cargo.lock index f504b4ccae..076c9aa099 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_constant_provenance/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_constant_provenance/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_payload_addr/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_payload_addr/Cargo.lock index df0cc37f42..2c811e274c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_payload_addr/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_payload_addr/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_state_transitions/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_state_transitions/Cargo.lock index f7235d4900..8cbdef629e 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_state_transitions/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_state_transitions/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_table_lookup/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_table_lookup/Cargo.lock index a3895bfdff..0961cc5b95 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_table_lookup/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/enum_table_lookup/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error/Cargo.lock index d1c009dd04..7efbea2d92 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -420,8 +422,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_enum_bool_payload_addr/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_enum_bool_payload_addr/Cargo.lock index 5621f9b7d5..f8e93156fb 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_enum_bool_payload_addr/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_enum_bool_payload_addr/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_enum_pointer_overlap/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_enum_pointer_overlap/Cargo.lock index b6a117690d..1a7be90510 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_enum_pointer_overlap/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_enum_pointer_overlap/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_enum_shared_pointer_layout/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_enum_shared_pointer_layout/Cargo.lock index 78af316b92..919eac4a5d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_enum_shared_pointer_layout/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_enum_shared_pointer_layout/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_generated_intrinsic_abi/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_generated_intrinsic_abi/Cargo.lock index cf79f57d10..9f53ae799d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_generated_intrinsic_abi/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_generated_intrinsic_abi/Cargo.lock @@ -193,6 +193,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -423,8 +425,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_generated_intrinsic_callable/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_generated_intrinsic_callable/Cargo.lock index f1e4b65bca..4b3088bc15 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_generated_intrinsic_callable/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_generated_intrinsic_callable/Cargo.lock @@ -193,6 +193,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -423,8 +425,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_generated_intrinsic_fn_pointer/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_generated_intrinsic_fn_pointer/Cargo.lock index 8f8b5bd07b..4595100adf 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_generated_intrinsic_fn_pointer/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_generated_intrinsic_fn_pointer/Cargo.lock @@ -193,6 +193,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -423,8 +425,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_generated_intrinsic_unknown_id/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_generated_intrinsic_unknown_id/Cargo.lock index 7c1967e280..8743f0cef1 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_generated_intrinsic_unknown_id/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_generated_intrinsic_unknown_id/Cargo.lock @@ -193,6 +193,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -423,8 +425,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_heap_alloc/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_heap_alloc/Cargo.lock index 4b145cbb9b..3b8e718f17 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_heap_alloc/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_heap_alloc/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_host_arch_intrinsic/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_host_arch_intrinsic/Cargo.lock index f13ccddf3d..ac6aa4a39c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_host_arch_intrinsic/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_host_arch_intrinsic/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_host_target_feature/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_host_target_feature/Cargo.lock index 896397563a..917616d1a2 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_host_target_feature/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_host_target_feature/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_kernel_shared_param/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_kernel_shared_param/Cargo.lock index 288baa5927..88f0339df3 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_kernel_shared_param/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_kernel_shared_param/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_kernel_shared_ptr_param/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_kernel_shared_ptr_param/Cargo.lock index e526285811..bbcf96364c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_kernel_shared_ptr_param/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_kernel_shared_ptr_param/Cargo.lock @@ -188,6 +188,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -418,8 +420,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_missing_device_attr/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_missing_device_attr/Cargo.lock index 73495fc325..9a7c04854e 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_missing_device_attr/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_missing_device_attr/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_set_discriminant_uninhabited/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_set_discriminant_uninhabited/Cargo.lock index 80b0290a9c..5c8d249189 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_set_discriminant_uninhabited/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_set_discriminant_uninhabited/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_union_shared_pointer_addr/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_union_shared_pointer_addr/Cargo.lock index b07deb9457..ec303531b0 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_union_shared_pointer_addr/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_union_shared_pointer_addr/Cargo.lock @@ -188,6 +188,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -418,8 +420,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_union_shared_pointer_layout/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_union_shared_pointer_layout/Cargo.lock index e6c274596d..a9ea669032 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/error_union_shared_pointer_layout/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/error_union_shared_pointer_layout/Cargo.lock @@ -188,6 +188,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -418,8 +420,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/ex2_approx_f16/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/ex2_approx_f16/Cargo.lock index a305299bef..4f21d05493 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/ex2_approx_f16/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/ex2_approx_f16/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/exact_div_chunks/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/exact_div_chunks/Cargo.lock index 7a093f77e0..8e95fc8c41 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/exact_div_chunks/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/exact_div_chunks/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/export_name_policy/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/export_name_policy/Cargo.lock index d19a8f5d96..0641a2487b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/export_name_policy/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/export_name_policy/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/extern_crate_closure/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/extern_crate_closure/Cargo.lock index f63c2d7694..e2cd44e9ae 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/extern_crate_closure/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/extern_crate_closure/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -424,8 +426,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/extern_libdevice/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/extern_libdevice/Cargo.lock index 3ccac81def..2d79555010 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/extern_libdevice/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/extern_libdevice/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/Cargo.lock new file mode 100644 index 0000000000..8098cf08c5 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/Cargo.lock @@ -0,0 +1,754 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", + "which", +] + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "cuda-artifact-finalizer" +version = "0.2.1" +dependencies = [ + "libnvvm-sys", + "nvjitlink-sys", + "serde", + "sha2", + "thiserror", +] + +[[package]] +name = "cuda-bindings" +version = "0.5.0" +dependencies = [ + "bindgen", + "libloading", + "prettyplease", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cuda-core" +version = "0.5.0" +dependencies = [ + "anyhow", + "cuda-bindings", + "cuda-core-derive", + "half", + "oxide-artifacts", +] + +[[package]] +name = "cuda-core-derive" +version = "0.5.0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cuda-device" +version = "0.2.1" +dependencies = [ + "cuda-macros", +] + +[[package]] +name = "cuda-host" +version = "0.2.1" +dependencies = [ + "cuda-artifact-finalizer", + "cuda-core", + "cuda-device", + "cuda-macros", + "half", + "libc", + "oxide-artifacts", + "ptx-parse", + "sha2", + "thiserror", +] + +[[package]] +name = "cuda-macros" +version = "0.2.1" +dependencies = [ + "oxide-tile-export", + "proc-macro2", + "quote", + "reserved-oxide-symbols", + "serde", + "serde_json", + "syn", +] + +[[package]] +name = "cuda-target-spec" +version = "0.1.0" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "external_ltoir" +version = "0.1.0" +dependencies = [ + "cuda-core", + "cuda-device", + "cuda-host", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libnvvm-sys" +version = "0.2.1" +dependencies = [ + "cuda-target-spec", + "libloading", + "thiserror", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nvjitlink-sys" +version = "0.2.1" +dependencies = [ + "libloading", + "thiserror", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "crc32fast", + "hashbrown 0.15.5", + "indexmap", + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oxide-artifacts" +version = "0.2.1" +dependencies = [ + "object", +] + +[[package]] +name = "oxide-tile-export" +version = "0.2.1" +dependencies = [ + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "ptx-parse" +version = "0.1.0" + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reserved-oxide-symbols" +version = "0.2.1" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/Cargo.toml b/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/Cargo.toml new file mode 100644 index 0000000000..0daa15796f --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/Cargo.toml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +[package] +name = "external_ltoir" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" +publish = false + +[workspace] + +[dependencies] +cuda-device = { path = "../../../cuda-device" } +cuda-host = { path = "../../../cuda-host" } +cuda-core = { path = "../../../../../cuda-core" } diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/README.md b/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/README.md new file mode 100644 index 0000000000..a81f51b014 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/README.md @@ -0,0 +1,36 @@ +# External device LTO-IR + +This example calls a CUDA C++ device function from a Rust kernel. The C++ +function calls another function in a second LTO-IR file. `cargo oxide` links +both files into the embedded cubin during the build, and the program uses the +ordinary `kernels::load` API. It checks `output[i] = 2 * input[i] + 3` on the GPU. + +From `cuda-oxide/`, with the CUDA Toolkit and a matching GPU available: + +```bash +example=crates/rustc-codegen-cuda/examples/external_ltoir +mkdir -p build/device-libs +nvcc --ltoir -dlto -arch=sm_120 \ + "$example/scale.cu" -o build/device-libs/scale.ltoir +nvcc --ltoir -dlto -arch=sm_120 \ + "$example/add.cu" -o build/device-libs/add.ltoir +cargo oxide run external_ltoir --arch sm_120 --materialize-cubin \ + --link-ltoir build/device-libs/scale.ltoir \ + --link-ltoir build/device-libs/add.ltoir +``` + +Use the same architecture for both compilers. Each `--link-ltoir` adds one +file in argument order; paths are relative to the command's working directory. +The inputs must be raw LTO-IR, not object files or fatbins. `--ltoir -dlto` +writes the explicit output path, including with architecture names such as +`sm_90a`. All supplied files are linked into each materialized device artifact +in the Cargo invocation. The executable contains the finished cubin and needs no external `.ltoir` files +or runtime link step. + +`scripts/check-external-ltoir.sh sm_120` exercises run, build and test, repeated +build freshness, content changes at the same path, a missing transitive symbol, +execution with the link inputs removed, corrupt LTO-IR, and rejection of an +input/output collision without changing either file. It prints the +temporary directory containing its logs and generated inputs; it uses the +same nvcc commands with those temporary output paths. The check requires a GPU +matching the requested architecture. diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/add.cu b/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/add.cu new file mode 100644 index 0000000000..d74d9bd981 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/add.cu @@ -0,0 +1,12 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef EXTERNAL_ADDEND +#define EXTERNAL_ADDEND 3 +#endif + +extern "C" __device__ int external_add(int value) { + return value + EXTERNAL_ADDEND; +} diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/scale.cu b/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/scale.cu new file mode 100644 index 0000000000..ea5be973c4 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/scale.cu @@ -0,0 +1,10 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +extern "C" __device__ int external_add(int value); + +extern "C" __device__ int external_scale_add(int value) { + return external_add(value * 2); +} diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/src/main.rs new file mode 100644 index 0000000000..04efbfab78 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/external_ltoir/src/main.rs @@ -0,0 +1,73 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Link two CUDA C++ device libraries during compilation and load the embedded cubin. + +use cuda_core::simt::LaunchConfig; +use cuda_core::{CudaContext, DeviceBuffer}; +use cuda_device::{DisjointSlice, device, kernel, thread}; +use cuda_host::cuda_module; + +#[device] +unsafe extern "C" { + fn external_scale_add(value: i32) -> i32; +} + +#[cuda_module] +mod kernels { + use super::*; + + #[kernel] + pub fn scale_add(input: &[i32], mut output: DisjointSlice) { + let index = thread::index_1d(); + let offset = index.get(); + if let Some(value) = output.get_mut(index) { + // SAFETY: the C++ definition takes and returns an int32 scalar; + // it accesses no pointers and has no participation requirement. + *value = unsafe { external_scale_add(input[offset]) }; + } + } +} + +fn verify(expected_addend: i32) -> Result<(), Box> { + let context = CudaContext::new(0)?; + let stream = context.default_stream(); + let input: Vec = (-128..129).collect(); + let device_input = DeviceBuffer::from_host(&stream, &input)?; + let mut device_output = DeviceBuffer::::zeroed(&stream, input.len())?; + let module = kernels::load(&context)?; + // SAFETY: the equal-length buffers cover every access, the bounds check + // handles the partial last block, and the kernel needs no shared memory. + unsafe { + module.scale_add( + &stream, + LaunchConfig::for_num_elems(input.len() as u32), + &device_input, + &mut device_output, + )?; + } + let actual = device_output.to_host_vec(&stream)?; + let expected: Vec = input + .iter() + .map(|value| value * 2 + expected_addend) + .collect(); + assert_eq!(actual, expected); + println!("PASS external LTO-IR: 257 values, addend={expected_addend}"); + Ok(()) +} + +fn main() -> Result<(), Box> { + let expected_addend = std::env::args() + .nth(1) + .map(|value| value.parse()) + .transpose()? + .unwrap_or(3); + verify(expected_addend) +} + +#[test] +fn linked_device_libraries_execute() { + verify(3).unwrap(); +} diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/f16_stress/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/f16_stress/Cargo.lock index 0ccebcd50f..52750f2137 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/f16_stress/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/f16_stress/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/f16x2_arith/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/f16x2_arith/Cargo.lock index 5a89dfb53e..9a211989d2 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/f16x2_arith/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/f16x2_arith/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/f32x2_arith/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/f32x2_arith/Cargo.lock index 1fef632193..316a55bb37 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/f32x2_arith/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/f32x2_arith/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/fcmp_ne_nan_smoke/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/fcmp_ne_nan_smoke/Cargo.lock index 5495cc6174..d3ab30b326 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/fcmp_ne_nan_smoke/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/fcmp_ne_nan_smoke/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/field_array_assign/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/field_array_assign/Cargo.lock index df607fcf01..e7fd2f8011 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/field_array_assign/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/field_array_assign/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/float_rounding/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/float_rounding/Cargo.lock index 0b6f57b623..fe69542aed 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/float_rounding/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/float_rounding/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/fmaxmin_smoke/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/fmaxmin_smoke/Cargo.lock index 4458b7ec54..c365d9ba1f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/fmaxmin_smoke/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/fmaxmin_smoke/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/fn_ptr_naming/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/fn_ptr_naming/Cargo.lock index 92d4321798..b582ea8179 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/fn_ptr_naming/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/fn_ptr_naming/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/fp_special_literal_smoke/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/fp_special_literal_smoke/Cargo.lock index 7507547040..43304c2bf6 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/fp_special_literal_smoke/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/fp_special_literal_smoke/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/function_item_call/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/function_item_call/Cargo.lock index d21991bc5c..df36e9b073 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/function_item_call/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/function_item_call/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/future_apis/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/future_apis/Cargo.lock index 767da36ce7..a3d096539c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/future_apis/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/future_apis/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -420,8 +422,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm/Cargo.lock index a16799a41f..9202cb8fa0 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_sol/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_sol/Cargo.lock index e634080e85..6f1cd70de5 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_sol/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_sol/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -420,8 +422,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_sol_final/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_sol_final/Cargo.lock index 5704e7e7e2..7b15463812 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_sol_final/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_sol_final/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -421,8 +423,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_views/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_views/Cargo.lock index d0a28e8c3a..827bd4141f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_views/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/gemm_views/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -420,8 +422,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_intrinsics/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_intrinsics/Cargo.lock index ab18e3fc3d..282ce54481 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_intrinsics/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_intrinsics/Cargo.lock @@ -193,6 +193,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -424,8 +426,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_intrinsics_blackwell/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_intrinsics_blackwell/Cargo.lock index adc7c6e4f4..edc7e4cfdb 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_intrinsics_blackwell/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_intrinsics_blackwell/Cargo.lock @@ -193,6 +193,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -424,8 +426,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_ldmatrix/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_ldmatrix/Cargo.lock index 2bd5d7e1ee..1cdc62eeb9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_ldmatrix/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/generated_ldmatrix/Cargo.lock @@ -193,6 +193,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -424,8 +426,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/generic/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/generic/Cargo.lock index ec5339be09..18695069fd 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/generic/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/generic/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap/Cargo.lock index 751a0a2687..d96c56fa6b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v2/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v2/Cargo.lock index 2646cae85d..ebbe469e48 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v2/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v2/Cargo.lock @@ -220,6 +220,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -454,8 +456,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v3/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v3/Cargo.lock index b834c90e43..91e60e181a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v3/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/hashmap_v3/Cargo.lock @@ -220,6 +220,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -454,8 +456,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/hello_constant/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/hello_constant/Cargo.lock index 1f7d33ddd1..b1b1ac3e53 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/hello_constant/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/hello_constant/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/helper_fn/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/helper_fn/Cargo.lock index 9bd0891bc8..51f934948a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/helper_fn/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/helper_fn/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/host_closure/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/host_closure/Cargo.lock index dc0217c032..5b880cebf5 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/host_closure/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/host_closure/Cargo.lock @@ -211,6 +211,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -559,8 +561,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/host_closure_target_feature/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/host_closure_target_feature/Cargo.lock index cb58b6ab38..a38841fc30 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/host_closure_target_feature/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/host_closure_target_feature/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/iket_trace/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/iket_trace/Cargo.lock index 879811c20f..38ea4c1863 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/iket_trace/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/iket_trace/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/image_convolution/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/image_convolution/Cargo.lock index 00cd57c087..505715ae60 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/image_convolution/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/image_convolution/Cargo.lock @@ -201,6 +201,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -472,8 +474,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/index2d_const/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/index2d_const/Cargo.lock index 87f3c88f76..1ce9174b25 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/index2d_const/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/index2d_const/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/index_field_assign/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/index_field_assign/Cargo.lock index f6b0a95cd1..d88c1a3899 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/index_field_assign/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/index_field_assign/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_intent/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_intent/Cargo.lock index 5a05446079..903a76259a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_intent/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_intent/Cargo.lock @@ -188,6 +188,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -418,8 +420,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_ptx/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_ptx/Cargo.lock index 9800de80bd..5b307a85eb 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_ptx/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/inline_ptx/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/integer_minmax/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/integer_minmax/Cargo.lock index 38a2e99ccf..d3ae3b4e48 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/integer_minmax/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/integer_minmax/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/interior_mutability/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/interior_mutability/Cargo.lock index 87d0c79a58..7bd9235a96 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/interior_mutability/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/interior_mutability/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/interop_cubin_identity/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/interop_cubin_identity/Cargo.lock index 61cff5801b..2d6cd89720 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/interop_cubin_identity/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/interop_cubin_identity/Cargo.lock @@ -346,8 +346,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/interop_cubin_identity/device/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/interop_cubin_identity/device/Cargo.lock index 0671dfd50c..b7b2fe63e1 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/interop_cubin_identity/device/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/interop_cubin_identity/device/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/kernel_reference_validity/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/kernel_reference_validity/Cargo.lock index 15c94ec2a7..0dd18c25e8 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/kernel_reference_validity/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/kernel_reference_validity/Cargo.lock @@ -211,6 +211,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -558,8 +560,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/lanemask_scan/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/lanemask_scan/Cargo.lock index 940f65d37c..f87706e7db 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/lanemask_scan/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/lanemask_scan/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_atomic_fadd/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_atomic_fadd/Cargo.lock index fd96ea2f4b..7d0be82721 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_atomic_fadd/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_atomic_fadd/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_atomic_rmw_cas/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_atomic_rmw_cas/Cargo.lock index 38e6fc926f..30e3bae681 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_atomic_rmw_cas/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_atomic_rmw_cas/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_nvvm_pointer_shapes/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_nvvm_pointer_shapes/Cargo.lock index 6a7f571741..1e13fc5c4e 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_nvvm_pointer_shapes/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/legacy_nvvm_pointer_shapes/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/libdevice_math/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/libdevice_math/Cargo.lock index 4be2167e8e..89d99163ad 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/libdevice_math/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/libdevice_math/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/libm_math/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/libm_math/Cargo.lock index 7838394dfe..42a9ae0801 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/libm_math/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/libm_math/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -426,8 +428,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/manual_launch_generic/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/manual_launch_generic/Cargo.lock index 4e99d341bd..0edc6e587c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/manual_launch_generic/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/manual_launch_generic/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/manual_launch_libdevice/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/manual_launch_libdevice/Cargo.lock index 4a2c19c285..cf6b477d56 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/manual_launch_libdevice/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/manual_launch_libdevice/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/map_sum/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/map_sum/Cargo.lock index 236e1a127f..d3fbea2769 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/map_sum/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/map_sum/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_atan/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_atan/Cargo.lock index 2e0d065824..2ead75795b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_atan/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_atan/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_hyperbolic/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_hyperbolic/Cargo.lock index 51267d64d2..4c6d0d95be 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_hyperbolic/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_hyperbolic/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_no_mir_inline/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_no_mir_inline/Cargo.lock index 5c67e8bc57..8fdf93f4cf 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_no_mir_inline/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_no_mir_inline/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_tan/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_tan/Cargo.lock index c8bddba41d..3c355ac7ee 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/math_tan/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/math_tan/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/mathdx_ffi_test/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/mathdx_ffi_test/Cargo.lock index 3b045b0c04..87ff45758a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/mathdx_ffi_test/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/mathdx_ffi_test/Cargo.lock @@ -209,6 +209,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -456,8 +458,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/mcast_barrier_test/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/mcast_barrier_test/Cargo.lock index 1c558ad854..e42b67543b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/mcast_barrier_test/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/mcast_barrier_test/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/mem_swap/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/mem_swap/Cargo.lock index a126752583..f18cbf447f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/mem_swap/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/mem_swap/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/mir_projected_call_destination/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/mir_projected_call_destination/Cargo.lock index af4602c219..b66ae5731f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/mir_projected_call_destination/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/mir_projected_call_destination/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -426,8 +428,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/mma_mxf8f6f4/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/mma_mxf8f6f4/Cargo.lock index 0070effb36..62ccba04ed 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/mma_mxf8f6f4/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/mma_mxf8f6f4/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/Cargo.lock new file mode 100644 index 0000000000..ce2730ffc8 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/Cargo.lock @@ -0,0 +1,756 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", + "which", +] + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "cuda-artifact-finalizer" +version = "0.2.1" +dependencies = [ + "libnvvm-sys", + "nvjitlink-sys", + "serde", + "sha2", + "thiserror", +] + +[[package]] +name = "cuda-bindings" +version = "0.5.0" +dependencies = [ + "bindgen", + "libloading", + "prettyplease", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cuda-core" +version = "0.5.0" +dependencies = [ + "anyhow", + "cuda-bindings", + "cuda-core-derive", + "half", + "oxide-artifacts", +] + +[[package]] +name = "cuda-core-derive" +version = "0.5.0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cuda-device" +version = "0.2.1" +dependencies = [ + "cuda-macros", +] + +[[package]] +name = "cuda-host" +version = "0.2.1" +dependencies = [ + "cuda-artifact-finalizer", + "cuda-core", + "cuda-device", + "cuda-macros", + "half", + "libc", + "oxide-artifacts", + "ptx-parse", + "sha2", + "thiserror", +] + +[[package]] +name = "cuda-macros" +version = "0.2.1" +dependencies = [ + "oxide-tile-export", + "proc-macro2", + "quote", + "reserved-oxide-symbols", + "serde", + "serde_json", + "syn", +] + +[[package]] +name = "cuda-target-spec" +version = "0.1.0" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libnvvm-sys" +version = "0.2.1" +dependencies = [ + "cuda-target-spec", + "libloading", + "thiserror", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "native_library" +version = "0.1.0" +dependencies = [ + "cuda-core", + "cuda-device", + "cuda-host", + "serde_json", + "sha2", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nvjitlink-sys" +version = "0.2.1" +dependencies = [ + "libloading", + "thiserror", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "crc32fast", + "hashbrown 0.15.5", + "indexmap", + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oxide-artifacts" +version = "0.2.1" +dependencies = [ + "object", +] + +[[package]] +name = "oxide-tile-export" +version = "0.2.1" +dependencies = [ + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "ptx-parse" +version = "0.1.0" + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reserved-oxide-symbols" +version = "0.2.1" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/Cargo.toml b/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/Cargo.toml new file mode 100644 index 0000000000..0c085d9f98 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/Cargo.toml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "native_library" +version = "0.1.0" +edition = "2024" +license = "Apache-2.0" +publish = false + +[workspace] + +[dependencies] +cuda-device = { path = "../../../cuda-device" } +cuda-host = { path = "../../../cuda-host" } +cuda-core = { path = "../../../../../cuda-core" } + +[build-dependencies] +serde_json = "1" +sha2 = "0.10" diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/README.md b/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/README.md new file mode 100644 index 0000000000..6dafcc085e --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/README.md @@ -0,0 +1,36 @@ +# Native library discovery + +This unsafe compiler fixture loads an i32 tile and adds seven through a native +CUDA implementation. Its generic dependency helper is instantiated with one +and four items per thread. The build script registers itself as the provider; +the compiler sends concrete requests and links the returned LTO-IR. No CUB +checkout, hand-written extern declaration or external artifact list is needed. + +From `cuda-oxide/`, with a CUDA toolkit and matching GPU: + +```sh +cargo oxide run native_library --arch sm_120 --materialize-cubin +cargo oxide test native_library --arch sm_120 --materialize-cubin +``` + +The provider invokes the selected toolkit's `nvcc`; `CUDAHOSTCXX` may select +its host compiler. Launch preparation uses inferred shared-memory facts and +rejects an equal-product wrong block shape. Both kernels run three blocks and +check every output. The raw helper requires full input/output tiles and exact +participation; this fixture does not claim a safe collective API. + +The smoke script supplies materialization and the target in each lane: + +```sh +scripts/smoketest.sh --only '^native_library$' +scripts/smoketest.sh --only '^native_library$' --compile-only +scripts/smoketest.sh --only '^native_library$' --full-debug +``` + +The artifact-only lanes run the fixture's `--check-artifact TARGET OUTPUT` +mode without creating a CUDA context. It verifies the embedded CUBIN and +both kernels' launch facts, then extracts the image to `OUTPUT`. Full debug +adds `--full-debug`, requires the recorded full-debug policy, and uses +`readelf` or `llvm-readelf` to check both kernels' DWARF. That lane uses the +modern libNVVM target (the host target on Blackwell or newer, otherwise +`sm_100`) and launches no kernels. diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/build.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/build.rs new file mode 100644 index 0000000000..2c5d5d075b --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/build.rs @@ -0,0 +1,141 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::io::{Read, Write}; + +fn serve() -> Result<(), Box> { + let mut input = String::new(); + std::io::stdin() + .take(1024 * 1024 + 1) + .read_to_string(&mut input)?; + assert!(input.len() <= 1024 * 1024); + let request: Value = serde_json::from_str(&input)?; + assert_eq!(request["version"], 1); + let target = request["target"].as_str().ok_or("missing target")?; + // nvcc emits PTX for -arch=compute_XX even with --ltoir. Select its + // matching real architecture to obtain raw LTO-IR for nvJitLink. + let nvcc_target = format!( + "sm_{}", + target.strip_prefix("compute_").ok_or("invalid target")? + ); + let cuda = request["cuda_root"].as_str().ok_or("missing toolkit")?; + let mut source = String::new(); + let mut entries = Vec::new(); + let scalar = json!({"kind":"signed", "bits":32}); + for (index, item) in request["requests"] + .as_array() + .ok_or("missing requests")? + .iter() + .enumerate() + { + assert_eq!(item["operation"], "load_add"); + assert_eq!(item["algorithm"], "direct"); + assert_eq!(item["dtype"], scalar); + let block = item["block"].as_array().ok_or("missing block")?; + assert_eq!(&block[1..], &[json!(1), json!(1)]); + let threads = block[0].as_u64().ok_or("invalid block")?; + let items = item["items_per_thread"].as_u64().ok_or("invalid extent")?; + assert!((1..=1024).contains(&threads) && (1..=16).contains(&items)); + let symbol = format!("fixture_load_b{threads}_i{items}"); + source.push_str(&format!( + "extern \"C\" __device__ void {symbol}(const int* memory, int* values, unsigned char*, int add) {{\n#pragma unroll\nfor (int i=0; i<{items}; ++i) values[i]=memory[threadIdx.x*{items}+i]+add;\n}}\n" + )); + entries.push(json!({ + "request_index": index, + "identity": Sha256::digest(symbol.as_bytes()).to_vec(), + "symbol": symbol, + "arguments": [ + {"kind":"const_pointer", "scalar":scalar}, + {"kind":"mut_pointer", "scalar":scalar}, + {"kind":"value", "scalar":scalar} + ], + "cub_layout": {"size":0,"alignment":1}, + "scratch": {"size":0,"alignment":1}, + "scratch_free": true, "mutates_payload": true + })); + } + // This fixture compiles a tiny native implementation without a CUB dependency. + // Each invocation owns its temporary files, including simultaneous consumers. + let directory = + std::env::temp_dir().join(format!("oxide-library-fixture-{}", std::process::id())); + std::fs::create_dir(&directory)?; + let result = (|| -> Result, Box> { + let cpp = directory.join("provider.cu"); + let lto = directory.join("provider.ltoir"); + std::fs::write(&cpp, source)?; + let mut command = std::process::Command::new(std::path::Path::new(cuda).join("bin/nvcc")); + command.args(["--ltoir", "-dlto", "-arch", &nvcc_target]); + if let Some(host) = std::env::var_os("CUDAHOSTCXX") { + command.arg("-ccbin").arg(host); + } + let status = command.arg(&cpp).arg("-o").arg(<o).status()?; + if !status.success() { + return Err("fixture native compilation failed".into()); + } + Ok(std::fs::read(lto)?) + })(); + std::fs::remove_dir_all(&directory)?; + let lto = result?; + let header = serde_json::to_vec(&json!({ + "version":1, "batch_identity":Sha256::digest(<o).to_vec(), + "specializations":entries + }))?; + let mut output = std::io::stdout().lock(); + output.write_all(b"OXDLIB01")?; + output.write_all(&(header.len() as u64).to_le_bytes())?; + output.write_all(&(lto.len() as u64).to_le_bytes())?; + output.write_all(&header)?; + output.write_all(<o)?; + Ok(()) +} + +fn serve_inputs() -> Result<(), Box> { + let mut bytes = Vec::new(); + std::io::stdin() + .take(1024 * 1024 + 1) + .read_to_end(&mut bytes)?; + assert!(bytes.len() <= 1024 * 1024); + let request: Value = serde_json::from_slice(&bytes)?; + assert_eq!(request["version"], 1); + let cuda = std::path::Path::new(request["cuda_root"].as_str().ok_or("missing toolkit")?); + let host = std::env::var_os("CUDAHOSTCXX").unwrap_or_else(|| "g++".into()); + let host = std::path::PathBuf::from(host); + let host = if host.is_absolute() { + host + } else { + std::env::split_paths(&std::env::var_os("PATH").unwrap_or_default()) + .map(|directory| directory.join(&host)) + .find(|path| path.is_file()) + .ok_or("cannot locate native host compiler")? + }; + // The example tracks CUDA tools/headers and the host compiler driver. + // The host compiler's installed system headers/libraries are assumed stable. + serde_json::to_writer( + std::io::stdout().lock(), + &json!({ + "version":1, + "paths":[cuda.join("bin"), cuda.join("nvvm"), cuda.join("include"), host], + "environment":["CUDAHOSTCXX", "PATH", "NVCC_PREPEND_FLAGS", "NVCC_APPEND_FLAGS", + "CPATH", "CPLUS_INCLUDE_PATH", "GCC_EXEC_PREFIX", "COMPILER_PATH"] + }), + )?; + Ok(()) +} + +fn main() -> Result<(), Box> { + match std::env::args().nth(1).as_deref() { + Some("--oxide-library-request-v1") => return serve(), + Some("--oxide-library-inputs-v1") => return serve_inputs(), + _ => {} + } + println!("cargo::rerun-if-changed=build.rs"); + let registration = json!({ + "version":1, "provider":"native_library_fixture", + "executable":std::env::current_exe()?, + "operation":"load_add", "algorithm":"direct" + }); + println!("cargo::rustc-env=FIXTURE_REGISTRATION={registration}"); + Ok(()) +} diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/src/lib.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/src/lib.rs new file mode 100644 index 0000000000..781dc7fa65 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/src/lib.rs @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![no_std] + +trait Recipe { + const REGISTRATION: &'static str; +} + +struct LoadAddRecipe; + +impl Recipe for LoadAddRecipe { + const REGISTRATION: &'static str = cuda_device::library::registration!("FIXTURE_REGISTRATION"); +} + +fn registration, const ITEMS: usize>() -> &'static str { + R::REGISTRATION +} + +/// Load a complete block tile through a native provider in a dependency crate. +/// +/// # Safety +/// +/// Launch 128-thread one-dimensional blocks. Input/output must cover every +/// block's complete tile; input stays immutable and output tiles are disjoint. +#[inline(never)] +pub unsafe fn load_add(input: *const i32, output: *mut i32) { + let lane = cuda_device::thread::threadIdx_x() as usize; + let origin = cuda_device::thread::blockIdx_x() as usize * 128 * ITEMS; + let mut values = [0_i32; ITEMS]; + // SAFETY: each thread owns its initialized local array, and the caller + // supplies the full input tile and required geometry. This recipe uses no scratch. + unsafe { + cuda_device::library::call_v1::( + registration::, + (input.add(origin), values.as_mut_ptr(), 7_i32), + ); + for (item, value) in values.iter().enumerate() { + output.add(origin + lane * ITEMS + item).write(*value); + } + } +} diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/src/main.rs b/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/src/main.rs new file mode 100644 index 0000000000..c7302dba27 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/native_library/src/main.rs @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use cuda_core::embedded::{ + ArtifactDebugPolicy, ArtifactPayloadKind, artifact_bundles_from_current_exe, +}; +use cuda_core::{CudaContext, DeviceBuffer, LaunchConfig3D}; +use cuda_device::{kernel, launch_bounds, launch_contract}; +use cuda_host::cuda_module; + +#[cuda_module] +mod kernels { + use super::*; + + #[kernel] + #[launch_bounds(256)] + #[launch_contract(domain = 3, dynamic_shared = inferred)] + pub unsafe fn one(input: *const i32, output: *mut i32) { + unsafe { + native_library::load_add::<1>(input, output); + } + } + + #[kernel] + #[launch_bounds(256)] + #[launch_contract(domain = 3, dynamic_shared = inferred)] + pub unsafe fn four(input: *const i32, output: *mut i32) { + unsafe { + native_library::load_add::<4>(input, output); + } + } +} + +fn verify() -> Result<(), Box> { + let context = CudaContext::new(0)?; + let stream = context.default_stream(); + // SAFETY: the embedded image was produced from this module's declarations. + let module = unsafe { kernels::load(&context)? }; + for items in [1, 4] { + let input: Vec = (0..3 * 128 * items).map(|i| i * 3 - 11).collect(); + let device_input = DeviceBuffer::from_host(&stream, &input)?; + let device_output = DeviceBuffer::::zeroed(&stream, input.len())?; + let config = LaunchConfig3D::auto_shared((3, 1, 1), (128, 1, 1)); + macro_rules! launch { + ($prepare:ident, $kernel:ident) => {{ + let prepared = module.$prepare(config)?; + let facts = prepared.function().launch_facts().unwrap(); + assert_eq!(facts.exact_block, Some((128, 1, 1))); + assert_eq!(facts.dynamic_shared_bytes, 0); + assert!( + module + .$prepare(LaunchConfig3D::auto_shared((3, 1, 1), (64, 2, 1))) + .is_err() + ); + // SAFETY: complete, immutable input and disjoint output tiles + // outlive execution. Preparation enforces exact geometry. + unsafe { + module.$kernel( + &stream, + &prepared, + device_input.cu_deviceptr() as *const i32, + device_output.cu_deviceptr() as *mut i32, + )?; + } + }}; + } + if items == 1 { + launch!(prepare_one, one); + } else { + launch!(prepare_four, four); + } + assert_eq!( + device_output.to_host_vec(&stream)?, + input.iter().map(|value| value + 7).collect::>() + ); + println!("PASS native library: items={items}, {} values", input.len()); + } + Ok(()) +} + +// Artifact-only smoke lanes call this before creating any CUDA context. Read +// the executable's actual bundle, including its launch facts and debug policy. +fn check_artifact( + target: &str, + output: &std::path::Path, + full_debug: bool, +) -> Result<(), Box> { + let bundles = artifact_bundles_from_current_exe()?; + let [bundle] = bundles.as_slice() else { + return Err("expected exactly one embedded device bundle".into()); + }; + assert_eq!(bundle.target, target); + for symbol in ["one", "four"] { + let facts = bundle.entry(symbol).unwrap().launch_facts.unwrap(); + assert_eq!(facts.exact_block, Some((128, 1, 1))); + assert_eq!(facts.dynamic_shared_bytes, 0); + } + if full_debug { + assert_eq!( + bundle.compile_options.debug_policy(), + ArtifactDebugPolicy::Full + ); + } + let cubin = bundle + .payload(ArtifactPayloadKind::Cubin) + .ok_or("missing embedded CUBIN")?; + assert!(cubin.starts_with(b"\x7fELF")); + std::fs::write(output, cubin)?; + println!("PASS embedded CUBIN: target={target}, kernels=one,four"); + Ok(()) +} + +fn main() -> Result<(), Box> { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + if args.first().is_some_and(|arg| arg == "--check-artifact") { + if !(args.len() == 3 || (args.len() == 4 && args[3] == "--full-debug")) { + return Err("expected --check-artifact TARGET OUTPUT [--full-debug]".into()); + } + return check_artifact( + args[1].to_str().ok_or("invalid target")?, + std::path::Path::new(&args[2]), + args.len() == 4, + ); + } + if !args.is_empty() { + return Err("unexpected fixture argument".into()); + } + verify() +} + +#[test] +fn native_provider_executes() { + verify().unwrap(); +} diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/ne_bytes_transmute/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/ne_bytes_transmute/Cargo.lock index 8d1dc68f4f..b59a96ef27 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/ne_bytes_transmute/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/ne_bytes_transmute/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/nested_index_assignment/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/nested_index_assignment/Cargo.lock index d82c25246e..0232109d4d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/nested_index_assignment/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/nested_index_assignment/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/numeric_stress/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/numeric_stress/Cargo.lock index 68d17a62ae..9df4737241 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/numeric_stress/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/numeric_stress/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/nvls_all_reduce/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/nvls_all_reduce/Cargo.lock index 58ef040c46..4bb000aca7 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/nvls_all_reduce/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/nvls_all_reduce/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/option_ref_transmute/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/option_ref_transmute/Cargo.lock index d6ccc1e244..6300b46811 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/option_ref_transmute/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/option_ref_transmute/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ dependencies = [ [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/option_ref_unwrap_or/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/option_ref_unwrap_or/Cargo.lock index 91de9b410d..8aba53327c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/option_ref_unwrap_or/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/option_ref_unwrap_or/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ dependencies = [ [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/option_unwrap_trap/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/option_unwrap_trap/Cargo.lock index d164620488..a145596b88 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/option_unwrap_trap/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/option_unwrap_trap/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ dependencies = [ [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/ord_cmp/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/ord_cmp/Cargo.lock index 44d05c995d..4dbe7e5b9a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/ord_cmp/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/ord_cmp/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -419,8 +421,6 @@ dependencies = [ [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/packed_aggregate_abi/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/packed_aggregate_abi/Cargo.lock index 972c6cd402..5ed992549c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/packed_aggregate_abi/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/packed_aggregate_abi/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/packed_atomic_add/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/packed_atomic_add/Cargo.lock index af832aba00..5e09d86471 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/packed_atomic_add/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/packed_atomic_add/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/padded_constants/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/padded_constants/Cargo.lock index 41a23e648d..1544f3a081 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/padded_constants/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/padded_constants/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/partial_warp_reduce/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/partial_warp_reduce/Cargo.lock index 7e0a385ee1..a5b5c6744e 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/partial_warp_reduce/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/partial_warp_reduce/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/pinned_overlap/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/pinned_overlap/Cargo.lock index 41c7b64525..bf0f428e20 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/pinned_overlap/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/pinned_overlap/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/place_read_fallbacks/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/place_read_fallbacks/Cargo.lock index fdd244e8ef..f0dcce1daa 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/place_read_fallbacks/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/place_read_fallbacks/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/policy_config/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/policy_config/Cargo.lock index 9cf6c64802..f3dde14485 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/policy_config/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/policy_config/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/primitive_stress/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/primitive_stress/Cargo.lock index 84929b541d..704e6e48c6 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/primitive_stress/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/primitive_stress/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/printf/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/printf/Cargo.lock index 9db04a93c7..f2892e1d78 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/printf/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/printf/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/projected_place_read/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/projected_place_read/Cargo.lock index 17642ea7b6..185d143ee6 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/projected_place_read/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/projected_place_read/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/proof_carrying_views/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/proof_carrying_views/Cargo.lock index 46bf7cfb6b..ef215039ff 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/proof_carrying_views/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/proof_carrying_views/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_copy/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_copy/Cargo.lock index 5b05e017c5..61edbb886d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_copy/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_copy/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_offset_from/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_offset_from/Cargo.lock index 3fa6d7132f..67d53497ee 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_offset_from/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_offset_from/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_without_provenance/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_without_provenance/Cargo.lock index 04e6a57cd5..0711022587 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_without_provenance/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/ptr_without_provenance/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/reborrow/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/reborrow/Cargo.lock index 73188db112..e961c3139f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/reborrow/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/reborrow/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_f32/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_f32/Cargo.lock index d7a8c23a4e..80b585365d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_f32/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_f32/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_minmax/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_minmax/Cargo.lock index 7b05be8590..d9422958e3 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_minmax/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_minmax/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_sum/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_sum/Cargo.lock index 3cc9bb1335..10ce5670dc 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_sum/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/redux_sum/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/ref_index_projections/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/ref_index_projections/Cargo.lock index 51d1860828..25a8ebc750 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/ref_index_projections/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/ref_index_projections/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/ref_operand_mul/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/ref_operand_mul/Cargo.lock index c4a0a8bf84..ef404cdb50 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/ref_operand_mul/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/ref_operand_mul/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_align_block_args/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_align_block_args/Cargo.lock index ad83d723f6..c9d929e986 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_align_block_args/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_align_block_args/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_transparent_abi/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_transparent_abi/Cargo.lock index 07ce1da2d0..cf1e50bad1 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_transparent_abi/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_transparent_abi/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_u32_enum_stride/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_u32_enum_stride/Cargo.lock index 79224818e2..09fe88dad6 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_u32_enum_stride/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/repr_u32_enum_stride/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/row_width_slice/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/row_width_slice/Cargo.lock index 1ed21433b6..68fc88597f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/row_width_slice/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/row_width_slice/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/rustlantis-smoke/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/rustlantis-smoke/Cargo.lock index 322099605b..8c4e70dfe3 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/rustlantis-smoke/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/rustlantis-smoke/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -414,8 +416,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/scoped_atomic_load_store/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/scoped_atomic_load_store/Cargo.lock index 683cadeb40..289425b07c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/scoped_atomic_load_store/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/scoped_atomic_load_store/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/select_unpredictable/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/select_unpredictable/Cargo.lock index 87763b5b63..09d09926ae 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/select_unpredictable/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/select_unpredictable/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/set_discriminant/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/set_discriminant/Cargo.lock index 72726e3e91..25ac8303dc 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/set_discriminant/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/set_discriminant/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/set_discriminant_niche/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/set_discriminant_niche/Cargo.lock index 5fbc5179e4..2b797afd8b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/set_discriminant_niche/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/set_discriminant_niche/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_debug/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_debug/Cargo.lock index c1225ffb45..907dde20ae 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_debug/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_debug/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_memory_helper/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_memory_helper/Cargo.lock index 989f46f684..8af4a40440 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_memory_helper/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_memory_helper/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_ptr_api/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_ptr_api/Cargo.lock index 35539d8e2b..5a54d8be1d 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_ptr_api/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_ptr_api/Cargo.lock @@ -188,6 +188,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -409,8 +411,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_ptr_transmute/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_ptr_transmute/Cargo.lock index 098e6fa943..00e90aee48 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_ptr_transmute/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_ptr_transmute/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_slice_unsize/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_slice_unsize/Cargo.lock index fd1d381de4..2daacc2c46 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_slice_unsize/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_slice_unsize/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_stage_select/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_stage_select/Cargo.lock index 5ac0486dc4..3fffbaaa53 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_stage_select/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/shared_stage_select/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/sharedmem/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/sharedmem/Cargo.lock index 2bee1e9097..013dd15032 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/sharedmem/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/sharedmem/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/shuffle_64/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/shuffle_64/Cargo.lock index 9c58993282..5d79c66b44 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/shuffle_64/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/shuffle_64/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_from_raw_parts_cast/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_from_raw_parts_cast/Cargo.lock index dcfee5f6a1..885a44d6b1 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_from_raw_parts_cast/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_from_raw_parts_cast/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_get_mut/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_get_mut/Cargo.lock index c94c1a3938..e7ca05fc38 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_get_mut/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_get_mut/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_index_bounds_check/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_index_bounds_check/Cargo.lock index a4045435e3..eb465ca60c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_index_bounds_check/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_index_bounds_check/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_iterators/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_iterators/Cargo.lock index 6483648ff7..c92e737f43 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_iterators/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_iterators/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_reslice/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_reslice/Cargo.lock index 44e93548de..829fd73c5f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_reslice/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/slice_reslice/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/small_type_ffi_test/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/small_type_ffi_test/Cargo.lock index 20196ba508..b3eb37a0cb 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/small_type_ffi_test/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/small_type_ffi_test/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/sparse_mma_fp8/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/sparse_mma_fp8/Cargo.lock index 8b7f279ddd..c5e7a160d3 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/sparse_mma_fp8/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/sparse_mma_fp8/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/sparse_mma_ordered_float/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/sparse_mma_ordered_float/Cargo.lock index 218f65f807..4e4ec9634c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/sparse_mma_ordered_float/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/sparse_mma_ordered_float/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/split_at_mut/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/split_at_mut/Cargo.lock index 4fa006d2b4..ce440f9c15 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/split_at_mut/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/split_at_mut/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/standalone_device_fn/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/standalone_device_fn/Cargo.lock index 4870b6e0d2..c3f5164164 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/standalone_device_fn/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/standalone_device_fn/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/static_slice_addend/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/static_slice_addend/Cargo.lock index 260515c3a0..e95f858218 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/static_slice_addend/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/static_slice_addend/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/static_slice_unsize/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/static_slice_unsize/Cargo.lock index d24fdb64fe..9c097cf19a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/static_slice_unsize/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/static_slice_unsize/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/step_by/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/step_by/Cargo.lock index 1b1df6b95f..05a567fbc9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/step_by/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/step_by/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/struct_constant_provenance/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/struct_constant_provenance/Cargo.lock index 481d2d5496..cc2442a84a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/struct_constant_provenance/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/struct_constant_provenance/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/struct_field_layout/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/struct_field_layout/Cargo.lock index 6e0c3e8691..8f0b4d2e3b 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/struct_field_layout/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/struct_field_layout/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/subslice_projection/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/subslice_projection/Cargo.lock index e32ae7b8bf..e8feb9de44 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/subslice_projection/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/subslice_projection/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/switchint_128bit/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/switchint_128bit/Cargo.lock index 1702573ea1..9a2370e300 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/switchint_128bit/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/switchint_128bit/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/swizzle_smem/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/swizzle_smem/Cargo.lock index 95370fb238..e9f58ecd23 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/swizzle_smem/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/swizzle_smem/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tcgen05/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/tcgen05/Cargo.lock index 83d5191784..539b769f66 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tcgen05/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tcgen05/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tcgen05_matmul/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/tcgen05_matmul/Cargo.lock index b25fb143ad..a99548a870 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tcgen05_matmul/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tcgen05_matmul/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/thread_runs/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/thread_runs/Cargo.lock index 608a68f483..3938401f83 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/thread_runs/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/thread_runs/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tiled_gemm/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/tiled_gemm/Cargo.lock index 1686557131..86f799e7ff 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tiled_gemm/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tiled_gemm/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tilelib_relu/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/tilelib_relu/Cargo.lock index 7696b70a7f..120b22c268 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tilelib_relu/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tilelib_relu/Cargo.lock @@ -173,6 +173,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -394,8 +396,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tma_copy/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/tma_copy/Cargo.lock index 0162d17bb3..c60e5878f6 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tma_copy/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tma_copy/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tma_multicast/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/tma_multicast/Cargo.lock index 41c5da3d69..0fc9b3c124 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tma_multicast/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tma_multicast/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/toolchain_pairing_smoke/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/toolchain_pairing_smoke/Cargo.lock index fa8576a5c6..a884c86124 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/toolchain_pairing_smoke/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/toolchain_pairing_smoke/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/transmute_scalar_struct/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/transmute_scalar_struct/Cargo.lock index b153af1014..a6f4838ad0 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/transmute_scalar_struct/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/transmute_scalar_struct/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_array_provenance/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_array_provenance/Cargo.lock index ad1c253f8c..3c3b725a9f 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_array_provenance/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_array_provenance/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_constant_provenance/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_constant_provenance/Cargo.lock index 161953c941..2cb93a51c8 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_constant_provenance/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_constant_provenance/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_field_lookup/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_field_lookup/Cargo.lock index 7bd6be0875..065332ffb6 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_field_lookup/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_field_lookup/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_return/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_return/Cargo.lock index fa83d0f633..f8d987e5d3 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_return/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/tuple_return/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/unaligned_memory/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/unaligned_memory/Cargo.lock index 1ffae9df91..1a8cd3a9f0 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/unaligned_memory/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/unaligned_memory/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/unchecked_indexing/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/unchecked_indexing/Cargo.lock index e7e441bcc4..91f1cd812c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/unchecked_indexing/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/unchecked_indexing/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/uninit_const/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/uninit_const/Cargo.lock index be29bf5921..94ad1a674e 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/uninit_const/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/uninit_const/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/union_aggregate/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/union_aggregate/Cargo.lock index 12fc11b53d..86b4e8f8b9 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/union_aggregate/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/union_aggregate/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/unroll_bounds_check/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/unroll_bounds_check/Cargo.lock index 9583827edc..e25d3c8100 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/unroll_bounds_check/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/unroll_bounds_check/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/unroll_smoke/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/unroll_smoke/Cargo.lock index a234aff07b..f57b9af485 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/unroll_smoke/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/unroll_smoke/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/vecadd/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/vecadd/Cargo.lock index 2e6d5cb154..c5bbde3ed3 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/vecadd/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/vecadd/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/vectorization/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/vectorization/Cargo.lock index ecc9160ea6..ca881cf6cd 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/vectorization/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/vectorization/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/volatile_memory/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/volatile_memory/Cargo.lock index 8d9e203b9d..cf7d0bf28a 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/volatile_memory/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/volatile_memory/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/warp_reduce/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/warp_reduce/Cargo.lock index 2dc4af0e91..740456d5dd 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/warp_reduce/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/warp_reduce/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/warp_sums/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/warp_sums/Cargo.lock index 45f571902d..22bcd8eadb 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/warp_sums/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/warp_sums/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/wgmma/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/wgmma/Cargo.lock index f0236ae67e..c3d17fd85e 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/wgmma/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/wgmma/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/examples/wgmma_mma_bf16/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/examples/wgmma_mma_bf16/Cargo.lock index e54e6354c7..4e4d008677 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/examples/wgmma_mma_bf16/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/examples/wgmma_mma_bf16/Cargo.lock @@ -189,6 +189,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -410,8 +412,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs b/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs index 078099ebaf..5619bb277c 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/src/collector.rs @@ -149,7 +149,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; /// host compilation session. The MIR importer deliberately emits every such /// query as `false` on device. Override only that operand shape here so call /// collection and MIR import cannot choose different switch arms. -fn device_mono_reachable_as_bitset<'tcx>( +pub(crate) fn device_mono_reachable_as_bitset<'tcx>( body: &rustc_middle::mir::Body<'tcx>, tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, @@ -493,27 +493,20 @@ pub struct DeviceExternDecl { /// NVVM attributes for device extern declarations. /// -/// NOTE: These attributes are currently **not emitted** to the LLVM IR output. -/// When linking LTOIR via nvJitLink, the external library's LTOIR already contains -/// proper attributes (convergent, nounwind, memory, etc.) on the function DEFINITIONS. -/// nvJitLink uses the definition's attributes during LTO, making attributes on our -/// declarations redundant. -/// -/// The `#[convergent]`, `#[pure]`, and `#[readonly]` Rust attributes are still parsed -/// but their values are not used in code generation. This struct is retained for -/// potential future use or debugging. +/// Convergence is emitted on the declaration before libNVVM optimization. +/// Memory-effect annotations remain inactive; they are not inferred from external +/// definitions that become available only during LTO. #[derive(Debug, Clone, Default)] pub struct DeviceExternAttrs { - /// Function is convergent (all threads must execute together). - /// NOTE: Not currently emitted - external LTOIR has proper convergent attrs. + /// Function has convergence requirements for its participating threads. pub is_convergent: bool, /// Function is pure (no side effects, result depends only on inputs). - /// NOTE: Not currently emitted - external LTOIR has proper memory attrs. + /// Inactive for Rust source annotations. pub is_pure: bool, /// Function is read-only (only reads memory, doesn't write). - /// NOTE: Not currently emitted - external LTOIR has proper memory attrs. + /// Inactive for Rust source annotations. pub is_readonly: bool, } @@ -523,6 +516,10 @@ pub struct DeviceExternAttrs { /// declarations (for FFI with external LTOIR). #[derive(Debug)] pub struct CollectionResult<'tcx> { + /// Calls to the exact compiler library marker, before body deduplication. + pub library_calls: Vec>, + /// Every resolved edge, retaining all kernel contexts of shared helpers. + pub call_edges: Vec>, /// Functions to compile (kernels and device helpers with MIR bodies). pub functions: Vec>, @@ -533,8 +530,8 @@ pub struct CollectionResult<'tcx> { /// /// This is true for a `#[cuda_module]` containing a generic kernel and /// for a concrete specialization of a generic kernel imported from a - /// dependency. Cubin materialization must reject this case until it can - /// reproduce the same cross-crate linking semantics. + /// dependency. Materialized loading instead requires one complete native + /// artifact; this existing marker remains the generic-module classification. pub requires_ptx_bundle_merge: bool, /// `#[tile_export]` descriptions read from this crate's metadata @@ -544,7 +541,50 @@ pub struct CollectionResult<'tcx> { pub tile_exports: Vec, } -/// Counts kernel functions across all codegen units. +/// Recovers a kernel root from a direct mono item or its host naming helper. +/// +/// The noinline type-hash helper retains the concrete `FnDef` without taking a +/// function pointer. This lets us compile its body for the device without also +/// requiring host codegen of device-only calls. Other type hashes are unrelated +/// to device collection; inspect only this helper's immediate type argument. +fn kernel_root_instance<'tcx>( + tcx: TyCtxt<'tcx>, + instance: Instance<'tcx>, +) -> Option> { + let def_id = instance.def_id(); + if is_kernel_function(tcx, def_id) { + return Some(instance); + } + // Use definition ownership, not a diagnostic path: rustc may print the + // public reexport as `cuda_host::type_id_u128_of_val`. + if tcx.crate_name(def_id.krate).as_str() != "cuda_host" + || tcx.opt_item_name(def_id)?.as_str() != "type_id_u128_of_val" + { + return None; + } + let module = tcx.opt_parent(def_id)?; + if tcx.opt_item_name(module)?.as_str() != "type_id" || !tcx.opt_parent(module)?.is_crate_root() + { + return None; + } + let [arg] = instance.args.as_slice() else { + return None; + }; + let TyKind::FnDef(def_id, args) = *arg.as_type()?.kind() else { + return None; + }; + if !is_kernel_function(tcx, def_id) { + return None; + } + let args = args + .no_bound_vars() + .expect("FnDef args in a monomorphized naming helper carry no late-bound vars"); + Instance::try_resolve(tcx, TypingEnv::fully_monomorphized(), def_id, args) + .ok() + .flatten() +} + +/// Counts kernel roots across all codegen units, including host naming helpers. /// /// This is a quick scan to determine if device compilation is needed. /// Returns 0 if no kernels are found, allowing the backend to skip device codegen entirely. @@ -556,8 +596,8 @@ pub fn count_kernels_in_cgus<'tcx>(tcx: TyCtxt<'tcx>, cgus: &[CodegenUnit<'tcx>] for cgu in cgus { for (item, _data) in cgu.items() { if let MonoItem::Fn(instance) = item - && is_kernel_function(tcx, instance.def_id()) - && is_fully_monomorphized(tcx, *instance) + && let Some(instance) = kernel_root_instance(tcx, *instance) + && is_fully_monomorphized(tcx, instance) { count += 1; } @@ -602,7 +642,9 @@ pub fn unsupported_codegen_protocol_root_in_cgus<'tcx>( let MonoItem::Fn(instance) = item else { return None; }; - let def_id = instance.def_id(); + let def_id = kernel_root_instance(tcx, *instance) + .unwrap_or(*instance) + .def_id(); let item_name = tcx.opt_item_name(def_id)?; unsupported_codegen_protocol_root(item_name.as_str()).then(|| tcx.def_path_str(def_id)) }) @@ -954,7 +996,10 @@ fn rust_alloc_shim_name(fn_path: &str) -> Option<&str> { /// fat-pointer string constants are inspected; everything else returns /// `None`. Evaluation failures (e.g. a constant in still-generic MIR) /// also return `None`, which simply means "no message found". -fn const_str_text<'tcx>(tcx: TyCtxt<'tcx>, constant: &ConstOperand<'tcx>) -> Option { +pub(crate) fn const_str_text<'tcx>( + tcx: TyCtxt<'tcx>, + constant: &ConstOperand<'tcx>, +) -> Option { let ty = constant.const_.ty(); let TyKind::Ref(_, pointee, _) = ty.kind() else { return None; @@ -1241,13 +1286,13 @@ pub fn collect_device_functions<'tcx>( for (item, _data) in cgu.items() { if let MonoItem::Fn(instance) = item && !is_tile_export_meta_symbol(&tcx.def_path_str(instance.def_id())) - && is_kernel_function(tcx, instance.def_id()) + && let Some(instance) = kernel_root_instance(tcx, *instance) { // Skip generic (non-monomorphized) instances. // For generic kernels like scale, the CGU contains both: // - The generic definition (scale) - skip this // - Concrete instantiations (scale::) - process this - if !is_fully_monomorphized(tcx, *instance) { + if !is_fully_monomorphized(tcx, instance) { if verbose { let name = tcx.def_path_str(instance.def_id()); eprintln!( @@ -1260,9 +1305,8 @@ pub fn collect_device_functions<'tcx>( // A downstream monomorphization of a generic kernel may be // present even when the defining crate's private module - // marker is not. Such modules use the all-PTX-bundles loader, - // so strict ahead-of-time cubin materialization cannot safely - // compile only this crate's artifact. + // marker is not. Keep its generic-module classification even + // when only the consuming crate has concrete kernel entries. requires_ptx_bundle_merge |= tcx .generics_of(instance.def_id()) .requires_monomorphization(tcx); @@ -1283,13 +1327,13 @@ pub fn collect_device_functions<'tcx>( // concrete kernel function-item type. The host-side // `ptx_name()` emitted by `#[kernel]` / `#[cuda_module]` // computes the same string from the same `FnDef` type. - let export_name = compute_kernel_export_name(tcx, *instance, &base_name); + let export_name = compute_kernel_export_name(tcx, instance, &base_name); if verbose { eprintln!("[collector] Found kernel: {} -> {}", name, export_name); } - collector.add_root(*instance, true, export_name); + collector.add_root(instance, true, export_name); } } } @@ -1420,6 +1464,9 @@ struct DeviceCollector<'tcx> { /// once per program — the message is procedural advice about the /// kernel-launch contract, not anything per-call. warned_dynamic_shared_array: bool, + library_defs: HashMap, + library_calls: Vec>, + call_edges: Vec>, /// Print progress to stderr. verbose: bool, } @@ -1436,6 +1483,9 @@ impl<'tcx> DeviceCollector<'tcx> { device_externs: Vec::new(), seen_device_externs: HashSet::new(), warned_dynamic_shared_array: false, + library_defs: crate::library::marker_definitions(tcx), + library_calls: Vec::new(), + call_edges: Vec::new(), verbose, } } @@ -1589,7 +1639,7 @@ impl<'tcx> DeviceCollector<'tcx> { continue; } if let Some(ref terminator) = bb_data.terminator { - self.process_terminator(terminator, mir, &func, &ctx); + self.process_terminator(bb, terminator, mir, &func, &ctx); } } } @@ -1598,6 +1648,8 @@ impl<'tcx> DeviceCollector<'tcx> { } CollectionResult { + library_calls: self.library_calls, + call_edges: self.call_edges, functions: self.result, device_externs: self.device_externs, requires_ptx_bundle_merge: false, @@ -1648,7 +1700,12 @@ impl<'tcx> DeviceCollector<'tcx> { .next() .is_some() }; - attrs.is_convergent = check("convergent"); + attrs.is_convergent = rustc_attr_ir::find_attr!( + self.tcx, def_id, Doc(doc) + if doc.aliases.contains_key(&Symbol::intern( + reserved_oxide_symbols::DEVICE_EXTERN_CONVERGENT_MARKER + )) + ); attrs.is_pure = check("pure"); attrs.is_readonly = check("readonly"); @@ -1663,6 +1720,7 @@ impl<'tcx> DeviceCollector<'tcx> { /// diagnostics to the right kernel and user-code span. fn process_terminator( &mut self, + block: BasicBlock, terminator: &rustc_middle::mir::Terminator<'tcx>, mir: &rustc_middle::mir::Body<'tcx>, caller: &CollectedFunction<'tcx>, @@ -1675,13 +1733,13 @@ impl<'tcx> DeviceCollector<'tcx> { // (The function operand's own span is reset to a dummy by MIR // inlining, so it is useless for diagnostics.) let call_span = outermost_user_span(mir, terminator.source_info); - self.process_call_operand(func, call_span, caller, ctx); + self.process_call_operand(block, func, call_span, caller, ctx); } TerminatorKind::Drop { place, .. } => { // Collect `drop_in_place::` when T has non-trivial drop glue. // Without this, the MIR translator would emit a call to a // `drop_in_place` symbol that has no definition in the module. - self.process_drop_place(place, mir, caller, ctx); + self.process_drop_place(block, place, mir, caller, ctx); } _ => {} } @@ -1706,6 +1764,7 @@ impl<'tcx> DeviceCollector<'tcx> { /// directly and enqueues it. fn process_drop_place( &mut self, + block: BasicBlock, place: &rustc_middle::mir::Place<'tcx>, mir: &rustc_middle::mir::Body<'tcx>, caller: &CollectedFunction<'tcx>, @@ -1733,6 +1792,12 @@ impl<'tcx> DeviceCollector<'tcx> { return; } + self.call_edges.push(crate::library::CallSite { + caller: caller.instance, + block, + callee: drop_instance, + kind: crate::library::EdgeKind::Drop, + }); let mangled = self.tcx.symbol_name(drop_instance).name.to_string(); if self.seen.contains(&mangled) { return; @@ -1829,6 +1894,7 @@ impl<'tcx> DeviceCollector<'tcx> { /// `ctx` is the caller's discovery provenance, used for diagnostics. fn process_call_operand( &mut self, + block: BasicBlock, func: &rustc_middle::mir::Operand<'tcx>, call_span: Span, caller: &CollectedFunction<'tcx>, @@ -2014,7 +2080,13 @@ impl<'tcx> DeviceCollector<'tcx> { if is_callable_trait_method && let Some(receiver_ty) = args.iter().next().and_then(|arg| arg.as_type()) { - self.enqueue_callable_trait_receiver_body(receiver_ty, call_span, caller, &callee_ctx); + self.enqueue_callable_trait_receiver_body( + block, + receiver_ty, + call_span, + caller, + &callee_ctx, + ); // Don't return - continue to try resolving the trait method too // (even though it may fail, we still want to try). } @@ -2029,6 +2101,38 @@ impl<'tcx> DeviceCollector<'tcx> { return; }; + // Preserve edges before deduplicating bodies: the same helper can be + // reached from several kernels with distinct resource requirements. + let site = crate::library::CallSite { + caller: caller.instance, + block, + callee: resolved, + kind: crate::library::EdgeKind::Direct, + }; + self.call_edges.push(site); + if let Some(&kind) = self.library_defs.get(&resolved.def_id()) { + if kind == crate::library::MarkerKind::Callback { + crate::library::callback_instance(self.tcx, resolved) + .unwrap_or_else(|error| self.tcx.dcx().span_fatal(call_span, error)); + // This callable edge exists even when the thunk body was already + // collected through another root. No Rust anchor call is needed. + self.enqueue_callable_trait_receiver_body( + block, + resolved.args.type_at(4), + call_span, + caller, + &callee_ctx, + ); + } + self.library_calls.push(crate::library::LibraryCall { + instance: resolved, + site, + kind, + span: call_span, + }); + return; + } + // Skip already-seen monomorphizations (use mangled name as unique key) let mangled = self.tcx.symbol_name(resolved).name.to_string(); if self.seen.contains(&mangled) { @@ -2229,6 +2333,7 @@ impl<'tcx> DeviceCollector<'tcx> { fn enqueue_callable_trait_receiver_body( &mut self, + block: BasicBlock, ty: Ty<'tcx>, call_span: Span, caller: &CollectedFunction<'tcx>, @@ -2267,6 +2372,18 @@ impl<'tcx> DeviceCollector<'tcx> { _ => return, }; + if self.library_defs.contains_key(&instance.def_id()) { + self.tcx.dcx().span_fatal( + call_span, + "the native library marker requires a direct call; pass a wrapper helper instead", + ); + } + self.call_edges.push(crate::library::CallSite { + caller: caller.instance, + block, + callee: instance, + kind: crate::library::EdgeKind::Callable, + }); let target = self.tcx.def_path_str(instance.def_id()); let crate_name = self.tcx.crate_name(instance.def_id().krate); if self.is_generated_intrinsic_placeholder_or_report_mismatch( diff --git a/cuda-oxide/crates/rustc-codegen-cuda/src/device_codegen.rs b/cuda-oxide/crates/rustc-codegen-cuda/src/device_codegen.rs index f22d0937d9..2118eb50ff 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/src/device_codegen.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/src/device_codegen.rs @@ -93,7 +93,7 @@ //! The cost is one extra type conversion step, but this happens once per function //! and is negligible compared to actual compilation time. -use crate::collector::{CollectedFunction, DeviceExternDecl}; +use crate::collector::{CollectedFunction, CollectionResult}; use llvm_export::ops::{ DebugInlinedScope, DebugSourcePosition, DebugSourceScope, DebugSourceScopeLocation, DebugSourceScopeMap, @@ -325,6 +325,8 @@ pub struct DeviceCodegenResult { pub debug_kind: llvm_export::export::DebugKind, /// Source launch bounds for kernel entries, keyed by exported kernel name. pub kernel_launch_bounds: BTreeMap, + /// Finalized resource requirements for each concrete kernel export. + pub kernel_launch_facts: BTreeMap, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -878,6 +880,36 @@ fn is_generic_device_collector_boundary<'tcx>(tcx: TyCtxt<'tcx>, instance: Insta .is_some_and(|name| reserved_oxide_symbols::is_device_symbol(name.as_str())) } +/// Capture source argument classes before the importer erases nominal types. +fn library_root_arguments( + tcx: TyCtxt<'_>, + instance: &rustc_public::mir::mono::Instance, +) -> Vec { + use mir_importer::LibraryRootArgumentClass as Class; + use rustc_middle::ty::TyKind; + use rustc_public::rustc_internal; + let Some(body) = instance.body() else { + return Vec::new(); + }; + (1..=body.arg_locals().len()) + .map(|arg| { + let ty = rustc_internal::internal(tcx, body.locals()[arg].ty); + if crate::library::resource_parameter(tcx, ty) { + Class::ResourceCandidate + } else if crate::library::storage_handle_type(tcx, ty) { + Class::StorageCandidate + } else if matches!( + ty.kind(), + TyKind::Bool | TyKind::Int(_) | TyKind::Uint(_) | TyKind::Float(_) + ) { + Class::Scalar + } else { + Class::Kernel + } + }) + .collect() +} + /// Generates PTX for device functions using the cuda-oxide pipeline. /// /// This is the main entry point for device codegen. It bridges between @@ -887,12 +919,12 @@ fn is_generic_device_collector_boundary<'tcx>(tcx: TyCtxt<'tcx>, instance: Insta /// ## Parameters /// /// - `tcx`: The type context from rustc -/// - `functions`: Collected device functions from the collector module +/// - `collection`: Collected device functions, native calls and externs /// - `config`: Output and diagnostic configuration /// /// ## Returns /// -/// - `Ok(DeviceCodegenResult)`: Paths to generated .ll and .ptx files +/// - `Ok((DeviceCodegenResult, ProviderLibraries))`: Code and owned native inputs /// - `Err(DeviceCodegenError)`: Description of what went wrong /// /// ## Pipeline Stages @@ -902,7 +934,7 @@ fn is_generic_device_collector_boundary<'tcx>(tcx: TyCtxt<'tcx>, instance: Insta /// │ /// ├──▶ rustc_internal::stable() ──▶ rustc_public::Instance /// │ -/// └──▶ mir_importer::run_pipeline() +/// └──▶ import → prepare → prove → provider → bind → finish /// │ /// ├──▶ `dialect-mir` (alloca form) /// │ @@ -917,11 +949,14 @@ fn is_generic_device_collector_boundary<'tcx>(tcx: TyCtxt<'tcx>, instance: Insta /// ``` pub fn generate_device_code<'tcx>( tcx: TyCtxt<'tcx>, - functions: &[CollectedFunction<'tcx>], - device_externs: &[DeviceExternDecl], + collection: &CollectionResult<'tcx>, config: &DeviceCodegenConfig, -) -> Result { +) -> Result<(DeviceCodegenResult, crate::library::ProviderLibraries), DeviceCodegenError> { use rustc_public::rustc_internal; + let functions = &collection.functions; + let device_externs = &collection.device_externs; + let authenticated = + crate::library::authenticate(tcx, collection).map_err(DeviceCodegenError::PtxGeneration)?; if functions.is_empty() { return Err(DeviceCodegenError::NoKernels); @@ -1120,7 +1155,7 @@ pub fn generate_device_code<'tcx>( .map(|func| crate::collector::device_mono_reachability(tcx, func.instance)) .collect(); - let result = rustc_internal::run(tcx, || { + let imported = rustc_internal::run(tcx, || { let mut debug_global_variables = BTreeMap::new(); for owned in &static_debug_identities { let stable_item = rustc_internal::stable(MonoItem::Static(owned.def_id)); @@ -1183,6 +1218,13 @@ pub fn generate_device_code<'tcx>( (((func, (export_name, is_kernel)), debug_source_scopes), inline), reachability, )| { + // A dependency can instantiate a marked helper before a + // kernel supplies its block contract. Keep its Rust MIR + // for that consumer, but do not emit an unplanned checked + // marker here. Independent standalone functions remain. + if authenticated.deferred.contains(&func.instance) { + return None; + } // Use rustc_internal::stable() to convert the Instance. // This is the key bridge between rustc_middle and rustc_public types. let stable_instance = rustc_internal::stable(func.instance); @@ -1203,7 +1245,21 @@ pub fn generate_device_code<'tcx>( return None; } + let library_root_arguments = if *is_kernel { + library_root_arguments(tcx, &stable_instance) + } else { + Vec::new() + }; Some(mir_importer::CollectedFunction { + library_source: Some(mir_importer::LibraryFunctionSourceAttr { + export: export_name.clone(), + mangled: tcx.symbol_name(func.instance).name.to_string(), + diagnostic: tcx.def_path_str(func.instance.def_id()), + checked_ancestor: authenticated + .checked_ancestors + .contains(&func.instance), + }), + library_root_arguments, instance: stable_instance, rustc_mir_block_count: reachability.block_count, rustc_mono_successors: reachability.successors.clone(), @@ -1282,70 +1338,84 @@ pub fn generate_device_code<'tcx>( index_mut_trait: lang_items.index_mut_trait().map(rustc_internal::stable), }; - // Run the cuda-oxide pipeline! - // Rust MIR → `dialect-mir` → mem2reg → unroll → LLVM dialect → LLVM IR → PTX. - // Device externs are emitted as `declare` statements in LLVM IR - mir_importer::run_pipeline( + // Import while rustc_public identities are live; preparation and proof + // continue on the same owned module outside this Rust context. + mir_importer::import_pipeline_with_libraries( &stable_functions, &stable_device_externs, &pipeline_config, known_defs, + authenticated.semantics.clone(), + authenticated.call_targets.clone(), + authenticated.selected_call_targets.clone(), + authenticated.callbacks.clone(), ) }); - // Handle the result from rustc_internal::run. - // We have nested Results: outer from run(), inner from run_pipeline(). - match result { - Ok(pipeline_result) => match pipeline_result { - Ok(compilation_result) => { - let artifact = read_compilation_artifact(&compilation_result)?; - let ptx_content = match artifact.as_ref() { - Some(artifact) if artifact.kind == DeviceCodegenArtifactKind::Ptx => { - Some(String::from_utf8(artifact.bytes.clone()).map_err(|e| { - DeviceCodegenError::PtxGeneration(format!( - "generated PTX is not valid UTF-8: {e}" - )) - })?) - } - _ => None, - }; + let mut imported = imported + .map_err(|error| DeviceCodegenError::StableMirError(format!("{error:?}")))? + .map_err(|error| DeviceCodegenError::PtxGeneration(error.to_string()))?; + // Prove the actual prepared MIR before provider I/O. Only owned admitted + // shapes cross back into this Rust boundary. + let mut prepared = imported + .prepare() + .map_err(|error| DeviceCodegenError::PtxGeneration(error.to_string()))?; + let shapes = prepared + .validate_libraries(&authenticated.rooted_functions) + .map_err(|error| DeviceCodegenError::PtxGeneration(error.to_string()))?; + let libraries = + crate::library::finalize_provider(tcx, collection, &shapes, prepared.callback_symbols()) + .map_err(DeviceCodegenError::PtxGeneration)?; + prepared + .finish_library_planning( + &libraries.native_plans, + libraries.declarations.clone(), + libraries.roots.clone(), + ) + .map_err(|error| DeviceCodegenError::PtxGeneration(error.to_string()))?; + let compilation_result = prepared + .finish() + .map_err(|error| DeviceCodegenError::PtxGeneration(error.to_string()))?; + + let artifact = read_compilation_artifact(&compilation_result)?; + let ptx_content = match artifact.as_ref() { + Some(artifact) if artifact.kind == DeviceCodegenArtifactKind::Ptx => { + Some(String::from_utf8(artifact.bytes.clone()).map_err(|e| { + DeviceCodegenError::PtxGeneration(format!("generated PTX is not valid UTF-8: {e}")) + })?) + } + _ => None, + }; - if config.verbose { - if let Some(artifact) = artifact.as_ref() { - eprintln!( - "[device_codegen] Embeddable artifact generated: {} ({:?}, target: {})", - artifact.name, artifact.kind, compilation_result.target - ); - } else { - eprintln!( - "[device_codegen] No embeddable artifact found for {} (target: {})", - compilation_result.ll_path.display(), - compilation_result.target - ); - } - } + if config.verbose { + if let Some(artifact) = artifact.as_ref() { + eprintln!( + "[device_codegen] Embeddable artifact generated: {} ({:?}, target: {})", + artifact.name, artifact.kind, compilation_result.target + ); + } else { + eprintln!( + "[device_codegen] No embeddable artifact found for {} (target: {})", + compilation_result.ll_path.display(), + compilation_result.target + ); + } + } - Ok(DeviceCodegenResult { - ptx_path: compilation_result.ptx_path, - ll_path: compilation_result.ll_path, - target: compilation_result.target, - ptx_content, - artifact, - allow_fma_contraction: compilation_result.allow_fma_contraction, - debug_kind, - kernel_launch_bounds: compilation_result.kernel_launch_bounds, - }) - } - Err(pipeline_err) => Err(DeviceCodegenError::PtxGeneration(format!( - "{}", - pipeline_err - ))), + Ok(( + DeviceCodegenResult { + ptx_path: compilation_result.ptx_path, + ll_path: compilation_result.ll_path, + target: compilation_result.target, + ptx_content, + artifact, + allow_fma_contraction: compilation_result.allow_fma_contraction, + debug_kind, + kernel_launch_bounds: compilation_result.kernel_launch_bounds, + kernel_launch_facts: compilation_result.kernel_launch_facts, }, - Err(stable_mir_err) => Err(DeviceCodegenError::StableMirError(format!( - "{:?}", - stable_mir_err - ))), - } + libraries, + )) } fn device_debug_kind(rustc_debug: DebugInfo) -> llvm_export::export::DebugKind { @@ -1477,6 +1547,7 @@ mod tests { target: "sm_90".to_string(), allow_fma_contraction: false, kernel_launch_bounds: BTreeMap::new(), + kernel_launch_facts: BTreeMap::new(), }; let artifact = read_compilation_artifact(&result).unwrap().unwrap(); @@ -1504,6 +1575,7 @@ mod tests { target: "sm_90".to_string(), allow_fma_contraction: true, kernel_launch_bounds: BTreeMap::new(), + kernel_launch_facts: BTreeMap::new(), }; let artifact = read_compilation_artifact(&result).unwrap().unwrap(); diff --git a/cuda-oxide/crates/rustc-codegen-cuda/src/lib.rs b/cuda-oxide/crates/rustc-codegen-cuda/src/lib.rs index 8564f84dbe..5ec073f6c8 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/src/lib.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/src/lib.rs @@ -332,16 +332,19 @@ extern crate rustc_codegen_llvm; mod collector; mod device_codegen; mod generated_intrinsics; +mod library; mod materialize; use rustc_codegen_ssa::traits::CodegenBackend; -use rustc_codegen_ssa::{CompiledModule, CompiledModules, CrateInfo, ModuleKind}; +use rustc_codegen_ssa::{CompiledModule, CompiledModules, CrateInfo, ModuleKind, NativeLib}; +use rustc_hir::attrs::NativeLibKind; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::WorkProductMap; use rustc_middle::ty::TyCtxt; use rustc_middle::ty::print::with_no_trimmed_paths; -use rustc_session::config::OutputFilenames; +use rustc_session::config::{CrateType, OutputFilenames}; use rustc_session::{IncrCompSession, Session}; +use rustc_span::Symbol; use std::any::Any; use std::collections::BTreeSet; use std::path::{Path, PathBuf}; @@ -715,7 +718,7 @@ impl CodegenBackend for CudaCodegenBackend { // Step 2: If device code exists, compile via cuda-oxide let _device_result = if has_device_code { - let materialization_request = + let mut materialization_request = materialize::request_from_env().unwrap_or_else(|error| { tcx.dcx().fatal(format!( "[rustc_codegen_cuda] Invalid cubin materialization request: {error}" @@ -740,10 +743,15 @@ impl CodegenBackend for CudaCodegenBackend { self.config.verbose, ); + if !collection_result.library_calls.is_empty() && materialization_request.is_none() + { + tcx.dcx().fatal( + "native library calls require `cargo oxide build --materialize-cubin`", + ); + } materialize::validate_collection( - materialization_request, + materialization_request.as_ref(), !collection_result.device_externs.is_empty(), - collection_result.requires_ptx_bundle_merge, ) .unwrap_or_else(|error| { tcx.dcx().fatal(format!( @@ -779,6 +787,16 @@ impl CodegenBackend for CudaCodegenBackend { dump_mir_dialect: self.config.dump_mir_dialect, dump_llvm_dialect: self.config.dump_llvm_dialect, }; + materialize::validate_output_paths( + materialization_request.as_ref(), + &device_config.output_dir, + &device_config.output_name, + ) + .unwrap_or_else(|error| { + tcx.dcx().fatal(format!( + "[rustc_codegen_cuda] Cannot write device artifacts: {error}" + )) + }); // Run the cuda-oxide pipeline, catching backend panics and // re-emitting them as a cuda-oxide diagnostic. A panic @@ -813,8 +831,7 @@ impl CodegenBackend for CudaCodegenBackend { let r = catch_unwind(AssertUnwindSafe(|| { device_codegen::generate_device_code( tcx, - device_functions, - &collection_result.device_externs, + &collection_result, &device_config, ) })); @@ -824,6 +841,11 @@ impl CodegenBackend for CudaCodegenBackend { match panic_outcome { Err(payload) => { + // rustc fatal diagnostics have already been emitted. Preserve + // their control flow instead of reporting a second ICE. + if payload.is::() { + std::panic::resume_unwind(payload); + } let msg = payload .downcast_ref::<&str>() .map(|s| s.to_string()) @@ -847,7 +869,22 @@ impl CodegenBackend for CudaCodegenBackend { Please file at https://github.com/NVIDIA/cuda-rust/issues" )); } - Ok(Ok(result)) => { + Ok(Ok((result, mut libraries))) => { + if self.config.verbose { + for (kernel, facts) in &result.kernel_launch_facts { + eprintln!( + "[library] {kernel}: exact block {:?}, scratch {} bytes aligned {}", + facts.exact_block, + facts.dynamic_shared_bytes, + facts.dynamic_shared_alignment + ); + } + } + // Native inputs are owned provider bytes; file output and + // collection checks remain before device generation. + if let Some(request) = &mut materialization_request { + request.add_library_inputs(std::mem::take(&mut libraries.inputs)); + } if self.config.verbose && let Some(artifact) = result.artifact.as_ref() { @@ -957,17 +994,12 @@ impl CodegenBackend for CudaCodegenBackend { crate_info, ); for (index, object) in ongoing.artifact_objects.into_iter().enumerate() { - compiled_modules.modules.push(CompiledModule { - name: format!("oxide_artifact_embed_{index}"), - kind: ModuleKind::Regular, - object: Some(object), - dwarf_object: None, - bytecode: None, - assembly: None, - llvm_ir: None, - global_asm_object: None, - links_from_incr_cache: Vec::new(), - }); + compiled_modules.modules.push(artifact_module( + index, + object, + &crate_info.crate_types, + sess.opts.unstable_opts.link_native_libraries, + )); } (compiled_modules, work_products) } @@ -976,10 +1008,25 @@ impl CodegenBackend for CudaCodegenBackend { &self, sess: &Session, compiled_modules: CompiledModules, - crate_info: CrateInfo, + mut crate_info: CrateInfo, metadata: EncodedMetadata, outputs: &OutputFilenames, ) { + for object in artifact_link_args(&compiled_modules.modules, &crate_info.crate_types) { + let Some(object) = object.to_str() else { + sess.dcx().fatal(format!( + "[rustc_codegen_cuda] The path of the artifact object is not UTF-8: {}", + object.display() + )); + }; + crate_info.used_libraries.push(NativeLib { + kind: NativeLibKind::LinkArg, + name: Symbol::intern(object), + cfg: None, + verbatim: true, + dll_imports: Vec::new(), + }); + } self.llvm_backend .link(sess, compiled_modules, crate_info, metadata, outputs); } @@ -1004,6 +1051,68 @@ fn write_tile_export_descriptions( Ok(path) } +/// The prefix of the names of the artifact modules. +const ARTIFACT_MODULE_PREFIX: &str = "oxide_artifact_embed_"; + +/// The module of an artifact object. +/// +/// rustc keeps a list of the Rust objects in each rlib. This list contains the +/// objects of all modules. With LTO, rustc reads bitcode from these objects and +/// removes them from the link. The artifact objects have no bitcode. If the +/// session writes an rlib, the artifact object is the DWARF object of its +/// module. rustc puts the DWARF objects in the rlib, but not in the list. +/// +/// The system linker does not get the DWARF objects. `link` gives the artifact +/// objects to this linker as link arguments. With `-Zlink-native-libraries=no`, +/// rustc does not give link arguments to the linker. Then, if the session also +/// uses the system linker, the artifact objects are Rust objects. +fn artifact_module( + index: usize, + object: PathBuf, + crate_types: &[CrateType], + link_native_libraries: bool, +) -> CompiledModule { + let dwarf = crate_types.contains(&CrateType::Rlib) + && (link_native_libraries || !crate_types.iter().any(uses_system_linker)); + let (object, dwarf_object) = if dwarf { + (None, Some(object)) + } else { + (Some(object), None) + }; + CompiledModule { + name: format!("{ARTIFACT_MODULE_PREFIX}{index}"), + kind: ModuleKind::Regular, + object, + dwarf_object, + bytecode: None, + assembly: None, + llvm_ir: None, + global_asm_object: None, + links_from_incr_cache: Vec::new(), + } +} + +/// The artifact objects that `link` gives to the system linker. +fn artifact_link_args<'a>( + modules: &'a [CompiledModule], + crate_types: &[CrateType], +) -> Vec<&'a Path> { + if !crate_types.iter().any(uses_system_linker) { + return Vec::new(); + } + modules + .iter() + .filter(|module| module.name.starts_with(ARTIFACT_MODULE_PREFIX)) + .filter_map(|module| module.dwarf_object.as_deref()) + .collect() +} + +/// rustc writes rlibs and static libraries. The system linker makes the other +/// crate types. +fn uses_system_linker(crate_type: &CrateType) -> bool { + !matches!(crate_type, CrateType::Rlib | CrateType::StaticLib) +} + #[allow(clippy::too_many_arguments)] fn write_device_artifact_object( tcx: TyCtxt<'_>, @@ -1069,10 +1178,11 @@ fn write_device_artifact_object( } else { oxide_artifacts::ArtifactEntryKind::DeviceFunction }; - spec = spec.with_entry(oxide_artifacts::ArtifactEntrySpec::new( - &function.export_name, - kind, - )); + let mut entry = oxide_artifacts::ArtifactEntrySpec::new(&function.export_name, kind); + if let Some(facts) = result.kernel_launch_facts.get(&function.export_name) { + entry = entry.with_launch_facts(*facts); + } + spec = spec.with_entry(entry); } let blob = oxide_artifacts::build_artifact_blob(&spec)?; @@ -1372,6 +1482,78 @@ pub(crate) fn host_target_supported( mod tests { use super::*; + #[test] + fn the_artifact_objects_of_an_rlib_are_not_rust_objects() { + let object = PathBuf::from("library.0.0.embed.o"); + let rlib_sessions: [&[CrateType]; 6] = [ + &[CrateType::Rlib], + &[CrateType::Rlib, CrateType::Cdylib], + &[CrateType::Rlib, CrateType::Dylib], + &[CrateType::Rlib, CrateType::StaticLib], + &[CrateType::Rlib, CrateType::StaticLib, CrateType::Cdylib], + &[CrateType::Rlib, CrateType::StaticLib, CrateType::Dylib], + ]; + for crate_types in rlib_sessions { + let module = artifact_module(0, object.clone(), crate_types, true); + assert_eq!(module.object, None, "{crate_types:?}"); + assert_eq!(module.dwarf_object, Some(object.clone()), "{crate_types:?}"); + } + + // Sessions without an rlib link the artifact objects. + let other_sessions: [&[CrateType]; 4] = [ + &[CrateType::Executable], + &[CrateType::Cdylib], + &[CrateType::Dylib], + &[CrateType::StaticLib], + ]; + for crate_types in other_sessions { + let module = artifact_module(0, object.clone(), crate_types, true); + assert_eq!(module.object, Some(object.clone()), "{crate_types:?}"); + assert_eq!(module.dwarf_object, None, "{crate_types:?}"); + } + + // With `-Zlink-native-libraries=no`, the system linker gets no link + // arguments. + let mixed = artifact_module( + 0, + object.clone(), + &[CrateType::Rlib, CrateType::Cdylib], + false, + ); + assert_eq!(mixed.object, Some(object.clone())); + let rlib = artifact_module(0, object.clone(), &[CrateType::Rlib], false); + assert_eq!(rlib.dwarf_object, Some(object)); + } + + #[test] + fn the_system_linker_gets_the_artifact_objects() { + let artifact = PathBuf::from("library.0.0.embed.o"); + let crate_types = [CrateType::Rlib, CrateType::Cdylib]; + let modules = [ + // A module of LLVM with split debug information. + CompiledModule { + name: "library.0".to_string(), + kind: ModuleKind::Regular, + object: Some(PathBuf::from("library.0.rcgu.o")), + dwarf_object: Some(PathBuf::from("library.0.rcgu.dwo")), + bytecode: None, + assembly: None, + llvm_ir: None, + global_asm_object: None, + links_from_incr_cache: Vec::new(), + }, + artifact_module(0, artifact.clone(), &crate_types, true), + ]; + assert_eq!( + artifact_link_args(&modules, &crate_types), + [artifact.as_path()] + ); + + // rustc puts the DWARF objects in rlibs and static libraries. + assert!(artifact_link_args(&modules, &[CrateType::Rlib]).is_empty()); + assert!(artifact_link_args(&modules, &[CrateType::Rlib, CrateType::StaticLib]).is_empty()); + } + #[test] fn host_targets_that_match_the_ptx_data_model_are_accepted() { use rustc_abi::Endian; diff --git a/cuda-oxide/crates/rustc-codegen-cuda/src/library.rs b/cuda-oxide/crates/rustc-codegen-cuda/src/library.rs new file mode 100644 index 0000000000..1e6bdef218 --- /dev/null +++ b/cuda-oxide/crates/rustc-codegen-cuda/src/library.rs @@ -0,0 +1,1678 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Resolve typed library markers and finalize one batch per native provider. + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::io::{Read, Write}; +use std::path::PathBuf; +use std::process::{Command, Stdio}; + +use oxide_artifacts::KernelLaunchFacts; +use rustc_hir::def_id::{CRATE_DEF_INDEX, DefId}; +use rustc_middle::mir::{BasicBlock, Operand, RETURN_PLACE, Rvalue, StatementKind, TerminatorKind}; +use rustc_middle::ty::{Instance, InstanceKind, ShimKind, Ty, TyCtxt, TyKind, TypingEnv}; +use rustc_span::Span; +use serde::{Deserialize, Serialize}; + +use crate::collector::CollectionResult; + +const MAX_HEADER: usize = 1024 * 1024; +const MAX_LTO: usize = 256 * 1024 * 1024; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum MarkerKind { + Raw, + CheckedBlock, + Callback, + CheckedBlockResource, + CheckedBlockStorage, + StorageNew, + StorageReconfigure, + StorageReserve, + StorageRelease, +} + +impl MarkerKind { + pub(crate) fn checked(self) -> bool { + matches!( + self, + Self::CheckedBlock + | Self::CheckedBlockResource + | Self::CheckedBlockStorage + | Self::StorageNew + | Self::StorageReconfigure + | Self::StorageReserve + | Self::StorageRelease + ) + } + + pub(crate) fn native(self) -> bool { + matches!( + self, + Self::Raw + | Self::Callback + | Self::CheckedBlock + | Self::CheckedBlockResource + | Self::CheckedBlockStorage + ) + } + + pub(crate) fn storage(self) -> bool { + matches!( + self, + Self::CheckedBlockStorage + | Self::StorageNew + | Self::StorageReconfigure + | Self::StorageReserve + | Self::StorageRelease + ) + } + + pub(crate) fn resource(self) -> bool { + matches!(self, Self::CheckedBlockResource | Self::CheckedBlockStorage) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum EdgeKind { + Direct, + Callable, + Drop, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct CallSite<'tcx> { + pub caller: Instance<'tcx>, + pub block: BasicBlock, + pub callee: Instance<'tcx>, + pub kind: EdgeKind, +} + +#[derive(Debug)] +pub(crate) struct LibraryCall<'tcx> { + pub instance: Instance<'tcx>, + pub site: CallSite<'tcx>, + pub kind: MarkerKind, + pub span: Span, +} + +/// Resolve definitions through the actual dependency module, then compare DefIds +/// at call sites. A user function with the same spelling is an ordinary function. +pub(crate) fn marker_definitions(tcx: TyCtxt<'_>) -> HashMap { + let mut definitions = HashMap::new(); + for &krate in tcx.crates(()) { + if tcx.crate_name(krate).as_str() != "cuda_device" { + continue; + } + let root = DefId { + krate, + index: CRATE_DEF_INDEX, + }; + for module in tcx.module_children(root) { + if module.ident.name.as_str() != "library" { + continue; + } + let Some(module) = module.res.opt_def_id() else { + continue; + }; + for item in tcx.module_children(module) { + let kind = match item.ident.name.as_str() { + reserved_oxide_symbols::LIBRARY_CALL_V1 => MarkerKind::Raw, + reserved_oxide_symbols::LIBRARY_CALLBACK_CALL_V1 => MarkerKind::Callback, + reserved_oxide_symbols::LIBRARY_CHECKED_BLOCK_CALL_V1 => { + MarkerKind::CheckedBlock + } + reserved_oxide_symbols::LIBRARY_CHECKED_BLOCK_RESOURCE_CALL_V1 => { + MarkerKind::CheckedBlockResource + } + reserved_oxide_symbols::LIBRARY_CHECKED_BLOCK_STORAGE_CALL_V1 => { + MarkerKind::CheckedBlockStorage + } + reserved_oxide_symbols::LIBRARY_STORAGE_NEW_V1 => MarkerKind::StorageNew, + reserved_oxide_symbols::LIBRARY_STORAGE_RECONFIGURE_V1 => { + MarkerKind::StorageReconfigure + } + reserved_oxide_symbols::LIBRARY_STORAGE_RESERVE_V1 => { + MarkerKind::StorageReserve + } + reserved_oxide_symbols::LIBRARY_STORAGE_RELEASE_V1 => { + MarkerKind::StorageRelease + } + _ => continue, + }; + if let Some(def) = item.res.opt_def_id() { + definitions.insert(def, kind); + } + } + } + } + definitions +} + +fn device_type(tcx: TyCtxt<'_>, module_name: &str, name: &str) -> Option { + for &krate in tcx.crates(()) { + if tcx.crate_name(krate).as_str() != "cuda_device" { + continue; + } + let root = DefId { + krate, + index: CRATE_DEF_INDEX, + }; + let module = tcx + .module_children(root) + .iter() + .find(|item| item.ident.name.as_str() == module_name)? + .res + .opt_def_id()?; + return tcx + .module_children(module) + .iter() + .find(|item| item.ident.name.as_str() == name) + .and_then(|item| item.res.opt_def_id()); + } + None +} + +/// Genuine collective carrier, independent of aliases or source spelling. +pub(crate) fn collective_carrier<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<(Ty<'tcx>, u64)> { + let TyKind::Adt(def, args) = ty.kind() else { + return None; + }; + if Some(def.did()) != device_type(tcx, "disjoint", "DisjointSlice") { + return None; + } + let mut types = args.types(); + let element = types.next()?; + let TyKind::Adt(space, args) = types.next()?.kind() else { + return None; + }; + if Some(space.did()) != device_type(tcx, "disjoint", "CollectiveTiles") { + return None; + } + Some((element, args.const_at(0).try_to_target_usize(tcx)?)) +} + +pub(crate) fn resource_parameter<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool { + collective_carrier(tcx, ty).is_some() + || matches!(ty.kind(), TyKind::Ref(_, inner, rustc_hir::Mutability::Not) + if matches!(inner.kind(), TyKind::Slice(_))) +} + +pub(crate) fn resource_write(call: &LibraryCall<'_>) -> bool { + call.instance + .args + .const_at(if call.kind == MarkerKind::CheckedBlockStorage { + 7 + } else { + 6 + }) + .try_to_leaf() + .and_then(|value| value.try_to_bool().ok()) + .expect("validated resource access") +} + +fn validate_resource<'tcx>(tcx: TyCtxt<'tcx>, call: &LibraryCall<'tcx>) -> Result<(), String> { + let args = call.instance.args; + let write = args + .const_at(if call.kind == MarkerKind::CheckedBlockStorage { + 7 + } else { + 6 + }) + .try_to_leaf() + .and_then(|value| value.try_to_bool().ok()) + .ok_or("resource access mode is not constant")?; + let TyKind::Ref(_, inner, mutability) = args.type_at(2).kind() else { + return Err( + "resource witness must borrow a genuine collective carrier or read-only slice".into(), + ); + }; + let element = if let TyKind::Slice(element) = inner.kind() + && !write + && *mutability == rustc_hir::Mutability::Not + { + *element + } else { + let (element, capacity) = collective_carrier(tcx, *inner) + .ok_or("resource witness requires DisjointSlice>")?; + if capacity == 0 { + return Err("resource carriers require positive capacity".into()); + } + if write && *mutability != rustc_hir::Mutability::Mut { + return Err("resource writes require a mutable carrier witness".into()); + } + element + }; + if element != args.type_at(0) { + return Err("resource witness element type disagrees with the library element type".into()); + } + Ok(()) +} + +fn storage_extent<'tcx>( + tcx: TyCtxt<'tcx>, + args: rustc_middle::ty::GenericArgsRef<'tcx>, + index: usize, +) -> Result { + args.const_at(index) + .try_to_target_usize(tcx) + .and_then(|n| u32::try_from(n).ok()) + .ok_or_else(|| "storage extent must be a concrete usize fitting u32".into()) +} +fn storage_boolean( + args: rustc_middle::ty::GenericArgsRef<'_>, + index: usize, +) -> Result { + args.const_at(index) + .try_to_leaf() + .and_then(|n| n.try_to_bool().ok()) + .ok_or_else(|| "storage policy must be a concrete bool".into()) +} +fn storage_alignment(value: u32) -> Result { + if value != 0 && !value.is_power_of_two() { + return Err("storage alignment must be zero or a power of two".into()); + } + Ok(value) +} + +fn storage_policy<'tcx>( + tcx: TyCtxt<'tcx>, + ty: Ty<'tcx>, +) -> Result { + let TyKind::Adt(def, args) = ty.kind() else { + return Err("expected a genuine StorageDescriptor".into()); + }; + if Some(def.did()) != device_type(tcx, "library", "StorageDescriptor") { + return Err("expected a genuine StorageDescriptor".into()); + } + Ok(mir_importer::LibraryStoragePolicy { + capacity: storage_extent(tcx, args, 0)?, + alignment: storage_alignment(storage_extent(tcx, args, 1)?)?, + auto_sync: storage_boolean(args, 2)?, + exclusive: storage_boolean(args, 3)?, + }) +} + +pub(crate) fn storage_handle_type(tcx: TyCtxt<'_>, ty: Ty<'_>) -> bool { + matches!(ty.kind(), TyKind::Adt(def, _) if ["StorageDescriptor","StorageReservation"].iter().any(|name|Some(def.did()) == device_type(tcx, "library", name))) +} + +fn storage_borrow_policy<'tcx>( + tcx: TyCtxt<'tcx>, + ty: Ty<'tcx>, +) -> Result { + let TyKind::Ref(_, inner, rustc_hir::Mutability::Mut) = ty.kind() else { + return Err("storage must mutably borrow a genuine descriptor".into()); + }; + storage_policy(tcx, *inner) +} + +fn storage_new_policy<'tcx>( + tcx: TyCtxt<'tcx>, + call: &LibraryCall<'tcx>, +) -> Result { + let args = call.instance.args; + Ok(mir_importer::LibraryStoragePolicy { + capacity: storage_extent(tcx, args, 0)?, + alignment: storage_alignment(storage_extent(tcx, args, 1)?)?, + auto_sync: storage_boolean(args, 2)?, + exclusive: storage_boolean(args, 3)?, + }) +} + +fn storage_reserve_layout<'tcx>( + tcx: TyCtxt<'tcx>, + call: &LibraryCall<'tcx>, +) -> Result<(u32, u32), String> { + let args = call.instance.args; + let element = args.type_at(0); + scalar(element)?; + // f16 is not in the frozen ten-type reservation set. + if matches!( + element.kind(), + TyKind::Float(rustc_middle::ty::FloatTy::F16) + ) { + return Err("storage reservations require one of the ten fixed-width numeric types".into()); + } + let count = storage_extent(tcx, args, 2)?; + if count == 0 { + return Err("storage reservation extent must be positive".into()); + } + let array = Ty::new_array(tcx, element, u64::from(count)); + let layout = tcx + .layout_of(TypingEnv::fully_monomorphized().as_query_input(array)) + .map_err(|_| "cannot compute storage reservation layout")?; + let bytes = + u32::try_from(layout.size.bytes()).map_err(|_| "storage reservation exceeds u32")?; + let align = u32::try_from(layout.align.abi.bytes()) + .map_err(|_| "storage reservation alignment exceeds u32")?; + Ok(( + bytes, + align.max(storage_alignment(storage_extent(tcx, args, 3)?)?), + )) +} + +fn validate_storage<'tcx>(tcx: TyCtxt<'tcx>, call: &LibraryCall<'tcx>) -> Result<(), String> { + let args = call.instance.args; + match call.kind { + MarkerKind::StorageNew => { + storage_new_policy(tcx, call)?; + } + MarkerKind::StorageReconfigure => { + storage_policy(tcx, args.type_at(0))?; + storage_policy(tcx, args.type_at(1))?; + } + MarkerKind::StorageReserve => { + storage_policy(tcx, args.type_at(1))?; + storage_reserve_layout(tcx, call)?; + } + MarkerKind::StorageRelease => { + scalar(args.type_at(0))?; + } + MarkerKind::CheckedBlockStorage => { + let ty = args.type_at(3); + if !matches!(ty.kind(), TyKind::Tuple(fields) if fields.is_empty()) { + storage_borrow_policy(tcx, ty)?; + } + } + _ => {} + } + Ok(()) +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Registration { + version: u32, + provider: String, + executable: PathBuf, + operation: String, + algorithm: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Hash)] +#[serde( + tag = "kind", + content = "bits", + rename_all = "snake_case", + deny_unknown_fields +)] +enum Scalar { + Signed(u16), + Unsigned(u16), + Float(u16), +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Hash)] +#[serde( + tag = "kind", + content = "scalar", + rename_all = "snake_case", + deny_unknown_fields +)] +enum Argument { + ConstPointer(Scalar), + MutPointer(Scalar), + Value(Scalar), +} + +#[derive(Debug, Serialize)] +struct BatchRequest<'a> { + version: u32, + target: &'a str, + cuda_root: PathBuf, + requests: &'a [Request], +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq, Hash)] +struct Request { + operation: String, + algorithm: String, + dtype: Scalar, + block: [u32; 3], + items_per_thread: u32, + #[serde(skip_serializing_if = "Option::is_none")] + callback: Option, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq, Hash)] +struct Callback { + symbol: String, + arguments: Vec, + result: Scalar, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Layout { + size: u64, + alignment: u64, +} + +impl Layout { + fn checked(&self) -> Result<(u32, u32), String> { + let size = u32::try_from(self.size).map_err(|_| "library layout exceeds u32")?; + let align = u32::try_from(self.alignment).map_err(|_| "library alignment exceeds u32")?; + if !align.is_power_of_two() { + return Err("library alignment must be a nonzero power of two".into()); + } + Ok((size, align)) + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Specialization { + request_index: u32, + identity: [u8; 32], + symbol: String, + arguments: Vec, + cub_layout: Layout, + scratch: Layout, + scratch_free: bool, + mutates_payload: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct BatchResponse { + version: u32, + batch_identity: [u8; 32], + specializations: Vec, +} + +/// Owned provider results, independent of Rust body and proof state. +#[derive(Default)] +pub(crate) struct ProviderLibraries { + pub native_plans: mir_importer::RootedNativePlans, + pub declarations: Vec, + pub inputs: Vec<(String, Vec)>, + pub roots: BTreeMap, +} + +fn scalar(ty: Ty<'_>) -> Result { + use rustc_middle::ty::{FloatTy, IntTy, UintTy}; + match ty.kind() { + TyKind::Int(kind) => match kind { + IntTy::I8 => Ok(Scalar::Signed(8)), + IntTy::I16 => Ok(Scalar::Signed(16)), + IntTy::I32 => Ok(Scalar::Signed(32)), + IntTy::I64 => Ok(Scalar::Signed(64)), + _ => Err(format!("unsupported library scalar `{ty}`")), + }, + TyKind::Uint(kind) => match kind { + UintTy::U8 => Ok(Scalar::Unsigned(8)), + UintTy::U16 => Ok(Scalar::Unsigned(16)), + UintTy::U32 => Ok(Scalar::Unsigned(32)), + UintTy::U64 => Ok(Scalar::Unsigned(64)), + _ => Err(format!("unsupported library scalar `{ty}`")), + }, + TyKind::Float(FloatTy::F16) => Ok(Scalar::Float(16)), + TyKind::Float(FloatTy::F32) => Ok(Scalar::Float(32)), + TyKind::Float(FloatTy::F64) => Ok(Scalar::Float(64)), + _ => Err(format!( + "unsupported library scalar `{ty}`; use a fixed-width numeric scalar" + )), + } +} + +fn argument(ty: Ty<'_>) -> Result { + match ty.kind() { + TyKind::RawPtr(pointee, mutability) => Ok(if mutability.is_mut() { + Argument::MutPointer(scalar(*pointee)?) + } else { + Argument::ConstPointer(scalar(*pointee)?) + }), + _ => Ok(Argument::Value(scalar(ty)?)), + } +} + +/// Resolve the selected witness without ever coercing it to a runtime pointer. +pub(crate) fn callback_instance<'tcx>( + tcx: TyCtxt<'tcx>, + marker: Instance<'tcx>, +) -> Result, String> { + let environment = marker.args.type_at(3); + match environment.kind() { + TyKind::FnDef(..) => {} + TyKind::Closure(_, args) => { + if args.as_closure().kind() != rustc_middle::ty::ClosureKind::Fn { + return Err("callback environment must be an immutable Fn closure".into()); + } + for capture in rustc_middle::ty::UpvarArgs::Closure(args).upvar_tys() { + scalar(capture).map_err(|_| format!( + "callback capture `{capture}` must be a fixed-width numeric scalar captured by value"))?; + } + } + _ => { + return Err( + "callback environment must be a function item or scalar-capturing closure".into(), + ); + } + } + if scalar(marker.args.type_at(0))? != Scalar::Signed(32) { + return Err("callback version one supports i32 only".into()); + } + let witness = marker.args.type_at(4); + let TyKind::FnDef(def, args) = witness.kind() else { + return Err( + "callback thunk must be a function item, not a closure or function pointer".into(), + ); + }; + let args = args + .no_bound_vars() + .ok_or("callback thunk has unresolved bound arguments")?; + let instance = Instance::try_resolve(tcx, TypingEnv::fully_monomorphized(), *def, args) + .map_err(|_| "cannot resolve callback thunk")? + .ok_or("cannot resolve callback thunk")?; + let sig = tcx.fn_sig(*def).instantiate(tcx, args).skip_binder(); + let correct_pointer = matches!(sig.inputs().first().map(|ty| ty.kind()), + Some(TyKind::RawPtr(pointee, mutability)) if !mutability.is_mut() && *pointee == environment); + if !matches!(sig.abi(), rustc_abi::ExternAbi::C { unwind: false }) + || sig.safety().is_safe() + || sig.c_variadic() + || sig.inputs().len() != 3 + || !correct_pointer + || sig.inputs()[1..].iter().any(|ty| *ty != tcx.types.i32) + || sig.output() != tcx.types.i32 + { + return Err("callback thunk requires unsafe extern \"C\" fn(*const Environment, i32, i32) -> i32 without unwinding".into()); + } + Ok(instance) +} + +fn read_registration<'tcx>(tcx: TyCtxt<'tcx>, witness: Ty<'tcx>) -> Result { + let TyKind::FnDef(def, args) = witness.kind() else { + return Err( + "library registration must be a function item, not a closure or function pointer" + .into(), + ); + }; + let args = args + .no_bound_vars() + .ok_or("registration has unresolved bound arguments")?; + let instance = Instance::try_resolve(tcx, TypingEnv::fully_monomorphized(), *def, args) + .map_err(|_| "cannot resolve library registration")? + .ok_or("cannot resolve library registration")?; + let body = tcx.instance_mir(instance.def); + // Deliberately accept only the constant getter contract. Do not evaluate + // arbitrary target code or search a more complicated body for a JSON string. + if body.basic_blocks.len() != 1 { + return Err("registration getter must return one constant string".into()); + } + if !matches!( + body.basic_blocks.iter().next().unwrap().terminator().kind, + TerminatorKind::Return + ) { + return Err("registration getter must return one constant string".into()); + } + let mut text = None; + for statement in &body.basic_blocks.iter().next().unwrap().statements { + if let StatementKind::Assign(assign) = &statement.kind { + let (place, value) = &**assign; + if place.as_local() != Some(RETURN_PLACE) { + return Err("registration getter must return one constant string".into()); + } + let Rvalue::Use(Operand::Constant(constant), _) = value else { + return Err("registration getter must return one constant string".into()); + }; + if text.is_some() { + return Err("registration getter writes its result more than once".into()); + } + // The getter body still contains its generic parameters. Resolve + // associated constants using this concrete function-item instance. + let constant = rustc_middle::mir::ConstOperand { + const_: instance.instantiate_mir_and_normalize_erasing_regions( + tcx, + TypingEnv::fully_monomorphized(), + rustc_middle::ty::EarlyBinder::bind(tcx, constant.const_), + ), + ..**constant + }; + text = crate::collector::const_str_text(tcx, &constant); + } + } + let text = text.ok_or("registration getter must return one constant UTF-8 string")?; + let registration: Registration = + serde_json::from_str(&text).map_err(|e| format!("invalid library registration: {e}"))?; + if registration.version != 1 + || registration.provider.is_empty() + || !registration.executable.is_absolute() + { + return Err("library registration requires version 1, a provider name, and an absolute executable path".into()); + } + Ok(registration) +} + +fn decode_response( + bytes: &[u8], + expected_count: usize, + version: u32, +) -> Result<(BatchResponse, Vec), String> { + let magic = if version == 2 { + b"OXDLIB02" + } else { + b"OXDLIB01" + }; + if bytes.len() < 24 || &bytes[..8] != magic { + return Err("invalid or truncated library response frame".into()); + } + let header_len = u64::from_le_bytes(bytes[8..16].try_into().unwrap()); + let lto_len = u64::from_le_bytes(bytes[16..24].try_into().unwrap()); + if header_len > MAX_HEADER as u64 || lto_len == 0 || lto_len > MAX_LTO as u64 { + return Err("library response length is out of bounds".into()); + } + let header_end = 24 + header_len as usize; + if bytes.len() != header_end + lto_len as usize { + return Err("library response is truncated or has trailing bytes".into()); + } + let response: BatchResponse = serde_json::from_slice(&bytes[24..header_end]) + .map_err(|e| format!("invalid library response JSON: {e}"))?; + if response.version != version || response.specializations.len() != expected_count { + return Err( + "library response version or specialization count does not match request".into(), + ); + } + let mut symbols = HashSet::new(); + let mut identities = HashSet::new(); + for (index, spec) in response.specializations.iter().enumerate() { + if usize::try_from(spec.request_index).ok() != Some(index) + || !symbols.insert(&spec.symbol) + || !identities.insert(spec.identity) + || spec.symbol.is_empty() + || !spec + .symbol + .bytes() + .enumerate() + .all(|(i, b)| b == b'_' || b.is_ascii_alphabetic() || (i > 0 && b.is_ascii_digit())) + { + return Err( + "library response has an invalid index, symbol, or duplicate identity".into(), + ); + } + spec.cub_layout.checked()?; + let (size, _) = spec.scratch.checked()?; + if spec.scratch_free != (size == 0) { + return Err("library scratch-free flag disagrees with its layout".into()); + } + if !matches!( + spec.arguments.first(), + Some(Argument::ConstPointer(_) | Argument::MutPointer(_)) + ) || !matches!( + spec.arguments.get(1), + Some(Argument::ConstPointer(_) | Argument::MutPointer(_)) + ) { + return Err( + "library ABI requires memory and payload pointers before scratch slot 2".into(), + ); + } + for argument in &spec.arguments { + let (Argument::ConstPointer(scalar) + | Argument::MutPointer(scalar) + | Argument::Value(scalar)) = argument; + if !matches!( + scalar, + Scalar::Signed(8 | 16 | 32 | 64) + | Scalar::Unsigned(8 | 16 | 32 | 64) + | Scalar::Float(16 | 32 | 64) + ) { + return Err("library response contains an unsupported scalar width".into()); + } + } + } + Ok((response, bytes[header_end..].to_vec())) +} + +fn invoke( + executable: &PathBuf, + request: &BatchRequest<'_>, +) -> Result<(BatchResponse, Vec), String> { + let json = serde_json::to_vec(request).map_err(|e| e.to_string())?; + if json.len() > MAX_HEADER { + return Err("library request exceeds 1 MiB".into()); + } + let mut child = Command::new(executable) + .arg(format!("--oxide-library-request-v{}", request.version)) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .map_err(|e| { + format!( + "cannot execute native library provider {}: {e}", + executable.display() + ) + })?; + child + .stdin + .take() + .unwrap() + .write_all(&json) + .map_err(|e| format!("cannot send library request: {e}"))?; + let mut bytes = Vec::new(); + child + .stdout + .take() + .unwrap() + .take((24 + MAX_HEADER + MAX_LTO + 1) as u64) + .read_to_end(&mut bytes) + .map_err(|e| format!("cannot read library response: {e}"))?; + if bytes.len() > 24 + MAX_HEADER + MAX_LTO { + let _ = child.kill(); + } + let status = child + .wait() + .map_err(|e| format!("cannot wait for library provider: {e}"))?; + if !status.success() { + return Err(format!( + "native library provider failed with {status}; see its stderr diagnostics" + )); + } + decode_response(&bytes, request.requests.len(), request.version) +} + +fn extern_scalar(scalar: &Scalar) -> mir_importer::DeviceExternType { + use mir_importer::DeviceExternType as E; + match *scalar { + Scalar::Signed(bits) | Scalar::Unsigned(bits) => E::Integer(u32::from(bits)), + Scalar::Float(16) => E::Float16, + Scalar::Float(32) => E::Float32, + Scalar::Float(64) => E::Float64, + _ => unreachable!("validated scalar type"), + } +} + +fn extern_argument(argument: &Argument) -> mir_importer::DeviceExternType { + use mir_importer::DeviceExternType as E; + match argument { + Argument::Value(Scalar::Signed(bits)) if *bits < 32 => E::SignExtInteger(u32::from(*bits)), + Argument::Value(Scalar::Unsigned(bits)) if *bits < 32 => { + E::ZeroExtInteger(u32::from(*bits)) + } + Argument::Value(s) => extern_scalar(s), + Argument::ConstPointer(s) | Argument::MutPointer(s) => E::pointer_to(extern_scalar(s), 0), + } +} + +fn call_arguments(call: &LibraryCall<'_>) -> Result, String> { + let args = call.instance.args; + let first = if call.kind == MarkerKind::CheckedBlockStorage { + 4 + } else if call.kind.resource() { + 3 + } else { + 2 + }; + let TyKind::Tuple(uniform) = args.type_at(first).kind() else { + return Err("library arguments must be a tuple".into()); + }; + if !call.kind.checked() { + return uniform.iter().map(argument).collect(); + } + let TyKind::Tuple(payload) = args.type_at(first + 1).kind() else { + return Err("checked block payload must be a tuple".into()); + }; + let dtype = scalar(args.type_at(0))?; + let uniform = uniform + .iter() + .map(argument) + .collect::, _>>()?; + let payload = payload + .iter() + .map(argument) + .collect::, _>>()?; + if !matches!(uniform.first(), Some(Argument::ConstPointer(t) | Argument::MutPointer(t)) if *t == dtype) + || !uniform[1..].iter().all(|a| matches!(a, Argument::Value(_))) + || !matches!(payload.first(), Some(Argument::ConstPointer(t) | Argument::MutPointer(t)) if *t == dtype) + || !payload + .iter() + .all(|a| matches!(a, Argument::ConstPointer(_) | Argument::MutPointer(_))) + { + return Err("checked block call requires (memory pointer, scalar controls...) and (payload pointers...) with matching element types".into()); + } + Ok(uniform[..1] + .iter() + .chain(&payload) + .chain(&uniform[1..]) + .cloned() + .collect()) +} + +/// Source semantics authenticated without provider or participation facts. +fn authenticated_metadata<'tcx>( + tcx: TyCtxt<'tcx>, + call: &LibraryCall<'tcx>, +) -> Result { + use mir_importer::{ + LibraryAccess, LibraryNativeMetadata as Native, LibraryReserveLayout, + LibraryRoleAttr as Role, LibraryScalar, LibraryStorageArg, LibraryWitness, NativeSpec, + }; + let args = call.instance.args; + let positive = |index| { + let value = args + .const_at(index) + .try_to_target_usize(tcx) + .ok_or("library extent is not a concrete usize")?; + u32::try_from(value) + .ok() + .filter(|value| *value != 0) + .ok_or("library extent must fit nonzero u32") + }; + let scalar_attr = |ty| { + scalar(ty).map(|value| match value { + Scalar::Signed(bits) => LibraryScalar::Signed(bits), + Scalar::Unsigned(bits) => LibraryScalar::Unsigned(bits), + Scalar::Float(bits) => LibraryScalar::Float(bits), + }) + }; + let role = if call.kind.native() { + let spec = NativeSpec { + scalar: scalar_attr(args.type_at(0))?, + items: positive( + if matches!( + call.kind, + MarkerKind::CheckedBlockStorage | MarkerKind::Callback + ) { + 6 + } else if call.kind.resource() { + 5 + } else { + 4 + }, + )?, + }; + let access_witness = || -> Result<_, String> { + let TyKind::Ref(_, inner, _) = args.type_at(2).kind() else { + return Err("resource witness must be a reference".into()); + }; + let witness = if matches!(inner.kind(), TyKind::Slice(_)) { + LibraryWitness::ReadOnlySlice + } else { + let (_, capacity) = collective_carrier(tcx, *inner) + .ok_or("resource witness requires DisjointSlice>")?; + LibraryWitness::CollectiveCarrier(capacity) + }; + let access = if resource_write(call) { + LibraryAccess::Write + } else { + LibraryAccess::Read + }; + Ok((access, witness)) + }; + Role::Native(match call.kind { + MarkerKind::Raw => Native::Raw { + spec, + block_threads: positive(3)?, + }, + MarkerKind::Callback => Native::Callback { + spec, + block_threads: positive(5)?, + }, + MarkerKind::CheckedBlock => Native::CheckedBlock { spec }, + MarkerKind::CheckedBlockResource => { + let (access, witness) = access_witness()?; + Native::CheckedBlockResource { + spec, + access, + witness, + } + } + MarkerKind::CheckedBlockStorage => { + let (access, witness) = access_witness()?; + let ty = args.type_at(3); + let storage = if matches!(ty.kind(), TyKind::Tuple(fields) if fields.is_empty()) { + LibraryStorageArg::Implicit + } else { + LibraryStorageArg::Explicit(storage_borrow_policy(tcx, ty)?) + }; + Native::CheckedBlockStorage { + spec, + access, + witness, + storage, + } + } + _ => unreachable!("native kind checked above"), + }) + } else { + match call.kind { + MarkerKind::StorageNew => Role::StorageNew(storage_new_policy(tcx, call)?), + MarkerKind::StorageReconfigure => Role::StorageReconfigure { + old: storage_policy(tcx, args.type_at(0))?, + new: storage_policy(tcx, args.type_at(1))?, + }, + MarkerKind::StorageReserve => { + let (bytes, alignment) = storage_reserve_layout(tcx, call)?; + Role::StorageReserve { + policy: storage_policy(tcx, args.type_at(1))?, + layout: LibraryReserveLayout { + scalar: scalar_attr(args.type_at(0))?, + count: positive(2)?, + bytes, + alignment, + }, + } + } + MarkerKind::StorageRelease => Role::StorageRelease, + _ => unreachable!("all marker kinds covered"), + } + }; + Ok(mir_importer::LibraryCallMetadata { + marker: tcx.symbol_name(call.instance).name.to_string(), + role, + }) +} + +/// Preserve original identity/type distinctions that lowering can erase. This +/// collects facts only; callback admission is applied by the MIR consumer. +fn callback_sources<'tcx>( + tcx: TyCtxt<'tcx>, + collection: &CollectionResult<'tcx>, + rooted: &HashSet>, + names: &HashMap, String>, +) -> Result { + use mir_importer::{ + LibraryCallbackCall as Call, LibraryCallbackDefinition as Definition, + LibraryCallbackDispatch as Dispatch, LibraryCallbackFunction, LibraryCallbackSources, + }; + let definition = |def: DefId| { + let name = tcx.crate_name(def.krate); + if name.as_str() == "cuda_device" + || crate::generated_intrinsics::is_generated_intrinsic_crate(name.as_str()) + { + Definition::Cuda + } else if tcx.is_foreign_item(def) { + Definition::Foreign + } else if tcx.intrinsic(def).is_some() { + Definition::Intrinsic + } else { + Definition::Rust + } + }; + let identity = |instance| tcx.symbol_name(instance).name.to_string(); + let mut sources = LibraryCallbackSources::default(); + let mut instances = HashSet::new(); + for call in &collection.library_calls { + if call.kind != MarkerKind::Callback || !rooted.contains(&call.site.caller) { + continue; + } + let thunk = callback_instance(tcx, call.instance)?; + sources.roots.insert( + mir_importer::LibraryCallSite { + caller: names + .get(&call.site.caller) + .ok_or("callback caller was not collected")? + .clone(), + block: call.site.block.index(), + }, + identity(thunk), + ); + instances.extend(reachable_instances(collection, thunk)); + } + for instance in instances { + let def = instance.def_id(); + let dispatch = match instance.def { + InstanceKind::Virtual(..) => Dispatch::Indirect, + InstanceKind::Shim(ShimKind::FnPtr(_, receiver)) => { + if matches!(receiver.kind(), TyKind::FnDef(..)) { + Dispatch::FunctionItemShim + } else { + Dispatch::Indirect + } + } + InstanceKind::Shim(ShimKind::ClosureOnce { .. }) => Dispatch::ClosureOnceShim, + _ => Dispatch::Direct, + }; + let mut source = LibraryCallbackFunction { + diagnostic: tcx.def_path_str(def), + export: names.get(&instance).cloned(), + definition: definition(def), + dispatch, + has_body: tcx.is_mir_available(def), + cuda_local_types: Vec::new(), + calls: Vec::new(), + callees: collection + .call_edges + .iter() + .filter(|edge| edge.caller == instance) + .map(|edge| identity(edge.callee)) + .collect(), + }; + if source.has_body + && source.definition == Definition::Rust + && dispatch != Dispatch::Indirect + { + let body = tcx.instance_mir(instance.def); + let normalize = |ty| { + instance.instantiate_mir_and_normalize_erasing_regions( + tcx, + TypingEnv::fully_monomorphized(), + rustc_middle::ty::EarlyBinder::bind(tcx, ty), + ) + }; + // All original locals, including unused declarations, carry the + // nominal restriction; promoted prepared values cannot recover it. + for local in &body.local_decls { + for argument in normalize(local.ty).walk() { + if let Some(ty) = argument.as_type() + && let TyKind::Adt(adt, _) = ty.kind() + && definition(adt.did()) == Definition::Cuda + { + source.cuda_local_types.push(ty.to_string()); + } + } + } + let reachable = crate::collector::device_mono_reachable_as_bitset(body, tcx, instance); + for (index, block) in body.basic_blocks.iter_enumerated() { + if !reachable.contains(index) { + continue; + } + match &block.terminator().kind { + TerminatorKind::InlineAsm { .. } => source.calls.push(Call::InlineAssembly), + TerminatorKind::Call { func, .. } => { + let ty = normalize(func.ty(body, tcx)); + source.calls.push(match ty.kind() { + TyKind::FnDef(callee, _) => Call::Direct { + definition: definition(*callee), + diagnostic: tcx.def_path_str(*callee), + }, + _ => Call::Indirect, + }); + } + _ => {} + } + } + } + sources.functions.insert(identity(instance), source); + } + Ok(sources) +} + +/// Rust identities stay at the import boundary; prepared MIR owns the proof. +#[derive(Default)] +pub(crate) struct AuthenticatedLibraries<'tcx> { + pub callbacks: mir_importer::LibraryCallbackSources, + pub semantics: BTreeMap, + pub deferred: HashSet>, + pub checked_ancestors: HashSet>, + pub selected_call_targets: + BTreeMap, + pub rooted_functions: BTreeMap>, + pub call_targets: BTreeMap>, +} + +pub(crate) fn authenticate<'tcx>( + tcx: TyCtxt<'tcx>, + collection: &CollectionResult<'tcx>, +) -> Result, String> { + let mut result = AuthenticatedLibraries::default(); + let definitions = marker_definitions(tcx); + let mut sites = HashSet::new(); + for call in &collection.library_calls { + if definitions.get(&call.instance.def_id()) != Some(&call.kind) { + return Err("library marker is not a genuine registered definition".into()); + } + validate_storage(tcx, call).unwrap_or_else(|error| tcx.dcx().span_fatal(call.span, error)); + if call.kind.native() { + call_arguments(call).unwrap_or_else(|error| tcx.dcx().span_fatal(call.span, error)); + } + if call.kind.resource() { + validate_resource(tcx, call) + .unwrap_or_else(|error| tcx.dcx().span_fatal(call.span, error)); + } + if !sites.insert((call.site.caller, call.site.block)) { + return Err("duplicate authenticated library source site".into()); + } + } + let names: HashMap<_, _> = collection + .functions + .iter() + .map(|function| (function.instance, function.export_name.clone())) + .collect(); + let mut rooted = HashSet::new(); + for root in collection + .functions + .iter() + .filter(|function| function.is_kernel) + { + let reached = reachable_instances(collection, root.instance); + result.rooted_functions.insert( + root.export_name.clone(), + reached + .iter() + .filter_map(|instance| names.get(instance).cloned()) + .collect(), + ); + rooted.extend(reached); + } + result.checked_ancestors = checked_helper_ancestors(collection); + result.deferred = result.checked_ancestors.clone(); + result + .deferred + .retain(|instance| !rooted.contains(instance)); + // A standalone checked helper may still have unevaluated generic extents. + // Its consumer supplies the concrete root before typed roles are required. + for call in &collection.library_calls { + if !rooted.contains(&call.site.caller) { + continue; + } + let caller = names + .get(&call.site.caller) + .ok_or("authenticated library caller was not collected")? + .clone(); + let metadata = authenticated_metadata(tcx, call) + .unwrap_or_else(|error| tcx.dcx().span_fatal(call.span, error)); + result.semantics.insert( + mir_importer::LibraryCallSite { + caller, + block: call.site.block.index(), + }, + metadata, + ); + } + result.callbacks = callback_sources(tcx, collection, &rooted, &names)?; + let mut selected_sites = HashSet::new(); + for edge in &collection.call_edges { + if result.deferred.contains(&edge.caller) { + continue; + } + let first = selected_sites.insert((edge.caller, edge.block)); + let target = names.get(&edge.callee).or_else(|| { + collection + .device_externs + .iter() + .find(|declaration| declaration.def_id == edge.callee.def_id()) + .map(|declaration| &declaration.export_name) + }); + if let (Some(caller), Some(target)) = (names.get(&edge.caller), target) { + let site = mir_importer::LibraryCallSite { + caller: caller.clone(), + block: edge.block.index(), + }; + if first { + result.selected_call_targets.insert( + site.clone(), + mir_importer::LibraryCallTargetAttr { + kind: match edge.kind { + EdgeKind::Direct => mir_importer::LibraryCallKind::Direct, + EdgeKind::Callable => mir_importer::LibraryCallKind::Callable, + EdgeKind::Drop => mir_importer::LibraryCallKind::Drop, + }, + export: target.clone(), + marker: tcx.symbol_name(edge.callee).name.to_string(), + }, + ); + } + result + .call_targets + .entry(site) + .or_default() + .insert(target.clone()); + } + } + for helper in &result.deferred { + let reached = reachable_instances(collection, *helper); + if let Some(call) = collection + .library_calls + .iter() + .find(|call| !call.kind.checked() && reached.contains(&call.site.caller)) + { + tcx.dcx().span_fatal( + call.span, + "raw and checked library calls cannot be mixed in one checked helper path", + ); + } + } + Ok(result) +} + +/// Resolve original-source requests only after the prepared MIR proof. +pub(crate) fn finalize_provider<'tcx>( + tcx: TyCtxt<'tcx>, + collection: &CollectionResult<'tcx>, + checked_shapes: &BTreeMap, + callback_symbols: &BTreeMap, +) -> Result { + let mut result = ProviderLibraries::default(); + if collection.library_calls.is_empty() { + return Ok(result); + } + let target = std::env::var("CUDA_OXIDE_TARGET") + .map_err(|_| "library calls require an explicit CUDA target")?; + let target = target + .strip_prefix("sm_") + .map(|v| format!("compute_{v}")) + .unwrap_or(target); + let cuda_root = std::env::var_os("CUDA_HOME") + .or_else(|| std::env::var_os("CUDA_PATH")) + .map(PathBuf::from) + .ok_or("library calls require CUDA_HOME or CUDA_PATH")?; + type RootedCall<'a, 'tcx> = (&'a LibraryCall<'tcx>, &'a str); + type ProviderBatch<'a, 'tcx> = Vec<(Request, Vec>)>; + let mut batches: BTreeMap<(String, PathBuf, u32), ProviderBatch<'_, 'tcx>> = BTreeMap::new(); + let affected = checked_helper_ancestors(collection); + for root in collection.functions.iter().filter(|f| f.is_kernel) { + let reached = reachable_instances(collection, root.instance); + let calls: Vec<_> = collection + .library_calls + .iter() + .filter(|c| reached.contains(&c.site.caller)) + .collect(); + if calls.is_empty() { + continue; + } + let storage_root = calls.iter().any(|call| call.kind.storage()); + for instance in &reached { + if !storage_root + && tcx.crate_name(instance.def_id().krate).as_str() == "cuda_device" + && tcx + .def_path_str(instance.def_id()) + .contains("DynamicSharedArray") + { + return Err(format!( + "kernel {} mixes library scratch with a native dynamic shared view", + root.export_name + )); + } + } + if checked_shapes.contains_key(&root.export_name) { + for function in &collection.functions { + if !function.is_kernel + && reached.contains(&function.instance) + && affected.contains(&function.instance) + && tcx + .codegen_fn_attrs(function.instance.def_id()) + .contains_extern_indicator() + { + tcx.dcx().struct_span_fatal(tcx.def_span(function.instance.def_id()), + "fixed external symbols are not supported on checked collective helper paths") + .with_note(format!("helper is reachable from kernel {}", root.export_name)) + .with_help("remove the helper's #[no_mangle] or #[export_name] attribute").emit(); + } + } + } + for call in calls.into_iter().filter(|call| call.kind.native()) { + let args = call.instance.args; + let report = |error: String| -> ! { + tcx.dcx() + .span_fatal(call.span, format!("native device library: {error}")) + }; + let registration = + read_registration(tcx, args.type_at(1)).unwrap_or_else(|error| report(error)); + let positive = |index| { + let value = args + .const_at(index) + .try_to_target_usize(tcx) + .ok_or("library extent is not a concrete usize")?; + u32::try_from(value) + .ok() + .filter(|v| *v != 0) + .ok_or("library extent must fit nonzero u32") + }; + let block = if call.kind.checked() { + checked_shapes + .get(&root.export_name) + .copied() + .ok_or_else(|| { + format!( + "checked library root {} has no admitted block shape", + root.export_name + ) + })? + } else { + [ + positive(if call.kind == MarkerKind::Callback { + 5 + } else { + 3 + }) + .map_err(str::to_owned) + .unwrap_or_else(|error| report(error)), + 1, + 1, + ] + }; + let request = Request { + operation: registration.operation, + algorithm: registration.algorithm, + dtype: scalar(args.type_at(0)).unwrap_or_else(|error| report(error)), + block, + items_per_thread: positive( + if matches!( + call.kind, + MarkerKind::CheckedBlockStorage | MarkerKind::Callback + ) { + 6 + } else if call.kind.resource() { + 5 + } else { + 4 + }, + ) + .map_err(str::to_owned) + .unwrap_or_else(|error| report(error)), + callback: if call.kind == MarkerKind::Callback { + let thunk = + callback_instance(tcx, call.instance).unwrap_or_else(|error| report(error)); + let identity = tcx.symbol_name(thunk).name.to_string(); + let symbol = callback_symbols.get(&identity).unwrap_or_else(|| { + report("callback thunk has no admitted emitted symbol".into()) + }); + Some(Callback { + symbol: symbol.clone(), + arguments: vec![ + Argument::ConstPointer(Scalar::Unsigned(8)), + Argument::Value(Scalar::Signed(32)), + Argument::Value(Scalar::Signed(32)), + ], + result: Scalar::Signed(32), + }) + } else { + None + }, + }; + let version = if request.callback.is_some() { 2 } else { 1 }; + let batch = batches + .entry((registration.provider, registration.executable, version)) + .or_default(); + if let Some((_, calls)) = batch.iter_mut().find(|(existing, _)| *existing == request) { + calls.push((call, &root.export_name)); + } else { + batch.push((request, vec![(call, &root.export_name)])); + } + } + } + let mut all_symbols = HashSet::new(); + for ((provider, executable, version), batch) in batches { + let requests: Vec<_> = batch.iter().map(|(request, _)| request.clone()).collect(); + let batch_span = batch[0].1[0].0.span; + let (response, lto) = invoke( + &executable, + &BatchRequest { + version, + target: &target, + cuda_root: cuda_root.clone(), + requests: &requests, + }, + ) + .unwrap_or_else(|error| { + tcx.dcx() + .span_fatal(batch_span, format!("native device library: {error}")) + }); + let identity: String = response + .batch_identity + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + result + .inputs + .push((format!("library-{provider}-{identity}.ltoir"), lto)); + for ((request, calls), spec) in batch.into_iter().zip(response.specializations) { + if !all_symbols.insert(spec.symbol.clone()) { + return Err(format!( + "different provider batches returned the same symbol {}", + spec.symbol + )); + } + let (scratch_bytes, scratch_alignment) = spec.scratch.checked()?; + let mut params: Vec<_> = spec.arguments.iter().map(extern_argument).collect(); + params.insert( + 2, + mir_importer::DeviceExternType::pointer_to( + mir_importer::DeviceExternType::Integer(8), + 0, + ), + ); + if request.callback.is_some() { + params.push(mir_importer::DeviceExternType::pointer_to( + mir_importer::DeviceExternType::Integer(8), + 0, + )); + } + result.declarations.push(mir_importer::DeviceExternDecl { + export_name: spec.symbol.clone(), + param_types: params, + return_type: mir_importer::DeviceExternType::Void, + attrs: mir_importer::DeviceExternAttrs { + is_convergent: true, + is_pure: false, + is_readonly: false, + }, + }); + for (call, root) in calls { + if call.kind == MarkerKind::CheckedBlockResource && !spec.scratch_free { + tcx.dcx().span_fatal( + call.span, + "resource-checked block calls currently require a scratch-free provider", + ); + } + let actual = call_arguments(call) + .unwrap_or_else(|error| tcx.dcx().span_fatal(call.span, error)); + if actual != spec.arguments { + tcx.dcx().span_fatal(call.span, format!( + "library runtime tuple for {} disagrees with provider ABI: actual {actual:?}, expected {:?}", + spec.symbol, spec.arguments + )); + } + let _ = spec.mutates_payload; + let plan = mir_importer::LibraryCallPlan { + operation: mir_importer::LibraryOperation::Native, + storage_witness: call.kind == MarkerKind::CheckedBlockStorage, + scratch_offset: 0, + planned_root: None, + barrier_before: false, + barrier_after: false, + symbol: spec.symbol.clone(), + marker: tcx.symbol_name(call.instance).name.to_string(), + checked_block: call.kind.checked(), + resource_witness: call.kind.resource(), + callback: call.kind == MarkerKind::Callback, + scratch_bytes, + scratch_alignment, + }; + let caller = collection + .functions + .iter() + .find(|function| function.instance == call.site.caller) + .ok_or("collected library caller is missing")? + .export_name + .clone(); + let site = mir_importer::LibraryCallSite { + caller, + block: call.site.block.index(), + }; + let plans = result.native_plans.entry(root.to_string()).or_default(); + if let Some(previous) = plans.insert(site, plan.clone()) + && previous != plan + { + return Err("library call site has conflicting provider specializations".into()); + } + let facts = KernelLaunchFacts { + exact_block: Some((request.block[0], request.block[1], request.block[2])), + dynamic_shared_bytes: scratch_bytes, + dynamic_shared_alignment: scratch_alignment, + }; + if let Some(current) = result.roots.get_mut(root) { + if current.exact_block != facts.exact_block { + tcx.dcx().span_fatal( + call.span, + format!("kernel {root} has conflicting exact library block shapes"), + ); + } + current.dynamic_shared_bytes = current.dynamic_shared_bytes.max(scratch_bytes); + current.dynamic_shared_alignment = + current.dynamic_shared_alignment.max(scratch_alignment); + } else { + result.roots.insert(root.to_string(), facts); + } + } + } + } + Ok(result) +} + +/// Only bodies that can reach a checked marker need root-specific emission. +fn checked_helper_ancestors<'tcx>(collection: &CollectionResult<'tcx>) -> HashSet> { + let mut affected = HashSet::new(); + let mut pending: Vec<_> = collection + .library_calls + .iter() + .filter(|call| call.kind.checked()) + .map(|call| call.site.caller) + .collect(); + while let Some(instance) = pending.pop() { + if affected.insert(instance) { + pending.extend( + collection + .call_edges + .iter() + .filter(|edge| edge.callee == instance) + .map(|edge| edge.caller), + ); + } + } + affected +} + +pub(crate) fn reachable_instances<'tcx>( + collection: &CollectionResult<'tcx>, + root: Instance<'tcx>, +) -> HashSet> { + let mut reached = HashSet::new(); + let mut pending = vec![root]; + while let Some(instance) = pending.pop() { + if reached.insert(instance) { + pending.extend( + collection + .call_edges + .iter() + .filter(|site| site.caller == instance) + .map(|site| site.callee), + ); + } + } + reached +} + +#[cfg(test)] +mod tests { + use super::*; + + fn response() -> serde_json::Value { + let scalar = serde_json::json!({"kind": "signed", "bits": 32}); + serde_json::json!({ + "version": 1, "batch_identity": vec![0; 32], + "specializations": [{ + "request_index": 0, "identity": vec![1; 32], "symbol": "library_i32", + "arguments": [ + {"kind": "const_pointer", "scalar": scalar}, + {"kind": "mut_pointer", "scalar": scalar} + ], + "cub_layout": {"size": 1, "alignment": 1}, + "scratch": {"size": 0, "alignment": 1}, + "scratch_free": true, "mutates_payload": true + }] + }) + } + + fn frame(response: &serde_json::Value) -> Vec { + let json = serde_json::to_vec(response).unwrap(); + let mut bytes = b"OXDLIB01".to_vec(); + bytes.extend((json.len() as u64).to_le_bytes()); + bytes.extend(3_u64.to_le_bytes()); + bytes.extend(json); + bytes.extend([1, 2, 3]); + bytes + } + + #[test] + fn response_frame_retains_bytes_and_rejects_truncation_or_trailing_data() { + let bytes = frame(&response()); + assert_eq!(decode_response(&bytes, 1, 1).unwrap().1, [1, 2, 3]); + for end in [0, 8, 16, 23, bytes.len() - 1] { + assert!(decode_response(&bytes[..end], 1, 1).is_err()); + } + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(decode_response(&trailing, 1, 1).is_err()); + let mut empty_lto = bytes; + empty_lto[16..24].copy_from_slice(&0_u64.to_le_bytes()); + assert!(decode_response(&empty_lto, 1, 1).is_err()); + } + + #[test] + fn response_rejects_inconsistent_identity_layout_and_abi() { + for (field, value) in [ + ("request_index", serde_json::json!(1)), + ("symbol", serde_json::json!("not a symbol")), + ("scratch_free", serde_json::json!(false)), + ("scratch", serde_json::json!({"size": 0, "alignment": 3})), + ( + "cub_layout", + serde_json::json!({"size": 4294967296_u64, "alignment": 1}), + ), + ("arguments", serde_json::json!([])), + ( + "arguments", + serde_json::json!([ + {"kind": "const_pointer", "scalar": {"kind": "signed", "bits": 32}}, + {"kind": "value", "scalar": {"kind": "signed", "bits": 32}} + ]), + ), + ] { + let mut value_response = response(); + value_response["specializations"][0][field] = value; + assert!( + decode_response(&frame(&value_response), 1, 1).is_err(), + "{field}" + ); + } + let mut duplicate = response(); + let mut second = duplicate["specializations"][0].clone(); + second["request_index"] = 1.into(); + duplicate["specializations"] + .as_array_mut() + .unwrap() + .push(second); + assert!(decode_response(&frame(&duplicate), 2, 1).is_err()); + assert!(decode_response(&frame(&response()), 2, 1).is_err()); + let mut version = response(); + version["version"] = 2.into(); + assert!(decode_response(&frame(&version), 1, 1).is_err()); + } + #[test] + fn callback_batch_requires_v2_frame_and_header() { + let mut value = response(); + value["version"] = 2.into(); + let mut bytes = frame(&value); + assert!(decode_response(&bytes, 1, 2).is_err()); + bytes[..8].copy_from_slice(b"OXDLIB02"); + assert!(decode_response(&bytes, 1, 2).is_ok()); + assert!(decode_response(&bytes, 1, 1).is_err()); + value["version"] = 1.into(); + let mut bytes = frame(&value); + bytes[..8].copy_from_slice(b"OXDLIB02"); + assert!(decode_response(&bytes, 1, 2).is_err()); + } + + #[test] + fn response_preserves_const_payload_pointer_mutability() { + let mut value = response(); + value["specializations"][0]["arguments"][1]["kind"] = "const_pointer".into(); + value["specializations"][0]["mutates_payload"] = false.into(); + let (decoded, _) = decode_response(&frame(&value), 1, 1).unwrap(); + assert_eq!( + decoded.specializations[0].arguments[1], + Argument::ConstPointer(Scalar::Signed(32)) + ); + assert_ne!( + decoded.specializations[0].arguments, + decode_response(&frame(&response()), 1, 1) + .unwrap() + .0 + .specializations[0] + .arguments + ); + } +} diff --git a/cuda-oxide/crates/rustc-codegen-cuda/src/materialize.rs b/cuda-oxide/crates/rustc-codegen-cuda/src/materialize.rs index 387c4ce4f6..8519ac61c7 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/src/materialize.rs +++ b/cuda-oxide/crates/rustc-codegen-cuda/src/materialize.rs @@ -27,10 +27,18 @@ pub(crate) const MATERIALIZER_HANDSHAKE_ENV: &str = reserved_oxide_symbols::MATERIALIZER_HANDSHAKE_ENV; pub(crate) const CODEGEN_FINGERPRINT_ENV: &str = reserved_oxide_symbols::CODEGEN_FINGERPRINT_ENV; -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Debug)] pub(crate) struct MaterializationRequest { expected_provenance: [u8; 32], tool_identity_handshake: cuda_artifact_finalizer::MaterializerHandshakeV1, + external_inputs: Vec<(String, Vec)>, + library_inputs: Vec<(String, Vec)>, +} + +impl MaterializationRequest { + pub(crate) fn add_library_inputs(&mut self, inputs: Vec<(String, Vec)>) { + self.library_inputs.extend(inputs); + } } /// Cubin bytes plus ptxas resource diagnostics from the final link. @@ -92,15 +100,13 @@ pub(crate) enum MaterializeError { ProvenanceMismatch { expected: String, actual: String }, #[error( - "build-time cubin materialization does not yet support generic #[cuda_module] loading because it merges PTX bundles across crates at run time" - )] - RequiresPtxBundleMerge, - - #[error( - "build-time cubin materialization does not yet support #[device] extern declarations because their ordered external link inputs are not available to the backend" + "build-time cubin materialization of #[device] extern declarations requires their definitions; supply ordered --link-ltoir inputs" )] HasDeviceExterns, + #[error("invalid cargo-oxide external LTO-IR input list: {reason}")] + InvalidExternalInputs { reason: String }, + #[error( "build-time cubin materialization requires an NVVM IR or LTOIR artifact, but codegen produced PTX; use cargo-oxide so materialization can force NVVM IR emission" )] @@ -147,12 +153,38 @@ pub(crate) fn request_from_env() -> Result, Mater })?; validate_tool_identity_handshake(expected_provenance, &handshake)?; validate_codegen_fingerprint()?; + let external_inputs = external_inputs_from_env()?; Ok(Some(MaterializationRequest { expected_provenance, tool_identity_handshake: handshake, + external_inputs, + library_inputs: Vec::new(), })) } +fn external_inputs_from_env() -> Result)>, MaterializeError> { + let value = match std::env::var(reserved_oxide_symbols::EXTERNAL_LTOIR_ENV) { + Ok(value) => value, + Err(std::env::VarError::NotPresent) => return Ok(Vec::new()), + Err(error) => { + return Err(MaterializeError::InvalidExternalInputs { + reason: error.to_string(), + }); + } + }; + let files: Vec = + serde_json::from_str(&value).map_err(|error| MaterializeError::InvalidExternalInputs { + reason: error.to_string(), + })?; + files + .into_iter() + .map(|file| { + let bytes = file.read_verified()?; + Ok((file.name()?.to_owned(), bytes)) + }) + .collect() +} + fn validate_tool_identity_handshake( expected_provenance: [u8; 32], handshake: &cuda_artifact_finalizer::MaterializerHandshakeV1, @@ -174,22 +206,43 @@ fn validate_tool_identity_handshake( /// Reject artifact-loading models the finalizer cannot reproduce, before any /// CUDA compiler library is discovered or loaded. pub(crate) fn validate_collection( - request: Option, + request: Option<&MaterializationRequest>, has_device_externs: bool, - requires_ptx_bundle_merge: bool, ) -> Result<(), MaterializeError> { - if request.is_none() { + let Some(request) = request else { return Ok(()); - } - if requires_ptx_bundle_merge { - return Err(MaterializeError::RequiresPtxBundleMerge); - } - if has_device_externs { + }; + if has_device_externs && request.external_inputs.is_empty() && request.library_inputs.is_empty() + { return Err(MaterializeError::HasDeviceExterns); } Ok(()) } +/// Use the actual rustc crate name and configured output directory, including +/// for Cargo passthrough builds whose working directory differs from the CLI's. +pub(crate) fn validate_output_paths( + request: Option<&MaterializationRequest>, + output_dir: &std::path::Path, + output_name: &str, +) -> Result<(), MaterializeError> { + let Some(request) = request else { + return Ok(()); + }; + let mut outputs = + mir_importer::pipeline::stale_compilation_artifact_paths(output_dir, output_name); + outputs.push(output_dir.join(format!( + "{output_name}.{}", + oxide_tile_export::DESCRIPTIONS_SUFFIX + ))); + for (input, _) in &request.external_inputs { + for output in &outputs { + cuda_artifact_finalizer::check_ltoir_input_output(std::path::Path::new(input), output)?; + } + } + Ok(()) +} + pub(crate) fn nvvm_ir_to_cubin( request: MaterializationRequest, nvvm_ir: &[u8], @@ -199,8 +252,13 @@ pub(crate) fn nvvm_ir_to_cubin( debug_policy: DebugPolicy, ) -> Result { let options = options(target, allow_fma_contraction, debug_policy)?; - let finalizer = checked_finalizer(request)?; - let report = finalizer.materialize_nvvm_ir_with_report(module_name, nvvm_ir, &options)?; + let finalizer = checked_finalizer(&request)?; + let ltoir = finalizer + .compiler() + .compile_nvvm_ir_to_ltoir(module_name, nvvm_ir, &options)?; + let name = format!("{module_name}.ltoir"); + let inputs = link_inputs(&request, &name, <oir); + let report = finalizer.link_ltoir_with_report(&inputs, &options, FinalizerOutput::Cubin)?; Ok(MaterializedCubin { bytes: report.image, resource_usage: report.resource_usage, @@ -216,18 +274,31 @@ pub(crate) fn ltoir_to_cubin( debug_policy: DebugPolicy, ) -> Result { let options = options(target, allow_fma_contraction, debug_policy)?; - let finalizer = checked_finalizer(request)?; - let report = finalizer.link_ltoir_with_report( - &[NamedInput::new(module_name, ltoir)], - &options, - FinalizerOutput::Cubin, - )?; + let finalizer = checked_finalizer(&request)?; + let inputs = link_inputs(&request, module_name, ltoir); + let report = finalizer.link_ltoir_with_report(&inputs, &options, FinalizerOutput::Cubin)?; Ok(MaterializedCubin { bytes: report.image, resource_usage: report.resource_usage, }) } +fn link_inputs<'a>( + request: &'a MaterializationRequest, + name: &'a str, + bytes: &'a [u8], +) -> Vec> { + std::iter::once(NamedInput::new(name, bytes)) + .chain( + request + .external_inputs + .iter() + .chain(&request.library_inputs) + .map(|(name, bytes)| NamedInput::new(name, bytes)), + ) + .collect() +} + fn options( target: &str, allow_fma_contraction: bool, @@ -239,7 +310,7 @@ fn options( .with_debug_policy(debug_policy)) } -fn checked_finalizer(request: MaterializationRequest) -> Result { +fn checked_finalizer(request: &MaterializationRequest) -> Result { let finalizer = Finalizer::discover_with_handshake(&request.tool_identity_handshake)?; let actual = finalizer .provenance_digest() @@ -385,21 +456,58 @@ mod tests { } #[test] - fn unsupported_collection_models_fail_without_tools() { + fn output_overlap_uses_the_actual_crate_and_output_directory() { + let root = std::env::temp_dir().join(format!( + "oxide-materialize-overlap-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&root).unwrap(); + let input = root.join("custom_target.ll"); + std::fs::write(&input, b"external input").unwrap(); let handshake = test_handshake(); - let request = Some(MaterializationRequest { + let request = MaterializationRequest { expected_provenance: handshake.provenance_sha256, tool_identity_handshake: handshake, - }); + library_inputs: Vec::new(), + external_inputs: vec![( + input.to_str().unwrap().to_owned(), + b"external input".to_vec(), + )], + }; + assert!(validate_output_paths(Some(&request), &root, "package_name").is_ok()); assert!(matches!( - validate_collection(request, false, true), - Err(MaterializeError::RequiresPtxBundleMerge) + validate_output_paths(Some(&request), &root, "custom_target"), + Err(MaterializeError::Finalizer( + FinalizerError::LtoirInputOutputOverlap { .. } + )) )); + assert_eq!(std::fs::read(&input).unwrap(), b"external input"); + std::fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn unsupported_collection_models_fail_without_tools() { + let handshake = test_handshake(); + let mut request = MaterializationRequest { + expected_provenance: handshake.provenance_sha256, + tool_identity_handshake: handshake, + library_inputs: Vec::new(), + external_inputs: Vec::new(), + }; + assert!(validate_collection(Some(&request), false).is_ok()); assert!(matches!( - validate_collection(request, true, false), + validate_collection(Some(&request), true), Err(MaterializeError::HasDeviceExterns) )); - assert!(validate_collection(None, true, true).is_ok()); + assert!(validate_collection(None, true).is_ok()); + request + .external_inputs + .push(("definition.ltoir".into(), vec![1])); + assert!(validate_collection(Some(&request), true).is_ok()); } #[test] diff --git a/cuda-oxide/crates/rustc-codegen-cuda/tests/fixtures/primitives_api_compile/Cargo.lock b/cuda-oxide/crates/rustc-codegen-cuda/tests/fixtures/primitives_api_compile/Cargo.lock index 939d7756c3..0c593e8cd0 100644 --- a/cuda-oxide/crates/rustc-codegen-cuda/tests/fixtures/primitives_api_compile/Cargo.lock +++ b/cuda-oxide/crates/rustc-codegen-cuda/tests/fixtures/primitives_api_compile/Cargo.lock @@ -196,6 +196,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn 2.0.119", ] @@ -417,8 +419,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/cuda-oxide-book/compiler/native-library-calls.md b/cuda-oxide/cuda-oxide-book/compiler/native-library-calls.md new file mode 100644 index 0000000000..67de90c621 --- /dev/null +++ b/cuda-oxide/cuda-oxide-book/compiler/native-library-calls.md @@ -0,0 +1,316 @@ +# Native library calls + +`cuda_device::library::call_v1` is an unsafe integration point for library +authors. It specializes native device implementations from concrete Rust +calls, then supplies their LTO-IR to ordinary CUBIN materialization. The +compiler owns discovery and linking; the library owns its operation semantics, +native code, ABI and scratch requirements. + +Use `cargo oxide build`, `run` or `test` with `--materialize-cubin`. The compiler +recognizes the actual marker definition, including calls reached through +generic dependency helpers. A same-named user function remains ordinary Rust. +The marker must be called directly; pass a wrapper helper rather than the +marker itself as a callable value. + +The `native_library` example is an executable provider with no CUB dependency. +Its dependency helper is instantiated at one and four items per thread, and +two concrete kernel roots run through the embedded module loader: + +```sh +cargo oxide run native_library --arch sm_120 --materialize-cubin +``` + +## Registration and call ABI + +The marker's generic parameters are `T`, `Registration`, `Args`, +`const BLOCK_THREADS: usize` and `const ITEMS_PER_THREAD: usize`. +Both extents must fit positive `u32`. It accepts a registration function item +and one tuple of scalar/raw-pointer arguments; it returns unit. + +The registration getter returns one constant UTF-8 JSON string. Its MIR must +contain one return block and one assignment of that constant to its result. +Closures, function pointers and computed getters are rejected. The compiler +reads the constant without running Rust device code. A Cargo build script can +embed the string with `cargo::rustc-env` and serve as the already-built native +provider. Read it through the registration macro so Cargo tracks native inputs: + +```rust +fn registration() -> &'static str { + cuda_device::library::registration!("MY_LIBRARY_REGISTRATION") +} +``` + +Generic getters may return an associated constant selected by concrete type +and const arguments. The compiler applies those arguments before reading the +string; the one-block constant-return restriction still applies. The raw calls +in `native_library` exercise this with one and four items per thread. + +The macro requires nonempty provider, operation and algorithm names and expands +to the same constant string. The provider executable must +recognize its protocol arguments before its ordinary Cargo build-script behavior. + +Registration fields are: + +| Field | Meaning | +| --- | --- | +| `version` | Integer `1` | +| `provider` | Nonempty library identifier | +| `executable` | Absolute path to the built host provider | +| `operation`, `algorithm` | Opaque recipe names interpreted by that provider | + +The runtime tuple starts with memory and payload pointers, each const or mutable. +Remaining fields are supported fixed-width numeric values or +raw pointers. The provider declares the exact ordered types and pointer +mutability; the compiler compares them with the Rust tuple. It inserts a +generic `unsigned char*` scratch parameter at native argument index two. +Small signed and unsigned integer values retain their native extension rules. +The native function returns `void`. +An operation that only reads its payload can accept const pointers derived +from shared Rust borrows. The native provider must honor that declaration. + +## Checked block helpers + +`cuda_device::library::checked_block_call_v1` derives its block shape from the +kernel's exact launch contract and checks supported participation and uniform +memory/control arguments. It remains unsafe for the memory, native ABI and +scratch reuse obligations described below. + +The same generic helper and marker site may serve kernels with different exact +block shapes. The compiler emits helper bodies for each root and routes their +calls to the corresponding native specialization and scratch plan, including +through intermediate helpers. No block-size type argument or inlining promise +is required. A dependency may instantiate a marked generic helper before a +kernel supplies its contract; the compiler retains its MIR and defers that +checked path's device emission until a concrete root is available. Independent +standalone device functions still emit normally. + +Device statics keep their original shared identity. Helpers with +fixed external symbols (`#[no_mangle]` or `#[export_name]`) are unsupported on +checked helper paths, including single-root paths. Remove those attributes to +allow root-specific naming. Unsupported indirect checked calls retain their +existing diagnostics. + +Each checked root receives its own helper context, including roots with equal +block shapes. Planned storage additionally distinguishes static call sites +within a root when their assigned offsets or synchronization differ. Helpers +with identical finalized bindings share one emitted body in that root. + +## Planned full-block storage + +`checked_block_storage_call_v1` extends the typed memory-witness boundary with +one storage operand: `()` selects managed reuse, while `&mut StorageDescriptor` +selects an explicit policy. These internal, unsafe integration types do not +provide a public collective API or safe arbitrary shared-memory references. +Roots using planned storage reject participation-only and raw native markers; +other native calls must use a typed resource or planned-storage boundary. + +`storage_new_v1::()` creates a descriptor +origin. Zero capacity or alignment requests inference. Each rooted static +constructor call is distinct; repeating it in a loop preserves that origin. +Moves and direct helper calls preserve identity. Consuming +`storage_reconfigure_v1::` preserves the origin and selects a policy +before use. Incompatible policies and ambiguous descriptor choices fail +compilation. + +Shared uses occupy one maximum-sized region. Exclusive uses receive separate +aligned regions per rooted static call site; repeated loop iterations reuse +the same site. Explicit capacity must fit the **complete** descriptor plan, +including exclusive padding. Its full capacity contributes to launch facts. +Distinct descriptors receive disjoint allocations. The compiler uses validated +provider extents and alignments; native specialization identity is unchanged. + +For scratchful calls, omitted storage inserts a block barrier before possible +reuse. Explicit manual storage requires the caller's block barrier. Explicit +`AUTO_SYNC=true` inserts a trailing barrier after every scratchful call. +Scratch-free operations allocate no region and gain no storage barrier. A +real full-block barrier also orders the current full-block output-resource +epochs. Different storage descriptors alone do not order output reuse. + +`storage_reserve_v1::(&mut descriptor)` reserves an actual Rust +`[T; N]` layout for one of the ten fixed-width integer/float scalar types. +It returns a non-Copy `StorageReservation` with `as_mut_ptr`, never a Rust +reference or slice. Raw accesses require initialization, bounds and aliasing +proof from the unsafe caller. Overlapping live reservations and primitive uses +are rejected. Different exclusive sites may remain live together. +`storage_release_v1(view)` consumes the handle but emits no barrier: synchronize +after release before overlapping reuse. A barrier while the view is live does +not release it, and dropping or forgetting it is not completion. Reservations +cannot use automatic trailing synchronization. + +Ordinary `DynamicSharedArray::get_fixed` views retain their aliased base prefix. +Planned regions follow that prefix with their required alignment. Unknown +`get/get_raw/offset` views are rejected in a planned root. The existing artifact +launch facts carry the total extent and exact block geometry. This boundary is +full-block and synchronous; warp partitions and resource callbacks require +separate binding support. + +## Host request and response + +For each `(provider, executable)` pair, the compiler invokes the executable +once with `--oxide-library-request-v1`. Stdin is one JSON object with `version: 1`, +`target`, `cuda_root`, and an ordered `requests` array. Each request contains +`operation`, `algorithm`, `dtype`, `block: [B,1,1]`, and `items_per_thread`. +Repeated concrete requests are deduplicated across roots. Scalar descriptions +use `{"kind":"signed|unsigned|float", "bits":N}`; integer widths are +8/16/32/64 and float widths are 16/32/64. A provider may support a narrower set +and must diagnose unsupported requests. + +Stdout contains only a binary response: eight bytes `OXDLIB01`, little-endian +u64 JSON-header length, little-endian u64 LTO-IR length, then the header and raw +LTO-IR. Diagnostics go to stderr; failure exits nonzero. Requests and response +headers are limited to 1 MiB, and the nonempty LTO-IR buffer to 256 MiB. + +The header has `version: 1`, a 32-byte `batch_identity` array, and one +`specializations` entry per request, in the same order: + +| Field | Meaning | +| --- | --- | +| `request_index` | Zero-based request position | +| `identity`, `symbol` | Unique 32-byte specialization identity and native link symbol | +| `arguments` | Ordered runtime tuple ABI, excluding the inserted scratch pointer | +| `cub_layout`, `scratch` | Native temporary-layout and required scratch `{size, alignment}` | +| `scratch_free` | Must agree with `scratch.size == 0` | +| `mutates_payload` | Semantic information; never permission to read an undefined value | + +Argument descriptions use `kind: "const_pointer"`, `"mut_pointer"` or +`"value"`, with a `scalar` description. Layout fields must fit `u32`, and +alignment must be a positive power of two. Symbol names use letters, digits +and underscores and cannot start with a digit. Different batches must not +return the same symbol. Truncated/trailing frames, mismatched versions, duplicate +identities and ABI mismatches fail compilation. + +The production provider is responsible for truthful layouts and identities +covering its sources, headers, recipes and toolchain. Returning owned bytes +avoids a second read of a mutable LTO file during finalization. The registration +macro tracks native inputs before rustc emits dep-info; +manual registration getters must arrange equivalent dependency tracking. + +## Synchronous scalar callbacks + +`call_with_callback_v1` accepts the ordinary registration and runtime tuple, +followed by a thunk function item and an immutable `&Environment`. Its generic +parameters are `T, Registration, Args, Environment, Thunk, BLOCK_THREADS, +ITEMS_PER_THREAD`. The initial callback ABI supports `T = i32`: + +```rust +unsafe extern "C" fn thunk i32 + Copy>( + environment: *const F, left: i32, right: i32, +) -> i32 { + unsafe { (&*environment)(left, right) } +} + +unsafe { + cuda_device::library::call_with_callback_v1::( + registration, arguments, thunk::, &operation, + ); +} +``` + +The compiler requires the actual function-item type with this exact non-unwinding +C signature; runtime function pointers and closure-valued thunk witnesses are +rejected. The environment may be a function item or an `Fn` closure capturing +fixed-width numeric scalars by value. Reference, pointer and aggregate captures +are rejected. Rust owns the layout and keeps the immutable environment live for +the entire synchronous call. Native code only carries its opaque address back to +the thunk; it must not inspect, mutate, retain, free or publish that address. + +The compiler discovers and retains the selected thunk even when no Rust call +invokes it directly. Its definition joins the kernel roots for optimization and +linking. Callback bodies may call ordinary scalar Rust helpers. CUDA resource +operations/types (including generated CUDA intrinsics), foreign or indirect +calls, inline assembly and unqualified compiler intrinsics are rejected +transitively. Resource-bearing callbacks need +an explicit root/storage binding and are outside this version. This raw marker +also requires block-uniform callable specialization and capture values; it does +not prove them, and lane-private environment addresses need not be equal. + +Callback calls use a separate batch invoked with `--oxide-library-request-v2`. +The registration and input-query protocols stay at version one. The request and +response JSON use `version: 2`, and the response prefix is `OXDLIB02`; all existing +length, layout and ordering checks apply. Each request adds `callback`: + +```json +{"symbol":"actual_emitted_thunk", "arguments":[ + {"kind":"const_pointer","scalar":{"kind":"unsigned","bits":8}}, + {"kind":"value","scalar":{"kind":"signed","bits":32}}, + {"kind":"value","scalar":{"kind":"signed","bits":32}} +], "result":{"kind":"signed","bits":32}} +``` + +The symbol is the compiler's final emitted identity. The byte-pointer spelling +is opaque and uses generic address space zero; it conveys no closure layout. +The native operation receives scratch at index two and the environment pointer +as its last argument. Its response `arguments` describes only the explicit Rust +tuple, excluding both compiler-inserted pointers. Native request deduplication +includes the callback descriptor, while runtime capture values do not affect +specialization identity. Rust body changes remain ordinary Rust build inputs. + +## Native input dependencies + +The registration macro tracks its environment variable, the provider executable, +CUDA_HOME/CUDA_PATH selection, and the device compilation settings. During CUBIN +materialization it invokes the provider with `--oxide-library-inputs-v1`. Ordinary +host builds and rustdoc do not query toolkit inputs or require installed CUDA. + +The input query reads stdin through EOF. Its JSON request is +`{"version":1,"cuda_root":"/selected/toolkit","target":"compute_120"}`. +Stdout is one JSON response with `version: 1`, `paths` (absolute file or directory +paths) and `environment` (additional environment variable names). Requests and +responses are limited to 1 MiB. Diagnostics go to stderr; failures exit nonzero. +Unknown fields, invalid versions, relative/missing paths and invalid environment +names are errors. The query declares dependencies without compiling native code. + +The library owns this inventory, including sources, headers, compiler tools and +any environment affecting their selection or behavior. Directories are tracked +recursively so adding a header also causes recompilation. The macro records +these dependencies in the library crate; Cargo propagates its rebuild to +consumers and concrete generic kernel instantiations. The compiler has no +library-specific header layout or native cache. + +Cargo's timestamp freshness rules apply. Replacements with unchanged or older +timestamps can remain undetected, including restored files or a symlink switched +to an older installation at the same path. Clean/rebuild affected crates after +such external changes. +The `native_library` example tracks its provider, CUDA tools/headers, host compiler +driver and relevant environment. It assumes the host compiler's installed system +headers and libraries remain unchanged during a build; its declarations are not +a general inventory of an arbitrary C++ toolchain. + +## Debug information + +Providers choose the debug information in their LTO-IR; the compiler preserves +it during linking. With CUDA 13.3, Rust `--lineinfo` and native nvcc +`--generate-line-info` can be combined, retaining both source line tables. +Rust `--device-debug` also works with a provider built without debug information, +as exercised by the `native_library` full-debug smoke route. + +CUDA 13.3 rejects nvcc `-G` with `-dlto`. Linking Rust full debug with native line +information also fails in nvJitLink when it sees more than one debug compilation +unit. Select a supported combination explicitly; provider metadata is not +silently stripped to make these combinations link. + +## Resources and unsafe obligations + +Every reachable library call contributes its exact block shape and required +scratch to its kernel root. Conflicting shapes and incompatible declared launch +contracts or maximum bounds are errors. The compiled artifact carries the +resulting facts to launch preparation; `dynamic_shared = inferred` with +`LaunchConfig3D::auto_shared` uses the compiled byte count. + +The raw and participation-only markers alias library scratch views at the dynamic shared base, +using the maximum required size and alignment. The caller must complete each +use before conflicting reuse. Mixing native dynamic shared views +with these markers is rejected. They supply no barrier, allocation token or +borrow-based completion. The planned full-block boundary above has explicit +allocation and synchronization rules. + +The library wrapper must enforce pointer validity, alignment, capacity, +initialization, nonaliasing, collective participation and resource reuse for +its native implementation. Convergent declarations preserve optimization +constraints; they do not prove participation or make the wrapper safe. The +marker is an unsafe compiler boundary, not a safe collective operation. + +The provider reads the complete request through stdin EOF before writing its +response. Reserve stdout for the binary response frame and use stderr for +diagnostics. The compiler waits for provider completion; provider execution has +the same trust boundary as a dependency's Cargo build script. diff --git a/cuda-oxide/cuda-oxide-book/compiler/rustc-codegen-cuda.md b/cuda-oxide/cuda-oxide-book/compiler/rustc-codegen-cuda.md index 683605591e..daf3b6d919 100644 --- a/cuda-oxide/cuda-oxide-book/compiler/rustc-codegen-cuda.md +++ b/cuda-oxide/cuda-oxide-book/compiler/rustc-codegen-cuda.md @@ -122,20 +122,17 @@ the host pipeline always runs. The `CodegenBackend` trait also requires `join_codegen()` and `link()`. These handle waiting for parallel codegen threads and invoking the linker, -respectively. In cuda-oxide, both are pure delegation: +respectively. In cuda-oxide, both call the LLVM backend. The backend also +writes artifact objects for the embedded artifact bundles. `join_codegen()` and +`link()` make sure that the outputs contain these objects: -```rust -fn join_codegen(&self, ongoing: Box, sess: &Session) -> ... { - self.llvm_backend.join_codegen(ongoing, sess) -} - -fn link(&self, sess: &Session, codegen: ..., outputs: &OutputFilenames) -> ... { - self.llvm_backend.link(sess, codegen, outputs) -} -``` +- `join_codegen()` adds a module for each artifact object. +- `link()` can give artifact objects to the system linker as link arguments. -No interception, no modification. The LLVM backend handles linking exactly as -it normally would. +With LTO, rustc reads bitcode from the Rust objects of each rlib. The artifact +objects contain no bitcode. The doc comment of `artifact_module()` shows which +artifact objects are Rust objects and which `link()` gives to the system +linker. ```{note} For most crates in the dependency tree -- `core`, `cuda-core`, random diff --git a/cuda-oxide/cuda-oxide-book/getting-started/installation.md b/cuda-oxide/cuda-oxide-book/getting-started/installation.md index f8114dcc48..697f352a3e 100644 --- a/cuda-oxide/cuda-oxide-book/getting-started/installation.md +++ b/cuda-oxide/cuda-oxide-book/getting-started/installation.md @@ -398,6 +398,22 @@ cargo oxide emit-ltoir my_kernels --arch sm_90 Every command takes `--help`, which lists the flags each one accepts. +For kernels that call external device functions, `--materialize-cubin` +links their CUDA LTO-IR definitions during the ordinary Cargo build. Pass each +input with `--link-ltoir` after the subcommand, in link order: + +```bash +cargo oxide run --materialize-cubin --arch sm_120 \ + --link-ltoir build/device-libs/scale.ltoir --link-ltoir build/device-libs/add.ltoir +``` + +Inputs must be raw CUDA LTO-IR, not object files or fatbins. Every supplied +file is linked into each materialized device artifact in the Cargo invocation. +This requires a build-host CUDA Toolkit compatible with the external inputs. +The resulting executable embeds the final cubin. Input names and contents are +part of Cargo's device-code fingerprint, so a changed definition rebuilds the +artifact. See the `external_ltoir` example for C++ sources and generation commands. + --- ## Verifying your installation @@ -435,3 +451,18 @@ If everything is configured correctly, this compiles a Rust kernel to PTX, launc - `cuda.h not found` -- Set `CUDA_TOOLKIT_PATH` to your CUDA install root, or ensure `/usr/local/cuda/include/cuda.h` exists. - `rust-src` / `llvm-tools` component missing -- Run `rustup component add rust-src llvm-tools --toolchain 1.99.1`. ::: + +Generic `#[cuda_module]` loaders support materialization when all required concrete +specializations are in one embedded native artifact. They retain exact symbol +lookup and compiled launch facts in one CUDA module. Multiple native artifacts +or mixed native/PTX bundles are rejected because separate module loads would not +preserve shared device globals. PTX-only builds continue to merge their bundles. + +On Linux, generic loaders read the mapped image containing the defining crate's +host loader code. Link the crate that instantiates the kernels into that same +image. This applies to both native and PTX loading and requires readable +`/proc/self/maps` and `/proc/self/map_files`; a missing or replaced mapped file +is an error. The loader does not fall back to the process executable. +A concrete kernel compiled in a dependency can create a second native artifact; +keep native specializations in one emitting crate. A missing specialization in +an otherwise selected artifact fails at kernel lookup. diff --git a/cuda-oxide/cuda-oxide-book/gpu-programming/closures-and-generics.md b/cuda-oxide/cuda-oxide-book/gpu-programming/closures-and-generics.md index 7055e8100c..34fb71a791 100644 --- a/cuda-oxide/cuda-oxide-book/gpu-programming/closures-and-generics.md +++ b/cuda-oxide/cuda-oxide-book/gpu-programming/closures-and-generics.md @@ -82,6 +82,11 @@ The generated method forces monomorphization of `scale::` so the instantiation appears in the compiled PTX even though it is never called directly on the CPU. +The generated name helper retains the concrete function-item type for device +collection. It does not take a host function pointer to the kernel, so looking +up a specialization's name does not require host definitions of its device-only +callees. The host and device still use the same function-item hash. + ### Compile-time kernel policies A policy type groups tuning choices for one generic kernel: diff --git a/cuda-oxide/cuda-oxide-book/index.md b/cuda-oxide/cuda-oxide-book/index.md index 92579e7d96..9869b926e6 100644 --- a/cuda-oxide/cuda-oxide-book/index.md +++ b/cuda-oxide/cuda-oxide-book/index.md @@ -212,6 +212,7 @@ compiler/compiler-optimizations compiler/mlir-dialects compiler/lowering-pipeline compiler/adding-new-intrinsics +compiler/native-library-calls compiler/catalog-generated-intrinsics compiler/fuzzing-and-differential-testing ``` diff --git a/cuda-oxide/cute-rs-preview/examples/blockscale_gemm_ctm/Cargo.lock b/cuda-oxide/cute-rs-preview/examples/blockscale_gemm_ctm/Cargo.lock index b53df69613..d68b7d2be6 100644 --- a/cuda-oxide/cute-rs-preview/examples/blockscale_gemm_ctm/Cargo.lock +++ b/cuda-oxide/cute-rs-preview/examples/blockscale_gemm_ctm/Cargo.lock @@ -209,6 +209,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -430,8 +432,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/cute-rs-preview/examples/blockscale_gemm_cute/Cargo.lock b/cuda-oxide/cute-rs-preview/examples/blockscale_gemm_cute/Cargo.lock index d1c4551376..67cdab33f1 100644 --- a/cuda-oxide/cute-rs-preview/examples/blockscale_gemm_cute/Cargo.lock +++ b/cuda-oxide/cute-rs-preview/examples/blockscale_gemm_cute/Cargo.lock @@ -207,6 +207,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -442,8 +444,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/cute-rs-preview/examples/elementwise_cute/Cargo.lock b/cuda-oxide/cute-rs-preview/examples/elementwise_cute/Cargo.lock index f9407efb1d..34cb4ddd5a 100644 --- a/cuda-oxide/cute-rs-preview/examples/elementwise_cute/Cargo.lock +++ b/cuda-oxide/cute-rs-preview/examples/elementwise_cute/Cargo.lock @@ -196,6 +196,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -441,8 +443,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/cute-rs-preview/examples/ffn_cute/Cargo.lock b/cuda-oxide/cute-rs-preview/examples/ffn_cute/Cargo.lock index 1bcc8c3ca2..0a1102d198 100644 --- a/cuda-oxide/cute-rs-preview/examples/ffn_cute/Cargo.lock +++ b/cuda-oxide/cute-rs-preview/examples/ffn_cute/Cargo.lock @@ -196,6 +196,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -442,8 +444,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/cute-rs-preview/examples/fmha_decode_sm120a_cute/Cargo.lock b/cuda-oxide/cute-rs-preview/examples/fmha_decode_sm120a_cute/Cargo.lock index fd61d454c1..20a5003b2c 100644 --- a/cuda-oxide/cute-rs-preview/examples/fmha_decode_sm120a_cute/Cargo.lock +++ b/cuda-oxide/cute-rs-preview/examples/fmha_decode_sm120a_cute/Cargo.lock @@ -196,6 +196,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -442,8 +444,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/cute-rs-preview/examples/fmha_prefill_sm120a_cute/Cargo.lock b/cuda-oxide/cute-rs-preview/examples/fmha_prefill_sm120a_cute/Cargo.lock index 5cf93cc223..c04df3b2ba 100644 --- a/cuda-oxide/cute-rs-preview/examples/fmha_prefill_sm120a_cute/Cargo.lock +++ b/cuda-oxide/cute-rs-preview/examples/fmha_prefill_sm120a_cute/Cargo.lock @@ -196,6 +196,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -442,8 +444,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/cute-rs-preview/examples/fp16_gemm_256x352_ctm/Cargo.lock b/cuda-oxide/cute-rs-preview/examples/fp16_gemm_256x352_ctm/Cargo.lock index c1474dc66d..4909f6a12e 100644 --- a/cuda-oxide/cute-rs-preview/examples/fp16_gemm_256x352_ctm/Cargo.lock +++ b/cuda-oxide/cute-rs-preview/examples/fp16_gemm_256x352_ctm/Cargo.lock @@ -198,6 +198,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -430,8 +432,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/cute-rs-preview/examples/fp16_gemm_256x352_cute/Cargo.lock b/cuda-oxide/cute-rs-preview/examples/fp16_gemm_256x352_cute/Cargo.lock index 5343fc92cb..e1531db6c5 100644 --- a/cuda-oxide/cute-rs-preview/examples/fp16_gemm_256x352_cute/Cargo.lock +++ b/cuda-oxide/cute-rs-preview/examples/fp16_gemm_256x352_cute/Cargo.lock @@ -196,6 +196,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -443,8 +445,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/cute-rs-preview/examples/nvfp4_gemv_cute/Cargo.lock b/cuda-oxide/cute-rs-preview/examples/nvfp4_gemv_cute/Cargo.lock index 8b256732c0..3ef5561201 100644 --- a/cuda-oxide/cute-rs-preview/examples/nvfp4_gemv_cute/Cargo.lock +++ b/cuda-oxide/cute-rs-preview/examples/nvfp4_gemv_cute/Cargo.lock @@ -196,6 +196,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -441,8 +443,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/cute-rs-preview/examples/rmsnorm_cute/Cargo.lock b/cuda-oxide/cute-rs-preview/examples/rmsnorm_cute/Cargo.lock index 89b1344256..a634e1da49 100644 --- a/cuda-oxide/cute-rs-preview/examples/rmsnorm_cute/Cargo.lock +++ b/cuda-oxide/cute-rs-preview/examples/rmsnorm_cute/Cargo.lock @@ -196,6 +196,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -431,8 +433,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/cute-rs-preview/examples/rope_cache_cute/Cargo.lock b/cuda-oxide/cute-rs-preview/examples/rope_cache_cute/Cargo.lock index 451bd568f8..721bb975e2 100644 --- a/cuda-oxide/cute-rs-preview/examples/rope_cache_cute/Cargo.lock +++ b/cuda-oxide/cute-rs-preview/examples/rope_cache_cute/Cargo.lock @@ -196,6 +196,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -431,8 +433,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/cute-rs-preview/examples/sm100_fmha_prefill_cute/Cargo.lock b/cuda-oxide/cute-rs-preview/examples/sm100_fmha_prefill_cute/Cargo.lock index ac93a53faa..c9584e5390 100644 --- a/cuda-oxide/cute-rs-preview/examples/sm100_fmha_prefill_cute/Cargo.lock +++ b/cuda-oxide/cute-rs-preview/examples/sm100_fmha_prefill_cute/Cargo.lock @@ -196,6 +196,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -431,8 +433,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/cute-rs-preview/examples/sm100_mla_decode_fp16_cute/Cargo.lock b/cuda-oxide/cute-rs-preview/examples/sm100_mla_decode_fp16_cute/Cargo.lock index 502c15a11c..dd63c84814 100644 --- a/cuda-oxide/cute-rs-preview/examples/sm100_mla_decode_fp16_cute/Cargo.lock +++ b/cuda-oxide/cute-rs-preview/examples/sm100_mla_decode_fp16_cute/Cargo.lock @@ -196,6 +196,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -431,8 +433,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/cute-rs-preview/examples/swiglu_cute/Cargo.lock b/cuda-oxide/cute-rs-preview/examples/swiglu_cute/Cargo.lock index 305d0f5d55..3bb4bb4760 100644 --- a/cuda-oxide/cute-rs-preview/examples/swiglu_cute/Cargo.lock +++ b/cuda-oxide/cute-rs-preview/examples/swiglu_cute/Cargo.lock @@ -196,6 +196,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -430,8 +432,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/cute-rs-preview/models/qwen3_8b_cute/Cargo.lock b/cuda-oxide/cute-rs-preview/models/qwen3_8b_cute/Cargo.lock index cb0a22c8bc..74d0467d29 100644 --- a/cuda-oxide/cute-rs-preview/models/qwen3_8b_cute/Cargo.lock +++ b/cuda-oxide/cute-rs-preview/models/qwen3_8b_cute/Cargo.lock @@ -202,6 +202,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -456,8 +458,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/Cargo.lock b/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/Cargo.lock index 5f28dd3f25..b52ddfeba9 100644 --- a/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/Cargo.lock +++ b/cuda-oxide/cute-rs-preview/models/qwen_decoder_cute/Cargo.lock @@ -202,6 +202,8 @@ dependencies = [ "proc-macro2", "quote", "reserved-oxide-symbols", + "serde", + "serde_json", "syn", ] @@ -456,8 +458,6 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object", ] diff --git a/cuda-oxide/scripts/check-compiled-launch-facts.sh b/cuda-oxide/scripts/check-compiled-launch-facts.sh new file mode 100755 index 0000000000..7dfd7e8652 --- /dev/null +++ b/cuda-oxide/scripts/check-compiled-launch-facts.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +arch="${1:-sm_120}" +oxide=(cargo oxide) +if [[ -n "${CARGO_OXIDE_BIN:-}" ]]; then + oxide=("$CARGO_OXIDE_BIN") +fi +log=$(mktemp) +trap 'rm -f "$log"' EXIT + +"${oxide[@]}" run compiled_launch_facts --arch "$arch" +"${oxide[@]}" run compiled_launch_facts --arch "$arch" --features generic +"${oxide[@]}" run compiled_launch_facts --arch "$arch" --materialize-cubin + +if "${oxide[@]}" run compiled_launch_facts --arch "$arch" --features invalid-raw >"$log" 2>&1; then + echo "error: inferred scratch accepted an unconstrained raw dynamic view" >&2 + exit 1 +fi +grep -F 'inferred dynamic shared memory cannot include unconstrained' "$log" + +"${oxide[@]}" run compiled_launch_facts --arch "$arch" --features generic --materialize-cubin +"${oxide[@]}" run cross_crate_kernel --arch "$arch" --features native-ambiguity +if "${oxide[@]}" run cross_crate_kernel --arch "$arch" --materialize-cubin --features native-ambiguity >"$log" 2>&1; then + echo "error: generic loading accepted separate native artifacts" >&2 + exit 1 +fi +grep -F 'IncompatibleGenericArtifacts' "$log" +grep -F 'kernel-lib' "$log" +echo 'PASS: compiled launch facts, incomplete-plan and native-ambiguity rejection' diff --git a/cuda-oxide/scripts/check-example-smoketest-contract.sh b/cuda-oxide/scripts/check-example-smoketest-contract.sh index 710032d4e8..d48e017fa0 100644 --- a/cuda-oxide/scripts/check-example-smoketest-contract.sh +++ b/cuda-oxide/scripts/check-example-smoketest-contract.sh @@ -228,7 +228,7 @@ if not re.search( r'\[\[ \$\{FULL_DEBUG\} -eq 0 \]\] && verify_nvvm_in_compile_only', source ): sys.exit("full-debug verdict contract missing: libNVVM artifact branch is not excluded") -if "FULL_DEBUG_CONFIGURED_ROUTE_EXAMPLES=(interop_cubin_identity)" not in source: +if "FULL_DEBUG_CONFIGURED_ROUTE_EXAMPLES=(interop_cubin_identity native_library)" not in source: sys.exit("full-debug route contract missing: explicit cubin-project accommodation") if "Full-debug configured route: %d / %d passed" not in source: sys.exit("full-debug route contract missing: configured-route summary") diff --git a/cuda-oxide/scripts/check-external-ltoir.sh b/cuda-oxide/scripts/check-external-ltoir.sh new file mode 100755 index 0000000000..ff2f87dbc5 --- /dev/null +++ b/cuda-oxide/scripts/check-external-ltoir.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Run with a CUDA Toolkit and GPU matching the requested architecture. +set -euo pipefail + +cd "$(dirname "$0")/.." +arch="${1:-sm_120}" +example="$PWD/crates/rustc-codegen-cuda/examples/external_ltoir" +artifacts="$(mktemp -d "${TMPDIR:-/tmp}/oxide-external-ltoir.XXXXXX")" +echo "External LTO-IR regression artifacts: $artifacts" +# Keep the example's executable location independent of the caller's Cargo cache. +unset CARGO_TARGET_DIR +oxide=(cargo oxide) +if [[ -n "${CARGO_OXIDE_BIN:-}" ]]; then + oxide=("$CARGO_OXIDE_BIN") +fi + +nvcc_flags=(--ltoir -dlto -arch="$arch") +if [[ -n "${NVCC_CCBIN:-${CUDAHOSTCXX:-}}" ]]; then + nvcc_flags+=(-ccbin="${NVCC_CCBIN:-$CUDAHOSTCXX}") +fi +nvcc "${nvcc_flags[@]}" "$example/scale.cu" -o "$artifacts/scale.ltoir" +nvcc "${nvcc_flags[@]}" -DEXTERNAL_ADDEND=3 "$example/add.cu" -o "$artifacts/add.ltoir" + +flags=(--arch "$arch" --materialize-cubin + --link-ltoir "$artifacts/scale.ltoir" --link-ltoir "$artifacts/add.ltoir") +cargo_args=(--manifest-path "$example/Cargo.toml" --release) + +run() { + local label="$1" + shift + if ! "$@" >"$artifacts/$label.log" 2>&1; then + cat "$artifacts/$label.log" >&2 + return 1 + fi + echo "PASS $label" +} + +run run "${oxide[@]}" run external_ltoir "${flags[@]}" +run test "${oxide[@]}" test "${flags[@]}" -- "${cargo_args[@]}" -- --nocapture +run build "${oxide[@]}" build "${flags[@]}" -- "${cargo_args[@]}" -v +run fresh "${oxide[@]}" build "${flags[@]}" -- "${cargo_args[@]}" -v +grep -q 'Fresh external_ltoir ' "$artifacts/fresh.log" + +# Change just the library content, retaining its path, size and modification +# time. The next passthrough build must invalidate Cargo's cached device code. +cp -p "$artifacts/add.ltoir" "$artifacts/add-original.ltoir" +nvcc "${nvcc_flags[@]}" -DEXTERNAL_ADDEND=1 "$example/add.cu" -o "$artifacts/add.ltoir" +if [[ "$(wc -c < "$artifacts/add-original.ltoir")" != "$(wc -c < "$artifacts/add.ltoir")" ]]; then + echo "error: this nvcc did not produce equal-sized addend variants" >&2 + exit 1 +fi +if cmp -s "$artifacts/add-original.ltoir" "$artifacts/add.ltoir"; then + echo "error: the addend variants have identical bytes" >&2 + exit 1 +fi +touch -r "$artifacts/add-original.ltoir" "$artifacts/add.ltoir" +run content-change "${oxide[@]}" build "${flags[@]}" -- "${cargo_args[@]}" -v +mv "$artifacts/scale.ltoir" "$artifacts/scale.saved" +mv "$artifacts/add.ltoir" "$artifacts/add.saved" +run changed-result "$example/target/release/external_ltoir" 1 +mv "$artifacts/scale.saved" "$artifacts/scale.ltoir" +mv "$artifacts/add.saved" "$artifacts/add.ltoir" + +if "${oxide[@]}" build --arch "$arch" --materialize-cubin \ + --link-ltoir "$artifacts/scale.ltoir" -- "${cargo_args[@]}" \ + >"$artifacts/missing-symbol.log" 2>&1; then + echo "error: a missing transitive device definition unexpectedly linked" >&2 + exit 1 +fi +grep -q 'external_add' "$artifacts/missing-symbol.log" +grep -qi 'nvJitLink' "$artifacts/missing-symbol.log" +echo "PASS missing-symbol" + +printf 'not an LTO-IR module\n' >"$artifacts/corrupt.ltoir" +if "${oxide[@]}" build --arch "$arch" --materialize-cubin \ + --link-ltoir "$artifacts/scale.ltoir" --link-ltoir "$artifacts/corrupt.ltoir" \ + -- "${cargo_args[@]}" >"$artifacts/corrupt-input.log" 2>&1; then + echo "error: corrupt LTO-IR unexpectedly linked" >&2 + exit 1 +fi +grep -qi 'nvJitLink' "$artifacts/corrupt-input.log" +grep -q 'corrupt.ltoir' "$artifacts/corrupt-input.log" +echo "PASS corrupt-input" + +# Exercise the backend guard using a disposable output directory. The supplied +# input and a previous output must both survive rejection before codegen writes. +collision="$artifacts/collision" +mkdir -p "$collision" +cp "$artifacts/scale.ltoir" "$collision/external_ltoir.ll" +printf 'previous output sentinel\n' >"$artifacts/previous-output" +cp "$artifacts/previous-output" "$collision/external_ltoir.ptx" +if CUDA_OXIDE_PTX_DIR="$collision" "${oxide[@]}" build --arch "$arch" --materialize-cubin \ + --link-ltoir "$collision/external_ltoir.ll" --link-ltoir "$artifacts/add.ltoir" \ + -- "${cargo_args[@]}" >"$artifacts/input-output-overlap.log" 2>&1; then + echo "error: an external input overlapping generated output was accepted" >&2 + exit 1 +fi +grep -q 'overlaps generated artifact' "$artifacts/input-output-overlap.log" +cmp "$artifacts/scale.ltoir" "$collision/external_ltoir.ll" +cmp "$artifacts/previous-output" "$collision/external_ltoir.ptx" +echo "PASS input-output-overlap" + +cp -p "$artifacts/add-original.ltoir" "$artifacts/add.ltoir" +run restored "${oxide[@]}" run external_ltoir "${flags[@]}" diff --git a/cuda-oxide/scripts/check-oxide-artifacts-parity.sh b/cuda-oxide/scripts/check-oxide-artifacts-parity.sh index 3d3d713239..8a0a4eacda 100755 --- a/cuda-oxide/scripts/check-oxide-artifacts-parity.sh +++ b/cuda-oxide/scripts/check-oxide-artifacts-parity.sh @@ -1,61 +1,61 @@ #!/usr/bin/env bash # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Verify the in-tree oxide-artifacts crate and the crates.io release everyone -# consumes are the same version. -# -# crates/oxide-artifacts is the source the release is published from, but no -# workspace consumes it by path any more: the backend writes artifact bundles -# and the shared cuda-core (cutile-rs) reads them, both through crates.io -# oxide-artifacts, so writer and reader are one crate. The in-tree copy can -# still drift ahead of the release; if it does, an edit to the format lands in -# git without reaching any consumer, and the next publish silently changes -# what every reader accepts. This guard fails as soon as the in-tree version, -# the workspace dependency requirements, and the versions resolved in every -# lock file disagree. -# -# Format changes therefore go: bump crates/oxide-artifacts, publish, then move -# the requirement in the SIMT and backend manifests and relock. +# Keep artifact writers and readers on one in-tree implementation and version. set -euo pipefail export LC_ALL=C cd "$(dirname "$0")/.." GIT_ROOT="$(git rev-parse --show-toplevel)" -in_tree="$(sed -n -E 's/^version = "([^"]+)"/\1/p' crates/oxide-artifacts/Cargo.toml | head -1)" -[ -n "${in_tree}" ] || { echo "error: crates/oxide-artifacts/Cargo.toml has no version" >&2; exit 1; } -echo "in-tree oxide-artifacts: ${in_tree}" +python3 - "${GIT_ROOT}" <<'PYTHON' +import pathlib +import subprocess +import sys +import tomllib -status=0 -req_of() { # req_of FILE -> the version requirement on the oxide-artifacts line - local line - line="$(grep -E '^oxide-artifacts[[:space:]]*=' "$1" | head -1 || true)" - if [[ "${line}" =~ version[[:space:]]*=[[:space:]]*\"([^\"]+)\" ]] || [[ "${line}" =~ ^oxide-artifacts[[:space:]]*=[[:space:]]*\"([^\"]+)\" ]]; then - echo "${BASH_REMATCH[1]}" - elif [[ "${line}" =~ path[[:space:]]*= ]]; then - echo "path" - else - echo "missing" - fi -} -for manifest in Cargo.toml crates/rustc-codegen-cuda/Cargo.toml; do - req="$(req_of "${manifest}")" - if [ "${req}" != "${in_tree}" ]; then - echo "error: ${manifest}: oxide-artifacts requirement is '${req}', in-tree version is ${in_tree}" >&2; status=1 - fi -done +root = pathlib.Path(sys.argv[1]) +artifact_dir = root / "cuda-oxide/crates/oxide-artifacts" -# Every lock that resolves oxide-artifacts must resolve exactly the in-tree -# version, from the registry (the SIMT lock also lists the in-tree member). -while IFS= read -r lock; do - versions="$(awk '/^name = "oxide-artifacts"$/{getline; print $3}' "${lock}" | tr -d '"' | sort -u)" - [ -n "${versions}" ] || continue - if [ "${versions}" != "${in_tree}" ]; then - echo "error: ${lock}: oxide-artifacts resolves to ${versions//$'\n'/, }, in-tree version is ${in_tree}" >&2; status=1 - fi - if ! grep -A3 '^name = "oxide-artifacts"$' "${lock}" | grep -q 'source = "registry+'; then - echo "error: ${lock}: oxide-artifacts is not consumed from crates.io" >&2; status=1 - fi -done < <(git -C "${GIT_ROOT}" ls-files 'Cargo.lock' '*/Cargo.lock' '**/Cargo.lock' | while read -r lock; do echo "${GIT_ROOT}/${lock}"; done) +def read_toml(path): + with path.open("rb") as source: + return tomllib.load(source) -[ "${status}" -eq 0 ] && echo "OK: oxide-artifacts is ${in_tree} in the tree, in both manifests, and in every lock (from crates.io)." -exit "${status}" +version = read_toml(artifact_dir / "Cargo.toml")["package"]["version"] +errors = [] +for relative, workspace in [ + ("cuda-core/Cargo.toml", False), + ("cuda-oxide/Cargo.toml", True), + ("cuda-oxide/crates/rustc-codegen-cuda/Cargo.toml", False), +]: + manifest = root / relative + data = read_toml(manifest) + dependencies = data["workspace"]["dependencies"] if workspace else data["dependencies"] + dependency = dependencies["oxide-artifacts"] + if not isinstance(dependency, dict) or dependency.get("version") != version: + errors.append(f"{relative}: oxide-artifacts must declare version {version}") + continue + path = dependency.get("path") + if path is None or (manifest.parent / path).resolve() != artifact_dir.resolve(): + errors.append(f"{relative}: oxide-artifacts must use {artifact_dir}") + +lockfiles = subprocess.check_output( + ["git", "-C", str(root), "ls-files", "-z", "*Cargo.lock"] +).decode().split("\0") +checked = 0 +for relative in filter(None, lockfiles): + packages = [ + package for package in read_toml(root / relative).get("package", []) + if package["name"] == "oxide-artifacts" + ] + if not packages: + continue + checked += 1 + if len(packages) != 1 or packages[0]["version"] != version or "source" in packages[0]: + errors.append(f"{relative}: oxide-artifacts must resolve once to the in-tree version {version}") + +for error in errors: + print(f"error: {error}", file=sys.stderr) +if errors: + sys.exit(1) +print(f"OK: oxide-artifacts {version} uses one in-tree source in all three manifests and {checked} locks.") +PYTHON diff --git a/cuda-oxide/scripts/smoketest.sh b/cuda-oxide/scripts/smoketest.sh index 35d8eb57a2..e6f75e64bb 100755 --- a/cuda-oxide/scripts/smoketest.sh +++ b/cuda-oxide/scripts/smoketest.sh @@ -90,7 +90,7 @@ set -uo pipefail TCGEN05_EXAMPLES=(gemm_sol gemm_sol_final tcgen05 tcgen05_matmul) WGMMA_EXAMPLES=(wgmma) BLACKWELL_MMA_EXAMPLES=(mma_mxf8f6f4) -LTOIR_EXAMPLES=(addressof_sharedarray constant_memory cpp_consumes_rust_device device_ffi_test legacy_atomic_fadd legacy_atomic_rmw_cas legacy_nvvm_pointer_shapes manual_launch_libdevice mathdx_ffi_test primitive_stress) +LTOIR_EXAMPLES=(addressof_sharedarray constant_memory cpp_consumes_rust_device device_ffi_test external_ltoir legacy_atomic_fadd legacy_atomic_rmw_cas legacy_nvvm_pointer_shapes manual_launch_libdevice mathdx_ffi_test primitive_stress) LTOIR_MODERN_EXAMPLES=(small_type_ffi_test) TILELIB_EXAMPLES=(tilelib_relu) AUTO_NVVM_EXAMPLES=(libdevice_math) @@ -127,13 +127,13 @@ NO_LAUNCH_EXAMPLES=(wgmma_mma_bf16) # afterwards purely to produce the artifacts the shape check reads. NO_OPT_SHAPE_EXAMPLES=(const_bool_dead_branch shared_memory_helper) -# This example intentionally configures a cubin artifact in Cargo metadata. -# Its supported project route necessarily includes NVVM IR + libNVVM + +# These examples require cubin artifacts, through Cargo metadata or native +# library materialization. Their supported route includes NVVM IR + libNVVM + # nvJitLink; the full-debug census must exercise that reality rather than # silently rewriting the project to PTX. It is reported separately from the -# direct-LLVM census so the one accommodation cannot be mistaken for direct +# direct-LLVM census so these accommodations cannot be mistaken for direct # backend coverage. -FULL_DEBUG_CONFIGURED_ROUTE_EXAMPLES=(interop_cubin_identity) +FULL_DEBUG_CONFIGURED_ROUTE_EXAMPLES=(interop_cubin_identity native_library) classify() { local ex="$1" cat @@ -263,14 +263,14 @@ OPTIONS is missing or too old). Error examples must still fail. Works on GPU-less CI. --full-debug Compile selected examples through the normal LLVM - `cargo oxide build --device-debug` route (except an - explicitly configured cubin project, reported as a + `cargo oxide build --device-debug` route (except the + configured cubin projects, reported as a separate configured-route accommodation). Validates fresh artifacts, ptxas assembly, permanent verify-debug-info.sh contracts, and intentional error diagnostics, but skips libNVVM and optimized code-shape gates because those do not accept/describe full-debug - output. Nothing is executed. Cannot be combined with + output. No kernels execute. Cannot be combined with --compile-only. -x, --fail-fast Stop at the first failure. -v, --verbose Stream cargo output live (instead of capturing to @@ -539,9 +539,9 @@ printf "%scuda-oxide smoketest%s @ %s%s%s (%s)\n" "${C_BOLD}" "${C_RESET}" "${C_ printf "GPU: %s\n" "${gpu_info}" printf "LTOIR arch: %s (modern: %s)\n" "${LTOIR_ARCH}" "${LTOIR_MODERN_ARCH}" if [[ ${COMPILE_ONLY} -eq 1 ]]; then - printf "Mode: compile-only (device artifacts only; nothing is executed)\n" + printf "Mode: compile-only (device artifacts only; no kernels execute)\n" elif [[ ${FULL_DEBUG} -eq 1 ]]; then - printf "Mode: full-debug compile census (direct LLVM plus explicit configured routes; nothing is executed)\n" + printf "Mode: full-debug compile census (direct LLVM plus explicit configured routes; no kernels execute)\n" fi if [[ -n "${ONLY}" ]]; then printf "Filter --only: %s\n" "${ONLY}"; fi if [[ -n "${SKIP}" ]]; then printf "Filter --skip: %s\n" "${SKIP}"; fi @@ -1026,6 +1026,10 @@ verdict_compile() { local artifact="${ex//-/_}" if [[ ${ec} -gt 128 ]]; then echo "FAIL (crashed, signal $((ec - 128)))"; return 1; fi if [[ ${ec} -ne 0 ]]; then echo "FAIL (exit=${ec})"; return 1; fi + if [[ "${ex}" == "native_library" ]]; then + native_library_cubin_verdict "${log}" + return + fi if [[ ${FULL_DEBUG} -eq 1 ]]; then case "${ex}" in device_ffi_test|mathdx_ffi_test|small_type_ffi_test) @@ -1038,6 +1042,11 @@ verdict_compile() { "${ex}" "${ex_dir}/${artifact}.ptx" return ;; + external_ltoir) + full_debug_relocatable_ffi_verdict \ + "${ex}" "${ex_dir}/${artifact}.ptx" + return + ;; cutile_inter_kernel) full_debug_ptx_verdict \ "${ex_dir}/simt/cutile_inter_kernel_simt.ptx" \ @@ -1180,6 +1189,9 @@ full_debug_relocatable_ffi_verdict() { mathdx_ffi_test) expected_undefined=$'cublasdx_gemm_32x32x32_block_dim_x\ncublasdx_gemm_32x32x32_block_dim_y\ncublasdx_gemm_32x32x32_block_dim_z\ncublasdx_gemm_32x32x32_f32\ncublasdx_gemm_32x32x32_f32_alphabeta\ncublasdx_gemm_32x32x32_smem_size\ncublasdx_gemm_32x32x32_smem_size_ab\ncufftdx_fft_16_c2c_f32_forward\ncufftdx_fft_16_c2c_f32_inverse\ncufftdx_fft_16_elements_per_thread\ncufftdx_fft_16_storage_size\ncufftdx_fft_32_storage_size\ncufftdx_fft_8_c2c_f32_forward\ncufftdx_fft_8_c2c_f32_inverse\ncufftdx_fft_8_elements_per_thread\ncufftdx_fft_8_storage_size\ndebug_extern_double_array' ;; + external_ltoir) + expected_undefined='external_scale_add' + ;; small_type_ffi_test) expected_undefined=$'small_add_u16\nsmall_add_u8\nsmall_half_add\nsmall_not_bool\nsmall_scale_i16\nsmall_scale_i8\nsmall_widen_i8\nsmall_widen_u16' ;; @@ -1266,6 +1278,56 @@ full_debug_relocatable_ffi_verdict() { echo "PASS (full-debug FFI PTX assembled relocatably for ${arch}; DWARF and undefined interface verified)" } +# The native-library fixture embeds its final image in the host executable. +# Its host-only check extracts that exact CUBIN without loading the CUDA driver. +native_library_cubin_verdict() { + local log="$1" target="${LTOIR_ARCH}" cubin elf_reader sections dwarf symbol + local binary="${CARGO_TARGET_DIR}/release/native_library" + # cargo oxide builds with the example as cwd; Cargo resolves relative + # target directories there, while this verdict runs from the repo root. + if [[ "${CARGO_TARGET_DIR}" != /* ]]; then + binary="${repo_root}/crates/rustc-codegen-cuda/examples/native_library/${binary}" + fi + local -a args=("--check-artifact") + if [[ ${FULL_DEBUG} -eq 1 ]]; then target="${LTOIR_MODERN_ARCH}"; fi + cubin="$(mktemp "${TMPDIR:-/tmp}/native-library.XXXXXX.cubin")" || return 1 + args+=("${target}" "${cubin}") + if [[ ${FULL_DEBUG} -eq 1 ]]; then args+=("--full-debug"); fi + if ! "${binary}" "${args[@]}" >>"${log}" 2>&1; then + rm -f "${cubin}" + echo "FAIL (native library embedded CUBIN check failed)" + return 1 + fi + if [[ ${FULL_DEBUG} -eq 1 ]]; then + elf_reader="$(command -v readelf || command -v llvm-readelf || true)" + if [[ -z "${elf_reader}" ]] \ + || ! sections="$("${elf_reader}" -W -S "${cubin}" 2>>"${log}")" \ + || ! dwarf="$("${elf_reader}" --debug-dump=info "${cubin}" 2>>"${log}")"; then + rm -f "${cubin}" + echo "FAIL (could not inspect native library CUBIN DWARF with readelf)" + return 1 + fi + rm -f "${cubin}" + if ! grep -q '[.]debug_info' <<<"${sections}" \ + || ! grep -q '[.]debug_line' <<<"${sections}" \ + || ! grep -q 'DW_TAG_compile_unit' <<<"${dwarf}" \ + || ! grep -q 'DW_TAG_subprogram' <<<"${dwarf}"; then + echo "FAIL (native library CUBIN is missing consumable DWARF)" + return 1 + fi + for symbol in one four; do + if ! grep -Eq "DW_AT_name.*: ${symbol}$" <<<"${dwarf}"; then + echo "FAIL (native library CUBIN is missing ${symbol} DWARF)" + return 1 + fi + done + echo "PASS (embedded CUBIN recorded debug=full and contains consumable DWARF)" + else + rm -f "${cubin}" + echo "PASS (embedded CUBIN and launch facts verified for ${target})" + fi +} + # The one configured cubin project cannot be judged through the root PTX # convention. Verify its complete versioned sidecar handshake and consume the # linked cubin's DWARF so a silently ignored interop debug policy cannot pass. @@ -1362,11 +1424,12 @@ run_full_debug_build() { # Full debug is unsupported by the legacy LLVM 7 NVVM dialect. Keep # the project's configured cubin path, but use the supported modern # dialect at the host target or its GPU-less sm_100 floor. - interop_cubin_identity) arch="${LTOIR_MODERN_ARCH}" ;; + interop_cubin_identity|native_library) arch="${LTOIR_MODERN_ARCH}" ;; esac if [[ -n "${arch}" ]]; then args+=("--arch=${arch}") fi + if [[ "${ex}" == "native_library" ]]; then args+=("--materialize-cubin"); fi if [[ ${VERBOSE} -eq 1 ]]; then cargo oxide "${args[@]}" 2>&1 | tee "${log}" @@ -1405,6 +1468,15 @@ run_cargo() { run_full_debug_build "${ex}" "${log}" "${cat}" return fi + # This fixture exercises compile-time linking of external device libraries. + # Its script supplies those inputs and verifies the embedded executable. + # Compile-only and full-debug retain the ordinary artifact-only FFI routes. + if [[ ${COMPILE_ONLY} -eq 0 && "${ex}" == "external_ltoir" ]]; then + bash scripts/check-external-ltoir.sh "${LTOIR_ARCH}" >"${log}" 2>&1 + CARGO_EC=$? + if [[ ${VERBOSE} -eq 1 ]]; then cat "${log}"; fi + return + fi # Rust `char` must remain plain i32 in both NVVM dialects. A runtime # round-trip cannot distinguish it from an incorrect zeroext i32 ABI. if [[ ${COMPILE_ONLY} -eq 1 && "${ex}" == "device_ffi_test" ]]; then @@ -2192,6 +2264,9 @@ run_cargo() { local verb="run" if [[ ${COMPILE_ONLY} -eq 1 ]]; then verb="build"; fi local -a args=("${verb}" "${ex}") + if [[ "${ex}" == "native_library" ]]; then + args+=("--materialize-cubin" "--arch=${LTOIR_ARCH}") + fi if [[ ${COMPILE_ONLY} -eq 0 && "${ex}" == "ex2_approx_f16" \ && "${host_cc}" =~ ^[0-9]+\.[0-9]+$ \ && $((10#${host_cc//./})) -lt 75 ]]; then diff --git a/cutile-rs/Cargo.lock b/cutile-rs/Cargo.lock index add95613f8..8e19022b97 100644 --- a/cutile-rs/Cargo.lock +++ b/cutile-rs/Cargo.lock @@ -1251,8 +1251,6 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "oxide-artifacts" version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d9a941c0c43d2dceaf25f5c9ca4c9f9cceb28dfa5e4e3cfd14e476f003e7c9" dependencies = [ "object 0.36.7", ]