From 84ac2ed4f04cc212cda9f8c0c0314ff59520ef54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 10 Nov 2026 04:51:36 +0100 Subject: [PATCH 1/3] codegen: cold call sites and their branches stay cold through the statepoint rewrite (experiment) --- crates/perry-codegen/src/inprocess.rs | 1 + .../perry-codegen/src/inprocess/cold_paths.rs | 165 ++++++++++++++++++ .../src/inprocess/optimize_emit.rs | 2 + 3 files changed, 168 insertions(+) create mode 100644 crates/perry-codegen/src/inprocess/cold_paths.rs diff --git a/crates/perry-codegen/src/inprocess.rs b/crates/perry-codegen/src/inprocess.rs index 5a93c00eec1..0e9bec055ba 100644 --- a/crates/perry-codegen/src/inprocess.rs +++ b/crates/perry-codegen/src/inprocess.rs @@ -17,6 +17,7 @@ mod assemble; pub(crate) use assemble::assemble_text; +mod cold_paths; mod coff; #[cfg(test)] mod dead_statepoint_tests; diff --git a/crates/perry-codegen/src/inprocess/cold_paths.rs b/crates/perry-codegen/src/inprocess/cold_paths.rs new file mode 100644 index 00000000000..e13175164c5 --- /dev/null +++ b/crates/perry-codegen/src/inprocess/cold_paths.rs @@ -0,0 +1,165 @@ +//! Cold paths stay cold through the statepoint rewrite. +//! +//! A `cold` helper declaration tells LLVM that the blocks leading to its +//! call are unlikely: block frequencies, block placement and, above all, the +//! register allocator's split and spill decisions follow. That information +//! lives on the callee. `rewrite-statepoints-for-gc` replaces a collecting +//! call by `gc.statepoint(..., @callee, ...)` and keeps only the attributes +//! of the call SITE, so for every collecting helper the fact was lost before +//! the optimizer and the backend ran: the loop poll's slow call, the shared +//! cold element read and the store slow paths were all weighed like the hot +//! path, and values live across them were spilled on the hot path too. +//! +//! This pass runs before the rewrite and states the fact where it survives: +//! - every call to a `cold` callee gets the `cold` call-site attribute; +//! - a conditional branch with exactly one successor that cannot avoid such +//! a call gets explicit `branch_weights`, like the worker-interrupt poll's +//! branch (`loop_interrupt`). +//! +//! Nothing about what the calls do changes: same callee, same convention, +//! same statepoint, same roots. + +use inkwell::module::Module; +use inkwell::values::{AsValueRef, InstructionOpcode}; +use llvm_sys::core::*; +use llvm_sys::prelude::*; +use llvm_sys::LLVMAttributeFunctionIndex; +use std::collections::HashSet; + +/// Taken : not taken, the ratio of the worker-interrupt poll's branch. +const COLD_WEIGHT: u64 = 1; +const HOT_WEIGHT: u64 = 1_048_575; + +/// `PERRY_COLD_PATHS=0` leaves the module as codegen built it. +pub(crate) fn enabled() -> bool { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| enabled_from_env(std::env::var("PERRY_COLD_PATHS").ok().as_deref())) +} + +pub(crate) fn enabled_from_env(value: Option<&str>) -> bool { + !matches!(value, Some("0" | "off" | "false")) +} + +pub(super) fn apply(module: &Module<'_>) -> (usize, usize) { + if !enabled() { + return (0, 0); + } + mark(module) +} + +/// Returns (call sites marked, branches weighted). +fn mark(module: &Module<'_>) -> (usize, usize) { + let (mut sites, mut branches) = (0, 0); + unsafe { + let ctx = LLVMGetModuleContext(module.as_mut_ptr()); + let cold_kind = LLVMGetEnumAttributeKindForName(c"cold".as_ptr(), 4); + let prof_kind = LLVMGetMDKindIDInContext(ctx, c"prof".as_ptr(), 4); + // EXPERIMENT knobs (removed before shipping). + let extra: Vec = std::env::var("PERRY_COLD_X") + .unwrap_or_default() + .split(',') + .map(str::to_string) + .collect(); + let no_prof = std::env::var("PERRY_COLD_PROF").as_deref() == Ok("0"); + let is_cold_callee = |callee: LLVMValueRef| { + if LLVMIsAFunction(callee).is_null() { + return false; + } + if !LLVMGetEnumAttributeAtIndex(callee, LLVMAttributeFunctionIndex, cold_kind).is_null() + { + return true; + } + let mut len = 0usize; + let name = LLVMGetValueName2(callee, &mut len); + let name = std::slice::from_raw_parts(name.cast::(), len); + extra.iter().any(|x| x.as_bytes() == name) + }; + for function in module.get_functions() { + let mut cold: HashSet = HashSet::new(); + let blocks = function.get_basic_blocks(); + for block in &blocks { + for instruction in block.get_instructions() { + if instruction.get_opcode() != InstructionOpcode::Call { + continue; + } + let call = instruction.as_value_ref(); + if !is_cold_callee(LLVMGetCalledValue(call)) { + continue; + } + LLVMAddCallSiteAttribute( + call, + LLVMAttributeFunctionIndex, + LLVMCreateEnumAttribute(ctx, cold_kind, 0), + ); + sites += 1; + cold.insert(block.as_mut_ptr()); + } + } + if cold.is_empty() || no_prof { + continue; + } + // A block whose every successor is cold cannot avoid a cold call. + loop { + let mut grew = false; + for block in &blocks { + let block = block.as_mut_ptr(); + if cold.contains(&block) { + continue; + } + let terminator = LLVMGetBasicBlockTerminator(block); + if terminator.is_null() + || LLVMGetInstructionOpcode(terminator) != llvm_sys::LLVMOpcode::LLVMBr + { + continue; + } + let n = LLVMGetNumSuccessors(terminator); + if n > 0 && (0..n).all(|i| cold.contains(&LLVMGetSuccessor(terminator, i))) { + cold.insert(block); + grew = true; + } + } + if !grew { + break; + } + } + for block in &blocks { + let block = block.as_mut_ptr(); + if cold.contains(&block) { + continue; + } + let terminator = LLVMGetBasicBlockTerminator(block); + if terminator.is_null() + || LLVMGetInstructionOpcode(terminator) != llvm_sys::LLVMOpcode::LLVMBr + || LLVMIsConditional(terminator) == 0 + || LLVMHasMetadata(terminator) != 0 + && !LLVMGetMetadata(terminator, prof_kind).is_null() + { + continue; + } + let taken = cold.contains(&LLVMGetSuccessor(terminator, 0)); + let not_taken = cold.contains(&LLVMGetSuccessor(terminator, 1)); + if taken == not_taken { + continue; + } + let i32_ty = LLVMInt32TypeInContext(ctx); + let weight = |is_cold: bool| { + LLVMValueAsMetadata(LLVMConstInt( + i32_ty, + if is_cold { COLD_WEIGHT } else { HOT_WEIGHT }, + 0, + )) + }; + let mut operands = [ + LLVMMDStringInContext2(ctx, c"branch_weights".as_ptr(), 14), + weight(taken), + weight(not_taken), + ]; + let node = LLVMMDNodeInContext2(ctx, operands.as_mut_ptr(), operands.len()); + LLVMSetMetadata(terminator, prof_kind, LLVMMetadataAsValue(ctx, node)); + branches += 1; + } + } + } + (sites, branches) +} diff --git a/crates/perry-codegen/src/inprocess/optimize_emit.rs b/crates/perry-codegen/src/inprocess/optimize_emit.rs index 3664626f2d4..63f6563b365 100644 --- a/crates/perry-codegen/src/inprocess/optimize_emit.rs +++ b/crates/perry-codegen/src/inprocess/optimize_emit.rs @@ -128,6 +128,8 @@ pub(super) fn optimize_and_emit( module.set_triple(&triple); module.set_data_layout(&tm.get_target_data().get_data_layout()); super::coff::coalesce_helpers(module, effective_target); + // Before the statepoint rewrite hides the callees (see `cold_paths`). + super::cold_paths::apply(module); // RS4GC must run BEFORE the optimization pipeline, and — critically — in // this process, against this LLVM. From 8a4f362d370986946bbc206d5f525257f1187a28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 10 Nov 2026 05:50:13 +0100 Subject: [PATCH 2/3] codegen: cold paths stay cold through the statepoint rewrite (N4) A pass before rewrite-statepoints-for-gc copies `cold` from a helper's declaration to its call sites and weights the branches that lead only into such a call; the loop poll's call, the shared cold element read and the F64-kind store note are declared cold; the guarded arithmetic fast path expects two Numbers. PERRY_COLD_PATHS=0 restores the previous code. --- changelog.d/codegen-cold-paths.md | 1 + .../src/expr/bigint_bitwise_tests.rs | 19 + crates/perry-codegen/src/expr/binary.rs | 7 + .../perry-codegen/src/inprocess/cold_paths.rs | 417 ++++++++++++++---- crates/perry-codegen/src/module/linkage.rs | 33 +- crates/perry-codegen/src/module/tests.rs | 53 +++ .../perry/src/commands/compile/compile_env.rs | 1 + .../src/commands/compile/compile_env/tests.rs | 10 + .../language/control-flow/test_cold_paths.ts | 187 ++++++++ 9 files changed, 641 insertions(+), 87 deletions(-) create mode 100644 changelog.d/codegen-cold-paths.md create mode 100644 tests/parity/language/control-flow/test_cold_paths.ts diff --git a/changelog.d/codegen-cold-paths.md b/changelog.d/codegen-cold-paths.md new file mode 100644 index 00000000000..d1ba52f4c4a --- /dev/null +++ b/changelog.d/codegen-cold-paths.md @@ -0,0 +1 @@ +Performance: cold slow paths inside hot loops no longer cost the loop its registers. LLVM learns that a helper call is unlikely from the `cold` attribute of the helper's declaration, and uses it to place blocks and to decide where the register allocator saves a value that is live across the call. The GC statepoint rewrite, which runs before the optimizer, keeps only the attributes of the call site, so for every helper that can collect the fact was lost: the loop poll's call, the shared cold element read and the inline-cache misses were weighed like the code around them, and values live across them were stored and reloaded on the likely path. A pass before the rewrite now copies `cold` to each call site of a `cold` helper and gives a branch with exactly one side that leads only into such a call the weights `llvm.expect` would give. Three helpers that exist only as the slow arm of an inline fast path are now declared `cold`: `js_gc_loop_safepoint`, `perry_array_read_cold_shared` and `js_array_note_numeric_write_value`; and the guard of the inline `-`, `*`, `/` and bitwise fast path expects two Numbers. Calls, conventions, statepoints and GC roots are unchanged. `PERRY_COLD_PATHS=0` restores the previous code. diff --git a/crates/perry-codegen/src/expr/bigint_bitwise_tests.rs b/crates/perry-codegen/src/expr/bigint_bitwise_tests.rs index 2897b3c06c3..596dfd288d2 100644 --- a/crates/perry-codegen/src/expr/bigint_bitwise_tests.rs +++ b/crates/perry-codegen/src/expr/bigint_bitwise_tests.rs @@ -197,6 +197,25 @@ fn unproven_bitwise_operands_take_a_guarded_int32_arm() { "{op:?} must mask its shift count to 5 bits:\n{ir}" ); } + // N4: the guard says which arm is the likely one (`PERRY_COLD_PATHS`). + let guard = ir + .lines() + .find(|l| l.contains("label %guarded_arith.numeric")) + .unwrap_or_else(|| panic!("{op:?} has no guard branch:\n{ir}")); + let cond = guard + .split_whitespace() + .nth(2) + .unwrap() + .trim_end_matches(','); + let expected = ir.lines().any(|l| { + l.contains(&format!("{cond} = call i1 @llvm.expect.i1(i1 ")) + && l.ends_with(", i1 true)") + }); + assert_eq!( + expected, + crate::module::cold_paths_enabled(), + "{op:?} guard `{guard}` expects two Numbers exactly under the switch:\n{ir}" + ); } } diff --git a/crates/perry-codegen/src/expr/binary.rs b/crates/perry-codegen/src/expr/binary.rs index 72212da8a97..eee4c4d56cf 100644 --- a/crates/perry-codegen/src/expr/binary.rs +++ b/crates/perry-codegen/src/expr/binary.rs @@ -377,6 +377,13 @@ fn lower_guarded_numeric_arith( let fast_label = ctx.block_label(fast_idx); let slow_label = ctx.block_label(slow_idx); let merge_label = ctx.block_label(merge_idx); + // The helper is the arm for a BigInt or a value that needs ToNumeric. + let all_num = if crate::module::cold_paths_enabled() { + ctx.block() + .call(I1, "llvm.expect.i1", &[(I1, &all_num), (I1, "true")]) + } else { + all_num + }; ctx.block().cond_br(&all_num, &fast_label, &slow_label); ctx.current_block = fast_idx; diff --git a/crates/perry-codegen/src/inprocess/cold_paths.rs b/crates/perry-codegen/src/inprocess/cold_paths.rs index e13175164c5..3c73a8c5f47 100644 --- a/crates/perry-codegen/src/inprocess/cold_paths.rs +++ b/crates/perry-codegen/src/inprocess/cold_paths.rs @@ -1,48 +1,40 @@ //! Cold paths stay cold through the statepoint rewrite. //! -//! A `cold` helper declaration tells LLVM that the blocks leading to its -//! call are unlikely: block frequencies, block placement and, above all, the -//! register allocator's split and spill decisions follow. That information -//! lives on the callee. `rewrite-statepoints-for-gc` replaces a collecting -//! call by `gc.statepoint(..., @callee, ...)` and keeps only the attributes -//! of the call SITE, so for every collecting helper the fact was lost before -//! the optimizer and the backend ran: the loop poll's slow call, the shared -//! cold element read and the store slow paths were all weighed like the hot -//! path, and values live across them were spilled on the hot path too. +//! A `cold` helper declaration tells LLVM that the blocks leading to its call +//! are unlikely. Block frequencies follow, and with them block placement and +//! the register allocator's split and spill decisions: a value that is live +//! across an unlikely call is saved and restored next to that call instead of +//! living in memory on the likely path. +//! +//! The fact lives on the callee. `rewrite-statepoints-for-gc` runs before the +//! optimizer and replaces a collecting call by +//! `gc.statepoint(..., @callee, ...)`, keeping the attributes of the call +//! SITE only. So for every collecting helper the fact was gone before any +//! pass could use it: the loop poll's call, the shared cold element read and +//! the inline-cache misses were weighed like the code around them. //! //! This pass runs before the rewrite and states the fact where it survives: //! - every call to a `cold` callee gets the `cold` call-site attribute; -//! - a conditional branch with exactly one successor that cannot avoid such -//! a call gets explicit `branch_weights`, like the worker-interrupt poll's -//! branch (`loop_interrupt`). +//! - a conditional branch with exactly one side that cannot avoid such a call +//! gets the `branch_weights` an `llvm.expect` on its condition would give. //! -//! Nothing about what the calls do changes: same callee, same convention, -//! same statepoint, same roots. +//! Nothing about the call changes: same callee, same convention, same +//! statepoint, same roots. use inkwell::module::Module; use inkwell::values::{AsValueRef, InstructionOpcode}; use llvm_sys::core::*; use llvm_sys::prelude::*; -use llvm_sys::LLVMAttributeFunctionIndex; +use llvm_sys::{LLVMAttributeFunctionIndex, LLVMOpcode}; use std::collections::HashSet; -/// Taken : not taken, the ratio of the worker-interrupt poll's branch. -const COLD_WEIGHT: u64 = 1; -const HOT_WEIGHT: u64 = 1_048_575; - -/// `PERRY_COLD_PATHS=0` leaves the module as codegen built it. -pub(crate) fn enabled() -> bool { - use std::sync::OnceLock; - static CACHED: OnceLock = OnceLock::new(); - *CACHED.get_or_init(|| enabled_from_env(std::env::var("PERRY_COLD_PATHS").ok().as_deref())) -} - -pub(crate) fn enabled_from_env(value: Option<&str>) -> bool { - !matches!(value, Some("0" | "off" | "false")) -} +/// LLVM's own `llvm.expect` weights (`-likely-branch-weight`, +/// `-unlikely-branch-weight`). +const LIKELY: u64 = 2000; +const UNLIKELY: u64 = 1; pub(super) fn apply(module: &Module<'_>) -> (usize, usize) { - if !enabled() { + if !crate::module::cold_paths_enabled() { return (0, 0); } mark(module) @@ -55,40 +47,36 @@ fn mark(module: &Module<'_>) -> (usize, usize) { let ctx = LLVMGetModuleContext(module.as_mut_ptr()); let cold_kind = LLVMGetEnumAttributeKindForName(c"cold".as_ptr(), 4); let prof_kind = LLVMGetMDKindIDInContext(ctx, c"prof".as_ptr(), 4); - // EXPERIMENT knobs (removed before shipping). - let extra: Vec = std::env::var("PERRY_COLD_X") - .unwrap_or_default() - .split(',') - .map(str::to_string) - .collect(); - let no_prof = std::env::var("PERRY_COLD_PROF").as_deref() == Ok("0"); - let is_cold_callee = |callee: LLVMValueRef| { - if LLVMIsAFunction(callee).is_null() { - return false; - } - if !LLVMGetEnumAttributeAtIndex(callee, LLVMAttributeFunctionIndex, cold_kind).is_null() - { - return true; - } - let mut len = 0usize; - let name = LLVMGetValueName2(callee, &mut len); - let name = std::slice::from_raw_parts(name.cast::(), len); - extra.iter().any(|x| x.as_bytes() == name) - }; + let i32_ty = LLVMInt32TypeInContext(ctx); for function in module.get_functions() { + let blocks: Vec = function + .get_basic_blocks() + .iter() + .map(|block| block.as_mut_ptr()) + .collect(); let mut cold: HashSet = HashSet::new(); - let blocks = function.get_basic_blocks(); - for block in &blocks { + for block in function.get_basic_blocks() { for instruction in block.get_instructions() { - if instruction.get_opcode() != InstructionOpcode::Call { + if !matches!( + instruction.get_opcode(), + InstructionOpcode::Call | InstructionOpcode::Invoke + ) { continue; } - let call = instruction.as_value_ref(); - if !is_cold_callee(LLVMGetCalledValue(call)) { + let site = instruction.as_value_ref(); + let callee = LLVMGetCalledValue(site); + if LLVMIsAFunction(callee).is_null() + || LLVMGetEnumAttributeAtIndex( + callee, + LLVMAttributeFunctionIndex, + cold_kind, + ) + .is_null() + { continue; } LLVMAddCallSiteAttribute( - call, + site, LLVMAttributeFunctionIndex, LLVMCreateEnumAttribute(ctx, cold_kind, 0), ); @@ -96,25 +84,26 @@ fn mark(module: &Module<'_>) -> (usize, usize) { cold.insert(block.as_mut_ptr()); } } - if cold.is_empty() || no_prof { + if cold.is_empty() { continue; } - // A block whose every successor is cold cannot avoid a cold call. + let branch = |block: LLVMBasicBlockRef| { + let terminator = LLVMGetBasicBlockTerminator(block); + (!terminator.is_null() + && LLVMGetInstructionOpcode(terminator) == LLVMOpcode::LLVMBr) + .then_some(terminator) + }; + // A block whose every way on leads into a cold block cannot avoid + // the call either. loop { let mut grew = false; - for block in &blocks { - let block = block.as_mut_ptr(); + for &block in &blocks { if cold.contains(&block) { continue; } - let terminator = LLVMGetBasicBlockTerminator(block); - if terminator.is_null() - || LLVMGetInstructionOpcode(terminator) != llvm_sys::LLVMOpcode::LLVMBr - { - continue; - } - let n = LLVMGetNumSuccessors(terminator); - if n > 0 && (0..n).all(|i| cold.contains(&LLVMGetSuccessor(terminator, i))) { + let Some(br) = branch(block) else { continue }; + let n = LLVMGetNumSuccessors(br); + if n > 0 && (0..n).all(|i| cold.contains(&LLVMGetSuccessor(br, i))) { cold.insert(block); grew = true; } @@ -123,32 +112,22 @@ fn mark(module: &Module<'_>) -> (usize, usize) { break; } } - for block in &blocks { - let block = block.as_mut_ptr(); + for &block in &blocks { if cold.contains(&block) { continue; } - let terminator = LLVMGetBasicBlockTerminator(block); - if terminator.is_null() - || LLVMGetInstructionOpcode(terminator) != llvm_sys::LLVMOpcode::LLVMBr - || LLVMIsConditional(terminator) == 0 - || LLVMHasMetadata(terminator) != 0 - && !LLVMGetMetadata(terminator, prof_kind).is_null() - { + let Some(br) = branch(block) else { continue }; + if LLVMIsConditional(br) == 0 || !LLVMGetMetadata(br, prof_kind).is_null() { continue; } - let taken = cold.contains(&LLVMGetSuccessor(terminator, 0)); - let not_taken = cold.contains(&LLVMGetSuccessor(terminator, 1)); + let taken = cold.contains(&LLVMGetSuccessor(br, 0)); + let not_taken = cold.contains(&LLVMGetSuccessor(br, 1)); if taken == not_taken { continue; } - let i32_ty = LLVMInt32TypeInContext(ctx); let weight = |is_cold: bool| { - LLVMValueAsMetadata(LLVMConstInt( - i32_ty, - if is_cold { COLD_WEIGHT } else { HOT_WEIGHT }, - 0, - )) + let w = if is_cold { UNLIKELY } else { LIKELY }; + LLVMValueAsMetadata(LLVMConstInt(i32_ty, w, 0)) }; let mut operands = [ LLVMMDStringInContext2(ctx, c"branch_weights".as_ptr(), 14), @@ -156,10 +135,276 @@ fn mark(module: &Module<'_>) -> (usize, usize) { weight(not_taken), ]; let node = LLVMMDNodeInContext2(ctx, operands.as_mut_ptr(), operands.len()); - LLVMSetMetadata(terminator, prof_kind, LLVMMetadataAsValue(ctx, node)); + LLVMSetMetadata(br, prof_kind, LLVMMetadataAsValue(ctx, node)); branches += 1; } } } (sites, branches) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::inprocess::{parse_ir_text, statepoint_rewritten_ir_with_passes}; + use crate::linker::STATEPOINT_REWRITE_PASSES; + use inkwell::context::Context; + + const IR: &str = r#" +declare void @slow() cold +declare void @plain() +declare i32 @personality(...) +define void @diamond(i1 %c, ptr %f) { +entry: + br i1 %c, label %arm, label %join +arm: + call void @slow() + call void %f() + br label %join +join: + ret void +} +define void @cold_on_false(i1 %c) { +entry: + br i1 %c, label %join, label %arm +arm: + call void @slow() + br label %join +join: + ret void +} +define void @through_a_block(i1 %c, i1 %d) { +entry: + br i1 %c, label %setup, label %join +setup: + br i1 %d, label %arm.a, label %arm.b +arm.a: + br label %arm +arm.b: + br label %arm +arm: + call void @slow() + br label %join +join: + ret void +} +define void @both_cold(i1 %c, i1 %d) { +entry: + br i1 %d, label %pick, label %out +pick: + br i1 %c, label %a, label %b +a: + call void @slow() + ret void +b: + call void @slow() + ret void +out: + ret void +} +define void @not_cold(i1 %c) { +entry: + br i1 %c, label %arm, label %join +arm: + call void @plain() + br label %join +join: + ret void +} +define void @weighted_already(i1 %c) { +entry: + br i1 %c, label %arm, label %join, !prof !0 +arm: + call void @slow() + br label %join +join: + ret void +} +define void @invoked(i1 %c) personality ptr @personality { +entry: + br i1 %c, label %arm, label %join +arm: + invoke void @slow() to label %join unwind label %pad +join: + ret void +pad: + %p = landingpad { ptr, i32 } cleanup + resume { ptr, i32 } %p +} +!0 = !{!"branch_weights", i32 7, i32 9} +"#; + + fn marked() -> (String, (usize, usize)) { + let ctx = Context::create(); + let module = parse_ir_text(&ctx, IR, "cold_paths").unwrap(); + let counts = mark(&module); + module.verify().expect("marked module verifies"); + (module.print_to_string().to_string(), counts) + } + + fn body<'a>(ir: &'a str, name: &str) -> &'a str { + let start = ir + .find(&format!("define void @{name}(")) + .unwrap_or_else(|| panic!("no @{name} in:\n{ir}")); + let end = ir[start..].find("\n}\n").unwrap() + start; + &ir[start..end] + } + + /// The attribute group of the first line of `ir` that contains `needle`. + fn site_attrs(ir: &str, needle: &str) -> String { + let site = ir + .lines() + .find(|l| l.contains(needle)) + .unwrap_or_else(|| panic!("no `{needle}` in:\n{ir}")); + let Some(group) = site.split(" #").nth(1) else { + return String::new(); + }; + let head = format!( + "attributes #{} = ", + group.split_whitespace().next().unwrap() + ); + ir.lines() + .find(|l| l.starts_with(&head)) + .unwrap_or("") + .to_string() + } + + /// The node the `!prof !N` of the line of `text` containing `needle` names. + fn weights<'a>(ir: &'a str, text: &str, needle: &str) -> Option<&'a str> { + let line = text.lines().find(|l| l.contains(needle))?; + let id = line.split("!prof !").nth(1)?.split_whitespace().next()?; + let def = format!("\n!{id} = "); + let at = ir.find(&def)? + def.len(); + ir[at..].lines().next() + } + + #[test] + fn a_cold_callee_marks_its_sites_and_the_branch_into_them() { + let (ir, (sites, branches)) = marked(); + // diamond, cold_on_false, through_a_block, both_cold x2, + // weighted_already, invoked. + assert_eq!(sites, 7, "{ir}"); + // diamond, cold_on_false, through_a_block (entry), both_cold (entry), + // invoked. + assert_eq!(branches, 5, "{ir}"); + let diamond = body(&ir, "diamond"); + assert!( + site_attrs(&ir, "call void @slow()").contains("cold"), + "{ir}" + ); + assert!( + site_attrs(&ir, "invoke void @slow()").contains("cold"), + "{ir}" + ); + assert_eq!( + weights(&ir, diamond, "br i1 %c"), + Some("!{!\"branch_weights\", i32 1, i32 2000}"), + "{ir}" + ); + assert_eq!( + weights(&ir, body(&ir, "cold_on_false"), "br i1 %c"), + Some("!{!\"branch_weights\", i32 2000, i32 1}"), + "{ir}" + ); + assert_eq!( + weights(&ir, body(&ir, "invoked"), "br i1 %c"), + Some("!{!\"branch_weights\", i32 1, i32 2000}"), + "{ir}" + ); + } + + #[test] + fn a_block_that_only_leads_into_a_cold_call_is_cold_too() { + let (ir, _) = marked(); + let f = body(&ir, "through_a_block"); + assert_eq!( + weights(&ir, f, "br i1 %c"), + Some("!{!\"branch_weights\", i32 1, i32 2000}"), + "{f}" + ); + // `setup` chooses between two ways into the same call: neither side + // is the likely one. + assert_eq!(weights(&ir, f, "br i1 %d"), None, "{f}"); + let f = body(&ir, "both_cold"); + assert_eq!(weights(&ir, f, "br i1 %c"), None, "{f}"); + assert_eq!( + weights(&ir, f, "br i1 %d"), + Some("!{!\"branch_weights\", i32 1, i32 2000}"), + "{f}" + ); + } + + #[test] + fn negative_controls_keep_their_calls_and_branches() { + let (ir, _) = marked(); + let plain = body(&ir, "not_cold"); + assert!(!plain.contains("!prof"), "{ir}"); + assert_eq!(site_attrs(&ir, "call void @plain()"), "", "{ir}"); + assert_eq!(site_attrs(&ir, "call void %f()"), "", "{ir}"); + assert_eq!( + weights(&ir, body(&ir, "weighted_already"), "br i1 %c"), + Some("!{!\"branch_weights\", i32 7, i32 9}"), + "weights a lowering wrote are kept:\n{ir}" + ); + } + + const GC_IR: &str = r#" +declare void @slow() cold +define ptr addrspace(1) @rooted(ptr addrspace(1) %p, i1 %c) gc "statepoint-example" { +entry: + br i1 %c, label %arm, label %join +arm: + call void @slow() + br label %join +join: + ret ptr addrspace(1) %p +} +"#; + + /// What this pass exists for: the rewrite drops the callee's `cold` + /// (negative control) and keeps the call site's. The roots are the same + /// either way. + #[test] + fn the_statepoint_rewrite_keeps_only_the_call_site_attribute() { + let target = "x86_64-unknown-linux-gnu"; + let statepoint = "@llvm.experimental.gc.statepoint"; + let unmarked = statepoint_rewritten_ir_with_passes( + GC_IR, + target, + "unmarked", + STATEPOINT_REWRITE_PASSES, + ) + .unwrap(); + assert!( + !site_attrs(&unmarked, statepoint).contains("cold"), + "without the pass the statepoint does not say cold:\n{unmarked}" + ); + assert!(!unmarked.contains("!prof"), "{unmarked}"); + + let ctx = Context::create(); + let module = parse_ir_text(&ctx, GC_IR, "marked").unwrap(); + assert_eq!(mark(&module), (1, 1)); + let marked = statepoint_rewritten_ir_with_passes( + &module.print_to_string().to_string(), + target, + "marked", + STATEPOINT_REWRITE_PASSES, + ) + .unwrap(); + assert!( + site_attrs(&marked, statepoint).contains("cold"), + "the statepoint carries `cold`:\n{marked}" + ); + assert!( + marked.contains("!\"branch_weights\", i32 1, i32 2000"), + "{marked}" + ); + let relocations = |ir: &str| ir.matches("@llvm.experimental.gc.relocate").count(); + assert!(relocations(&marked) > 0, "{marked}"); + assert_eq!(relocations(&marked), relocations(&unmarked)); + assert_eq!( + marked.matches("\"gc-live\"").count(), + unmarked.matches("\"gc-live\"").count() + ); + } +} diff --git a/crates/perry-codegen/src/module/linkage.rs b/crates/perry-codegen/src/module/linkage.rs index 9885d32f4da..cc03f0a3d5e 100644 --- a/crates/perry-codegen/src/module/linkage.rs +++ b/crates/perry-codegen/src/module/linkage.rs @@ -445,6 +445,37 @@ const COLD_HELPERS: &[&str] = &[ "js_stack_overflow", ]; +/// Helpers that exist only as the slow arm of an inline fast path: the loop +/// poll's call behind the arming word, the outlined element read behind the +/// plain-array guards, the F64-kind store's note behind its kind test (it +/// clears the kind, so it runs once per array). `cold` like the list above; +/// kept apart because `PERRY_COLD_PATHS=0` takes these back out. +const COLD_ARM_HELPERS: &[&str] = &[ + "js_array_note_numeric_write_value", + "js_gc_loop_safepoint", + "perry_array_read_cold_shared", +]; + +/// `PERRY_COLD_PATHS` (default on): cold arms are stated to LLVM where the +/// statement survives the statepoint rewrite (`inprocess/cold_paths.rs`). +/// `=0`/`off`/`false` restores the previous module exactly. +pub(crate) fn cold_paths_enabled() -> bool { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + cold_paths_enabled_from_env(std::env::var("PERRY_COLD_PATHS").ok().as_deref()) + }) +} + +pub(crate) fn cold_paths_enabled_from_env(value: Option<&str>) -> bool { + !matches!(value, Some("0" | "off" | "false")) +} + +/// Whether `name` is declared `cold`; `cold_arms` is [`cold_paths_enabled`]. +pub(crate) fn is_cold_helper(name: &str, cold_arms: bool) -> bool { + COLD_HELPERS.contains(&name) || (cold_arms && COLD_ARM_HELPERS.contains(&name)) +} + // The two memory-contract lists below are audited facts about what a // helper's body reads and writes, which no table carries. Whether a call can // unwind is not part of them: `nounwind` comes only from the table of the @@ -543,7 +574,7 @@ const READONLY_HELPERS: &[&str] = &[ /// `js_value_length_f64` (Buffer/TypedArray registry lookups take locks — /// a lock acquisition writes memory). fn helper_decl_contract(name: &str) -> HelperDeclContract { - if COLD_HELPERS.contains(&name) { + if is_cold_helper(name, cold_paths_enabled()) { HelperDeclContract::Cold } else if PURE_HELPERS.contains(&name) { HelperDeclContract::Pure diff --git a/crates/perry-codegen/src/module/tests.rs b/crates/perry-codegen/src/module/tests.rs index effbce96098..af3a1eab255 100644 --- a/crates/perry-codegen/src/module/tests.rs +++ b/crates/perry-codegen/src/module/tests.rs @@ -1135,3 +1135,56 @@ fn read_site_collecting_declarations_preserve_the_original_contract() { "{ir}" ); } + +/// N4: the slow-arm helpers are `cold` under `PERRY_COLD_PATHS` (default on) +/// and ordinary without it; the older list does not depend on the switch. +#[test] +fn cold_arm_helpers_follow_the_cold_path_switch() { + use super::{cold_paths_enabled_from_env, is_cold_helper}; + assert!(cold_paths_enabled_from_env(None)); + assert!(cold_paths_enabled_from_env(Some("1"))); + for off in ["0", "off", "false"] { + assert!(!cold_paths_enabled_from_env(Some(off)), "{off}"); + } + for name in [ + "js_gc_loop_safepoint", + "perry_array_read_cold_shared", + "js_array_note_numeric_write_value", + ] { + assert!(is_cold_helper(name, true), "{name}"); + assert!(!is_cold_helper(name, false), "{name} with the switch off"); + } + for on in [true, false] { + assert!(is_cold_helper("js_object_get_field_slow", on)); + // Negative controls: helpers that are also called on ordinary paths. + for name in [ + "js_dynamic_bitand", + "js_write_barrier_slot", + "js_array_push_f64", + ] { + assert!(!is_cold_helper(name, on), "{name}"); + } + } + + let mut m = module_under("js_gc_loop_safepoint\tReenters\tno\n"); + m.declare_function("js_gc_loop_safepoint", VOID, &[]); + m.declare_function( + "perry_array_read_cold_shared", + DOUBLE, + &[DOUBLE, DOUBLE, I64], + ); + let f = m.define_function("main", I32, vec![]); + f.create_block("entry").ret(I32, "0"); + let ir = m.to_ir(); + let cold = super::cold_paths_enabled(); + for (decl, attrs) in [ + ("declare void @js_gc_loop_safepoint()", " nounwind"), + ( + "declare double @perry_array_read_cold_shared(double, double, i64)", + "", + ), + ] { + let line = format!("{decl}{attrs}{}\n", if cold { " cold" } else { "" }); + assert!(ir.contains(&line), "missing {line:?} in:\n{ir}"); + } +} diff --git a/crates/perry/src/commands/compile/compile_env.rs b/crates/perry/src/commands/compile/compile_env.rs index 35cfc2f1aaf..e9002e05408 100644 --- a/crates/perry/src/commands/compile/compile_env.rs +++ b/crates/perry/src/commands/compile/compile_env.rs @@ -38,6 +38,7 @@ pub(super) const COMPILE_ENV_VARS: &[&str] = &[ "PERRY_CODEGEN_UNITS", "PERRY_CODEGEN_UNIT_BYTES", "PERRY_CODEGEN_UNIT_SIZE", + "PERRY_COLD_PATHS", "PERRY_CONCAT_SITE_CACHE", "PERRY_CONSTFN_SHAPE", "PERRY_CONST_ARRAY_DESCRIPTOR", diff --git a/crates/perry/src/commands/compile/compile_env/tests.rs b/crates/perry/src/commands/compile/compile_env/tests.rs index e91586730aa..0a66089f62a 100644 --- a/crates/perry/src/commands/compile/compile_env/tests.rs +++ b/crates/perry/src/commands/compile/compile_env/tests.rs @@ -345,3 +345,13 @@ fn const_number_switch_is_a_cache_input() { "PERRY_FACTS_CONST_NUMBERS must key the compile caches" ); } + +/// N4: the cold-path switch changes helper declarations, call-site +/// attributes and branch weights, so `=0` must never reuse an `=1` object. +#[test] +fn cold_path_switch_is_a_cache_input() { + assert!( + COMPILE_ENV_VARS.contains(&"PERRY_COLD_PATHS"), + "PERRY_COLD_PATHS must key the compile caches" + ); +} diff --git a/tests/parity/language/control-flow/test_cold_paths.ts b/tests/parity/language/control-flow/test_cold_paths.ts new file mode 100644 index 00000000000..33b8798ce19 --- /dev/null +++ b/tests/parity/language/control-flow/test_cold_paths.ts @@ -0,0 +1,187 @@ +// Cold paths inside hot loops: the loop poll's slow call, the shared cold +// element read, store slow paths, a throw from a cold arm, the stack guard. +// Every loop keeps many numbers, integers and heap values alive across the +// cold arm, and every cold arm really runs (the negative controls of +// PERRY_COLD_PATHS: with the arm weighed as unlikely, a value that was not +// saved around it, or a root the collector did not see, changes a line). +// Every line is a value node prints too. + +class P { + x: number; + y: number; + tag: string; + constructor(x: number, y: number, tag: string) { + this.x = x; + this.y = y; + this.tag = tag; + } +} + +// 1. An allocating loop (it polls) with ten doubles, four integers and three +// heap values live across the back edge. +function poll(n: number): string { + let a = 0.5, b = 1.25, c = 2.75, d = 3.5, e = 4.125, f = 5.0625, g = 6.5, h = 7.75, i2 = 8.25, j = 9.5; + let k = 1, l = 2, m = 3, o = 4; + let keep = new P(1, 2, "keep"); + const list: P[] = []; + let text = ""; + for (let i = 0; i < n; i++) { + const p = new P(i, a + b, "p" + (i & 7)); + if ((i & 63) === 0) list.push(p); + a += b * 0.5; b += c * 0.25; c += d * 0.125; d += e * 0.5; e += f * 0.25; + f += g * 0.125; g += h * 0.5; h += i2 * 0.25; i2 += j * 0.125; j += a * 1e-9; + k = (k + l) | 0; l = (l ^ m) | 0; m = (m + o + i) | 0; o = (o * 3 + 1) & 0xffff; + if ((i & 255) === 0) { keep = new P(keep.x + p.x, keep.y + 1, keep.tag); text += p.tag; } + if (a > 1e12) { a = 0.5; b = 1.25; c = 2.75; d = 3.5; e = 4.125; f = 5.0625; g = 6.5; h = 7.75; i2 = 8.25; } + } + let sum = 0; + for (const p of list) sum += p.x + p.y; + return [a, b, c, d, e, f, g, h, i2, j].map((v) => v.toFixed(3)).join(" ") + + " | " + [k, l, m, o].join(" ") + " | " + keep.x + " " + keep.y + " " + keep.tag + + " | " + list.length + " " + sum.toFixed(3) + " " + text.length; +} +console.log("poll", poll(20000)); + +// 2. Element reads whose cold arm runs: a hole, past the end, a negative +// key, a fraction, a string key, a receiver that is not an array. +function reads(a: any, keys: any[], n: number): string { + let s = 0.5, t = 1.5, u = 2.5, w = 3.5; + let und = 0, str = 0; + for (let i = 0; i < n; i++) { + const key = keys[i % keys.length]; + const v = a[key]; + s += t * 0.5; t += u * 0.25; u += w * 0.125; w += 1; + if (v === undefined) und++; + else if (typeof v === "string") str += v.length; + else if (typeof v === "number") s += v; + else str += 100; + if (s > 1e9) { s = 0.5; t = 1.5; u = 2.5; } + } + return s.toFixed(3) + " " + t.toFixed(3) + " " + u.toFixed(3) + " " + w + " " + und + " " + str; +} +const holey: any[] = [1, 2, , 4, 5.5, "six", , 8]; +const KEYS: any[] = [0, 1, 2, 3, 4, 5, 6, 7, 8, -1, 0.5, "3", "length", 100]; +console.log("reads array", reads(holey, KEYS, 5000)); +console.log("reads string", reads("hello world", KEYS, 5000)); +console.log("reads object", reads({ 0: 7, 3: "x", length: 2 }, KEYS, 5000)); + +function pawnFiles(own: number[], n: number): number { + let passed = 0, acc = 0.25; + for (let i = 0; i < n; i++) { + const file = i & 7; + for (let f = file - 1; f <= file + 1; f++) { + const v = own[f]; + acc += 0.5; + if (v === undefined) passed += 1; + else passed += v; + } + } + return passed + acc; +} +console.log("pawn files", pawnFiles([0, 1, 0, 2, 0, 0, 3, 0], 4000)); + +// 3. Element stores whose slow path runs: growth, a hole, a kind change. +function stores(n: number): string { + const a: any[] = [1, 2, 3]; + let s = 0.5, t = 1.5, u = 2.5; + for (let i = 0; i < n; i++) { + s += t * 0.5; t += u * 0.25; u += 1; + if (i % 97 === 0) a[a.length + 2] = i; + else if (i % 89 === 0) a[i % a.length] = "s" + i; + else if (i % 83 === 0) a[i % a.length] = { i }; + else a[i % a.length] = s; + if (s > 1e9) { s = 0.5; t = 1.5; } + } + let holes = 0, strings = 0, objects = 0, numbers = 0; + for (let i = 0; i < a.length; i++) { + const v = a[i]; + if (!(i in a)) holes++; + else if (typeof v === "string") strings++; + else if (typeof v === "object") objects++; + else numbers++; + } + return a.length + " " + holes + " " + strings + " " + objects + " " + numbers + " " + s.toFixed(3) + " " + u; +} +console.log("stores", stores(6000)); + +// 4. A throw out of a cold arm, caught in the loop, with live values used in +// the handler and after it. +function throws(rows: any[], n: number): string { + let s = 0.5, t = 1.5, u = 2.5, caught = 0, last = ""; + for (let i = 0; i < n; i++) { + s += t * 0.5; t += u * 0.25; u += 1; + try { + const row = rows[i % rows.length]; + s += row.cells[i & 3]; + } catch (err) { + caught++; + last = (err as Error).constructor.name + ":" + i + ":" + t.toFixed(2); + s -= 1; + } + if (s > 1e9) { s = 0.5; t = 1.5; } + } + return s.toFixed(3) + " " + t.toFixed(3) + " " + u + " " + caught + " " + last; +} +const ROWS: any[] = [{ cells: [1, 2, 3, 4] }, { cells: [5, 6, 7, 8] }, undefined, { cells: [9, 10, 11, 12] }, null]; +console.log("throws", throws(ROWS, 3000)); + +function thrower(a: number[], i: number): number { + if (i > a.length) throw new RangeError("past " + i); + return a[i]; +} +function throwsFromCallee(n: number): string { + const a = [1.5, 2.5, 3.5]; + let s = 0.25, t = 0.75, caught = 0, und = 0; + for (let i = 0; i < n; i++) { + t += 0.5; + try { + const v = thrower(a, i % 6); + if (v === undefined) und++; else s += v; + } catch (err) { + caught++; + s += (err as Error).message.length + t; + } + } + return s.toFixed(3) + " " + t + " " + caught + " " + und; +} +console.log("throws from callee", throwsFromCallee(3000)); + +// 5. The stack guard's arm: locals of every frame stay right after the +// RangeError unwinds through them. +function deep(n: number, a: number, b: number): number { + const local = a * 0.5 + b * 0.25 + 1; + if (n === 0) return local; + return deep(n - 1, b, local) + 1; +} +console.log("deep ok", deep(2000, 1, 2).toFixed(3)); +function overflow(a: number): number { + const local = a + 0.5; + return overflow(local) + local; +} +let guard = "none", before = 1.5, after = 2.5; +try { + overflow(1); +} catch (err) { + guard = (err as Error).constructor.name; +} +console.log("deep overflow", guard, before + after, deep(100, 1, 2).toFixed(3)); + +// 6. Heap values in locals across the poll and across a cold read, while +// the loop allocates: moved objects are read through their new address. +function moved(n: number): string { + const board: any[] = []; + for (let i = 0; i < 64; i++) board.push(i % 5 === 0 ? undefined : new P(i, i * 2, "sq" + i)); + let first = board[1] as P, s = 0.5, missing = 0; + let name = ""; + for (let i = 0; i < n; i++) { + const sq = board[(i * 7) % 70]; + const fresh = new P(i, s, "f" + (i % 3)); + if (sq === undefined) { missing++; if (i % 11 === 0) board[(i * 7) % 64] = fresh; } + else { s += sq.x + sq.y * 0.5; name = sq.tag; } + if (i % 13 === 0) first = fresh; + s += first.x * 0.001; + if (s > 1e9) s = 0.5; + } + return s.toFixed(3) + " " + missing + " " + name + " " + first.x + " " + first.tag; +} +console.log("moved", moved(20000)); From c4edbf06cc18cc2bf3e6fb20ca840d1a3a469e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 10 Nov 2026 07:17:52 +0100 Subject: [PATCH 3/3] codegen: only the slow-arm helpers are marked at their call sites Marking the older cold helper list (inline-cache misses) as well grew TypeScript-from-source by 725 KB of code for 0.04% fewer instructions. --- .../perry-codegen/src/inprocess/cold_paths.rs | 88 +++++++++++-------- crates/perry-codegen/src/module/linkage.rs | 16 +++- 2 files changed, 67 insertions(+), 37 deletions(-) diff --git a/crates/perry-codegen/src/inprocess/cold_paths.rs b/crates/perry-codegen/src/inprocess/cold_paths.rs index 3c73a8c5f47..69cbd171c41 100644 --- a/crates/perry-codegen/src/inprocess/cold_paths.rs +++ b/crates/perry-codegen/src/inprocess/cold_paths.rs @@ -1,25 +1,27 @@ -//! Cold paths stay cold through the statepoint rewrite. +//! Cold arms stay cold through the statepoint rewrite. //! -//! A `cold` helper declaration tells LLVM that the blocks leading to its call -//! are unlikely. Block frequencies follow, and with them block placement and -//! the register allocator's split and spill decisions: a value that is live -//! across an unlikely call is saved and restored next to that call instead of -//! living in memory on the likely path. +//! LLVM learns that a call is unlikely from `cold`. Block frequencies follow, +//! and with them block placement and the register allocator's split and spill +//! decisions: a value that is live across an unlikely call is saved and +//! restored next to that call instead of living in memory on the likely path. //! -//! The fact lives on the callee. `rewrite-statepoints-for-gc` runs before the -//! optimizer and replaces a collecting call by -//! `gc.statepoint(..., @callee, ...)`, keeping the attributes of the call -//! SITE only. So for every collecting helper the fact was gone before any -//! pass could use it: the loop poll's call, the shared cold element read and -//! the inline-cache misses were weighed like the code around them. +//! On a helper's declaration the fact does not survive. +//! `rewrite-statepoints-for-gc` runs before the optimizer and replaces a +//! collecting call by `gc.statepoint(..., @callee, ...)`, keeping the +//! attributes of the call SITE only. The loop poll's call and the shared cold +//! element read are collecting calls inside the hottest loops: LLVM weighed +//! them like the code around them, and values live across them were stored +//! and reloaded on the likely path. //! -//! This pass runs before the rewrite and states the fact where it survives: -//! - every call to a `cold` callee gets the `cold` call-site attribute; +//! This pass runs before the rewrite. For the helpers that exist only as the +//! slow arm of an inline fast path (`module::is_cold_arm_helper`): +//! - every call gets the `cold` call-site attribute; //! - a conditional branch with exactly one side that cannot avoid such a call //! gets the `branch_weights` an `llvm.expect` on its condition would give. //! //! Nothing about the call changes: same callee, same convention, same -//! statepoint, same roots. +//! statepoint, same roots. The cost is code size: the save and restore that +//! one spill used to share now sit at each cold call. use inkwell::module::Module; use inkwell::values::{AsValueRef, InstructionOpcode}; @@ -65,14 +67,13 @@ fn mark(module: &Module<'_>) -> (usize, usize) { } let site = instruction.as_value_ref(); let callee = LLVMGetCalledValue(site); - if LLVMIsAFunction(callee).is_null() - || LLVMGetEnumAttributeAtIndex( - callee, - LLVMAttributeFunctionIndex, - cold_kind, - ) - .is_null() - { + if LLVMIsAFunction(callee).is_null() { + continue; + } + let mut len = 0usize; + let name = LLVMGetValueName2(callee, &mut len); + let name = std::slice::from_raw_parts(name.cast::(), len); + if !std::str::from_utf8(name).is_ok_and(crate::module::is_cold_arm_helper) { continue; } LLVMAddCallSiteAttribute( @@ -151,14 +152,15 @@ mod tests { use inkwell::context::Context; const IR: &str = r#" -declare void @slow() cold +declare void @js_gc_loop_safepoint() cold +declare void @js_object_get_field_slow() cold declare void @plain() declare i32 @personality(...) define void @diamond(i1 %c, ptr %f) { entry: br i1 %c, label %arm, label %join arm: - call void @slow() + call void @js_gc_loop_safepoint() call void %f() br label %join join: @@ -168,7 +170,7 @@ define void @cold_on_false(i1 %c) { entry: br i1 %c, label %join, label %arm arm: - call void @slow() + call void @js_gc_loop_safepoint() br label %join join: ret void @@ -183,7 +185,7 @@ arm.a: arm.b: br label %arm arm: - call void @slow() + call void @js_gc_loop_safepoint() br label %join join: ret void @@ -194,10 +196,10 @@ entry: pick: br i1 %c, label %a, label %b a: - call void @slow() + call void @js_gc_loop_safepoint() ret void b: - call void @slow() + call void @js_gc_loop_safepoint() ret void out: ret void @@ -211,11 +213,20 @@ arm: join: ret void } +define void @declared_cold_only(i1 %c) { +entry: + br i1 %c, label %arm, label %join +arm: + call void @js_object_get_field_slow() + br label %join +join: + ret void +} define void @weighted_already(i1 %c) { entry: br i1 %c, label %arm, label %join, !prof !0 arm: - call void @slow() + call void @js_gc_loop_safepoint() br label %join join: ret void @@ -224,7 +235,7 @@ define void @invoked(i1 %c) personality ptr @personality { entry: br i1 %c, label %arm, label %join arm: - invoke void @slow() to label %join unwind label %pad + invoke void @js_gc_loop_safepoint() to label %join unwind label %pad join: ret void pad: @@ -289,11 +300,11 @@ pad: assert_eq!(branches, 5, "{ir}"); let diamond = body(&ir, "diamond"); assert!( - site_attrs(&ir, "call void @slow()").contains("cold"), + site_attrs(&ir, "call void @js_gc_loop_safepoint()").contains("cold"), "{ir}" ); assert!( - site_attrs(&ir, "invoke void @slow()").contains("cold"), + site_attrs(&ir, "invoke void @js_gc_loop_safepoint()").contains("cold"), "{ir}" ); assert_eq!( @@ -340,6 +351,13 @@ pad: let plain = body(&ir, "not_cold"); assert!(!plain.contains("!prof"), "{ir}"); assert_eq!(site_attrs(&ir, "call void @plain()"), "", "{ir}"); + // `cold` on a declaration alone does not make a helper a cold arm. + assert!(!body(&ir, "declared_cold_only").contains("!prof"), "{ir}"); + assert_eq!( + site_attrs(&ir, "call void @js_object_get_field_slow()"), + "", + "{ir}" + ); assert_eq!(site_attrs(&ir, "call void %f()"), "", "{ir}"); assert_eq!( weights(&ir, body(&ir, "weighted_already"), "br i1 %c"), @@ -349,12 +367,12 @@ pad: } const GC_IR: &str = r#" -declare void @slow() cold +declare void @js_gc_loop_safepoint() cold define ptr addrspace(1) @rooted(ptr addrspace(1) %p, i1 %c) gc "statepoint-example" { entry: br i1 %c, label %arm, label %join arm: - call void @slow() + call void @js_gc_loop_safepoint() br label %join join: ret ptr addrspace(1) %p diff --git a/crates/perry-codegen/src/module/linkage.rs b/crates/perry-codegen/src/module/linkage.rs index cc03f0a3d5e..914ef3c69f9 100644 --- a/crates/perry-codegen/src/module/linkage.rs +++ b/crates/perry-codegen/src/module/linkage.rs @@ -448,8 +448,15 @@ const COLD_HELPERS: &[&str] = &[ /// Helpers that exist only as the slow arm of an inline fast path: the loop /// poll's call behind the arming word, the outlined element read behind the /// plain-array guards, the F64-kind store's note behind its kind test (it -/// clears the kind, so it runs once per array). `cold` like the list above; -/// kept apart because `PERRY_COLD_PATHS=0` takes these back out. +/// clears the kind, so it runs once per array). Declared `cold` like the +/// list above, under `PERRY_COLD_PATHS`. +/// +/// These, and only these, are also marked at their call sites +/// (`inprocess/cold_paths.rs`). The statepoint rewrite keeps a call site's +/// attributes and drops the callee's, so `COLD_HELPERS`' `cold` reaches LLVM +/// only where the call is not a statepoint. Marking those sites as well was +/// measured and left out: TypeScript compiled from source grew by 725 KB of +/// code (1.1%) for 0.04% fewer instructions. const COLD_ARM_HELPERS: &[&str] = &[ "js_array_note_numeric_write_value", "js_gc_loop_safepoint", @@ -471,6 +478,11 @@ pub(crate) fn cold_paths_enabled_from_env(value: Option<&str>) -> bool { !matches!(value, Some("0" | "off" | "false")) } +/// Whether calls of `name` are marked cold at the call site. +pub(crate) fn is_cold_arm_helper(name: &str) -> bool { + COLD_ARM_HELPERS.contains(&name) +} + /// Whether `name` is declared `cold`; `cold_arms` is [`cold_paths_enabled`]. pub(crate) fn is_cold_helper(name: &str, cold_arms: bool) -> bool { COLD_HELPERS.contains(&name) || (cold_arms && COLD_ARM_HELPERS.contains(&name))