diff --git a/changelog.d/codegen-cold-paths.md b/changelog.d/codegen-cold-paths.md new file mode 100644 index 00000000000..d1ba52f4c4a --- /dev/null +++ b/changelog.d/codegen-cold-paths.md @@ -0,0 +1 @@ +Performance: cold slow paths inside hot loops no longer cost the loop its registers. LLVM learns that a helper call is unlikely from the `cold` attribute of the helper's declaration, and uses it to place blocks and to decide where the register allocator saves a value that is live across the call. The GC statepoint rewrite, which runs before the optimizer, keeps only the attributes of the call site, so for every helper that can collect the fact was lost: the loop poll's call, the shared cold element read and the inline-cache misses were weighed like the code around them, and values live across them were stored and reloaded on the likely path. A pass before the rewrite now copies `cold` to each call site of a `cold` helper and gives a branch with exactly one side that leads only into such a call the weights `llvm.expect` would give. Three helpers that exist only as the slow arm of an inline fast path are now declared `cold`: `js_gc_loop_safepoint`, `perry_array_read_cold_shared` and `js_array_note_numeric_write_value`; and the guard of the inline `-`, `*`, `/` and bitwise fast path expects two Numbers. Calls, conventions, statepoints and GC roots are unchanged. `PERRY_COLD_PATHS=0` restores the previous code. diff --git a/crates/perry-codegen/src/expr/bigint_bitwise_tests.rs b/crates/perry-codegen/src/expr/bigint_bitwise_tests.rs index 2897b3c06c3..596dfd288d2 100644 --- a/crates/perry-codegen/src/expr/bigint_bitwise_tests.rs +++ b/crates/perry-codegen/src/expr/bigint_bitwise_tests.rs @@ -197,6 +197,25 @@ fn unproven_bitwise_operands_take_a_guarded_int32_arm() { "{op:?} must mask its shift count to 5 bits:\n{ir}" ); } + // N4: the guard says which arm is the likely one (`PERRY_COLD_PATHS`). + let guard = ir + .lines() + .find(|l| l.contains("label %guarded_arith.numeric")) + .unwrap_or_else(|| panic!("{op:?} has no guard branch:\n{ir}")); + let cond = guard + .split_whitespace() + .nth(2) + .unwrap() + .trim_end_matches(','); + let expected = ir.lines().any(|l| { + l.contains(&format!("{cond} = call i1 @llvm.expect.i1(i1 ")) + && l.ends_with(", i1 true)") + }); + assert_eq!( + expected, + crate::module::cold_paths_enabled(), + "{op:?} guard `{guard}` expects two Numbers exactly under the switch:\n{ir}" + ); } } diff --git a/crates/perry-codegen/src/expr/binary.rs b/crates/perry-codegen/src/expr/binary.rs index 72212da8a97..eee4c4d56cf 100644 --- a/crates/perry-codegen/src/expr/binary.rs +++ b/crates/perry-codegen/src/expr/binary.rs @@ -377,6 +377,13 @@ fn lower_guarded_numeric_arith( let fast_label = ctx.block_label(fast_idx); let slow_label = ctx.block_label(slow_idx); let merge_label = ctx.block_label(merge_idx); + // The helper is the arm for a BigInt or a value that needs ToNumeric. + let all_num = if crate::module::cold_paths_enabled() { + ctx.block() + .call(I1, "llvm.expect.i1", &[(I1, &all_num), (I1, "true")]) + } else { + all_num + }; ctx.block().cond_br(&all_num, &fast_label, &slow_label); ctx.current_block = fast_idx; diff --git a/crates/perry-codegen/src/inprocess.rs b/crates/perry-codegen/src/inprocess.rs index 5a93c00eec1..0e9bec055ba 100644 --- a/crates/perry-codegen/src/inprocess.rs +++ b/crates/perry-codegen/src/inprocess.rs @@ -17,6 +17,7 @@ mod assemble; pub(crate) use assemble::assemble_text; +mod cold_paths; mod coff; #[cfg(test)] mod dead_statepoint_tests; diff --git a/crates/perry-codegen/src/inprocess/cold_paths.rs b/crates/perry-codegen/src/inprocess/cold_paths.rs new file mode 100644 index 00000000000..69cbd171c41 --- /dev/null +++ b/crates/perry-codegen/src/inprocess/cold_paths.rs @@ -0,0 +1,428 @@ +//! Cold arms stay cold through the statepoint rewrite. +//! +//! LLVM learns that a call is unlikely from `cold`. Block frequencies follow, +//! and with them block placement and the register allocator's split and spill +//! decisions: a value that is live across an unlikely call is saved and +//! restored next to that call instead of living in memory on the likely path. +//! +//! On a helper's declaration the fact does not survive. +//! `rewrite-statepoints-for-gc` runs before the optimizer and replaces a +//! collecting call by `gc.statepoint(..., @callee, ...)`, keeping the +//! attributes of the call SITE only. The loop poll's call and the shared cold +//! element read are collecting calls inside the hottest loops: LLVM weighed +//! them like the code around them, and values live across them were stored +//! and reloaded on the likely path. +//! +//! This pass runs before the rewrite. For the helpers that exist only as the +//! slow arm of an inline fast path (`module::is_cold_arm_helper`): +//! - every call gets the `cold` call-site attribute; +//! - a conditional branch with exactly one side that cannot avoid such a call +//! gets the `branch_weights` an `llvm.expect` on its condition would give. +//! +//! Nothing about the call changes: same callee, same convention, same +//! statepoint, same roots. The cost is code size: the save and restore that +//! one spill used to share now sit at each cold call. + +use inkwell::module::Module; +use inkwell::values::{AsValueRef, InstructionOpcode}; +use llvm_sys::core::*; +use llvm_sys::prelude::*; +use llvm_sys::{LLVMAttributeFunctionIndex, LLVMOpcode}; +use std::collections::HashSet; + +/// LLVM's own `llvm.expect` weights (`-likely-branch-weight`, +/// `-unlikely-branch-weight`). +const LIKELY: u64 = 2000; +const UNLIKELY: u64 = 1; + +pub(super) fn apply(module: &Module<'_>) -> (usize, usize) { + if !crate::module::cold_paths_enabled() { + return (0, 0); + } + mark(module) +} + +/// Returns (call sites marked, branches weighted). +fn mark(module: &Module<'_>) -> (usize, usize) { + let (mut sites, mut branches) = (0, 0); + unsafe { + let ctx = LLVMGetModuleContext(module.as_mut_ptr()); + let cold_kind = LLVMGetEnumAttributeKindForName(c"cold".as_ptr(), 4); + let prof_kind = LLVMGetMDKindIDInContext(ctx, c"prof".as_ptr(), 4); + let i32_ty = LLVMInt32TypeInContext(ctx); + for function in module.get_functions() { + let blocks: Vec = function + .get_basic_blocks() + .iter() + .map(|block| block.as_mut_ptr()) + .collect(); + let mut cold: HashSet = HashSet::new(); + for block in function.get_basic_blocks() { + for instruction in block.get_instructions() { + if !matches!( + instruction.get_opcode(), + InstructionOpcode::Call | InstructionOpcode::Invoke + ) { + continue; + } + let site = instruction.as_value_ref(); + let callee = LLVMGetCalledValue(site); + if LLVMIsAFunction(callee).is_null() { + continue; + } + let mut len = 0usize; + let name = LLVMGetValueName2(callee, &mut len); + let name = std::slice::from_raw_parts(name.cast::(), len); + if !std::str::from_utf8(name).is_ok_and(crate::module::is_cold_arm_helper) { + continue; + } + LLVMAddCallSiteAttribute( + site, + LLVMAttributeFunctionIndex, + LLVMCreateEnumAttribute(ctx, cold_kind, 0), + ); + sites += 1; + cold.insert(block.as_mut_ptr()); + } + } + if cold.is_empty() { + continue; + } + let branch = |block: LLVMBasicBlockRef| { + let terminator = LLVMGetBasicBlockTerminator(block); + (!terminator.is_null() + && LLVMGetInstructionOpcode(terminator) == LLVMOpcode::LLVMBr) + .then_some(terminator) + }; + // A block whose every way on leads into a cold block cannot avoid + // the call either. + loop { + let mut grew = false; + for &block in &blocks { + if cold.contains(&block) { + continue; + } + let Some(br) = branch(block) else { continue }; + let n = LLVMGetNumSuccessors(br); + if n > 0 && (0..n).all(|i| cold.contains(&LLVMGetSuccessor(br, i))) { + cold.insert(block); + grew = true; + } + } + if !grew { + break; + } + } + for &block in &blocks { + if cold.contains(&block) { + continue; + } + let Some(br) = branch(block) else { continue }; + if LLVMIsConditional(br) == 0 || !LLVMGetMetadata(br, prof_kind).is_null() { + continue; + } + let taken = cold.contains(&LLVMGetSuccessor(br, 0)); + let not_taken = cold.contains(&LLVMGetSuccessor(br, 1)); + if taken == not_taken { + continue; + } + let weight = |is_cold: bool| { + let w = if is_cold { UNLIKELY } else { LIKELY }; + LLVMValueAsMetadata(LLVMConstInt(i32_ty, w, 0)) + }; + let mut operands = [ + LLVMMDStringInContext2(ctx, c"branch_weights".as_ptr(), 14), + weight(taken), + weight(not_taken), + ]; + let node = LLVMMDNodeInContext2(ctx, operands.as_mut_ptr(), operands.len()); + LLVMSetMetadata(br, prof_kind, LLVMMetadataAsValue(ctx, node)); + branches += 1; + } + } + } + (sites, branches) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::inprocess::{parse_ir_text, statepoint_rewritten_ir_with_passes}; + use crate::linker::STATEPOINT_REWRITE_PASSES; + use inkwell::context::Context; + + const IR: &str = r#" +declare void @js_gc_loop_safepoint() cold +declare void @js_object_get_field_slow() cold +declare void @plain() +declare i32 @personality(...) +define void @diamond(i1 %c, ptr %f) { +entry: + br i1 %c, label %arm, label %join +arm: + call void @js_gc_loop_safepoint() + call void %f() + br label %join +join: + ret void +} +define void @cold_on_false(i1 %c) { +entry: + br i1 %c, label %join, label %arm +arm: + call void @js_gc_loop_safepoint() + br label %join +join: + ret void +} +define void @through_a_block(i1 %c, i1 %d) { +entry: + br i1 %c, label %setup, label %join +setup: + br i1 %d, label %arm.a, label %arm.b +arm.a: + br label %arm +arm.b: + br label %arm +arm: + call void @js_gc_loop_safepoint() + br label %join +join: + ret void +} +define void @both_cold(i1 %c, i1 %d) { +entry: + br i1 %d, label %pick, label %out +pick: + br i1 %c, label %a, label %b +a: + call void @js_gc_loop_safepoint() + ret void +b: + call void @js_gc_loop_safepoint() + ret void +out: + ret void +} +define void @not_cold(i1 %c) { +entry: + br i1 %c, label %arm, label %join +arm: + call void @plain() + br label %join +join: + ret void +} +define void @declared_cold_only(i1 %c) { +entry: + br i1 %c, label %arm, label %join +arm: + call void @js_object_get_field_slow() + br label %join +join: + ret void +} +define void @weighted_already(i1 %c) { +entry: + br i1 %c, label %arm, label %join, !prof !0 +arm: + call void @js_gc_loop_safepoint() + br label %join +join: + ret void +} +define void @invoked(i1 %c) personality ptr @personality { +entry: + br i1 %c, label %arm, label %join +arm: + invoke void @js_gc_loop_safepoint() to label %join unwind label %pad +join: + ret void +pad: + %p = landingpad { ptr, i32 } cleanup + resume { ptr, i32 } %p +} +!0 = !{!"branch_weights", i32 7, i32 9} +"#; + + fn marked() -> (String, (usize, usize)) { + let ctx = Context::create(); + let module = parse_ir_text(&ctx, IR, "cold_paths").unwrap(); + let counts = mark(&module); + module.verify().expect("marked module verifies"); + (module.print_to_string().to_string(), counts) + } + + fn body<'a>(ir: &'a str, name: &str) -> &'a str { + let start = ir + .find(&format!("define void @{name}(")) + .unwrap_or_else(|| panic!("no @{name} in:\n{ir}")); + let end = ir[start..].find("\n}\n").unwrap() + start; + &ir[start..end] + } + + /// The attribute group of the first line of `ir` that contains `needle`. + fn site_attrs(ir: &str, needle: &str) -> String { + let site = ir + .lines() + .find(|l| l.contains(needle)) + .unwrap_or_else(|| panic!("no `{needle}` in:\n{ir}")); + let Some(group) = site.split(" #").nth(1) else { + return String::new(); + }; + let head = format!( + "attributes #{} = ", + group.split_whitespace().next().unwrap() + ); + ir.lines() + .find(|l| l.starts_with(&head)) + .unwrap_or("") + .to_string() + } + + /// The node the `!prof !N` of the line of `text` containing `needle` names. + fn weights<'a>(ir: &'a str, text: &str, needle: &str) -> Option<&'a str> { + let line = text.lines().find(|l| l.contains(needle))?; + let id = line.split("!prof !").nth(1)?.split_whitespace().next()?; + let def = format!("\n!{id} = "); + let at = ir.find(&def)? + def.len(); + ir[at..].lines().next() + } + + #[test] + fn a_cold_callee_marks_its_sites_and_the_branch_into_them() { + let (ir, (sites, branches)) = marked(); + // diamond, cold_on_false, through_a_block, both_cold x2, + // weighted_already, invoked. + assert_eq!(sites, 7, "{ir}"); + // diamond, cold_on_false, through_a_block (entry), both_cold (entry), + // invoked. + assert_eq!(branches, 5, "{ir}"); + let diamond = body(&ir, "diamond"); + assert!( + site_attrs(&ir, "call void @js_gc_loop_safepoint()").contains("cold"), + "{ir}" + ); + assert!( + site_attrs(&ir, "invoke void @js_gc_loop_safepoint()").contains("cold"), + "{ir}" + ); + assert_eq!( + weights(&ir, diamond, "br i1 %c"), + Some("!{!\"branch_weights\", i32 1, i32 2000}"), + "{ir}" + ); + assert_eq!( + weights(&ir, body(&ir, "cold_on_false"), "br i1 %c"), + Some("!{!\"branch_weights\", i32 2000, i32 1}"), + "{ir}" + ); + assert_eq!( + weights(&ir, body(&ir, "invoked"), "br i1 %c"), + Some("!{!\"branch_weights\", i32 1, i32 2000}"), + "{ir}" + ); + } + + #[test] + fn a_block_that_only_leads_into_a_cold_call_is_cold_too() { + let (ir, _) = marked(); + let f = body(&ir, "through_a_block"); + assert_eq!( + weights(&ir, f, "br i1 %c"), + Some("!{!\"branch_weights\", i32 1, i32 2000}"), + "{f}" + ); + // `setup` chooses between two ways into the same call: neither side + // is the likely one. + assert_eq!(weights(&ir, f, "br i1 %d"), None, "{f}"); + let f = body(&ir, "both_cold"); + assert_eq!(weights(&ir, f, "br i1 %c"), None, "{f}"); + assert_eq!( + weights(&ir, f, "br i1 %d"), + Some("!{!\"branch_weights\", i32 1, i32 2000}"), + "{f}" + ); + } + + #[test] + fn negative_controls_keep_their_calls_and_branches() { + let (ir, _) = marked(); + let plain = body(&ir, "not_cold"); + assert!(!plain.contains("!prof"), "{ir}"); + assert_eq!(site_attrs(&ir, "call void @plain()"), "", "{ir}"); + // `cold` on a declaration alone does not make a helper a cold arm. + assert!(!body(&ir, "declared_cold_only").contains("!prof"), "{ir}"); + assert_eq!( + site_attrs(&ir, "call void @js_object_get_field_slow()"), + "", + "{ir}" + ); + assert_eq!(site_attrs(&ir, "call void %f()"), "", "{ir}"); + assert_eq!( + weights(&ir, body(&ir, "weighted_already"), "br i1 %c"), + Some("!{!\"branch_weights\", i32 7, i32 9}"), + "weights a lowering wrote are kept:\n{ir}" + ); + } + + const GC_IR: &str = r#" +declare void @js_gc_loop_safepoint() cold +define ptr addrspace(1) @rooted(ptr addrspace(1) %p, i1 %c) gc "statepoint-example" { +entry: + br i1 %c, label %arm, label %join +arm: + call void @js_gc_loop_safepoint() + br label %join +join: + ret ptr addrspace(1) %p +} +"#; + + /// What this pass exists for: the rewrite drops the callee's `cold` + /// (negative control) and keeps the call site's. The roots are the same + /// either way. + #[test] + fn the_statepoint_rewrite_keeps_only_the_call_site_attribute() { + let target = "x86_64-unknown-linux-gnu"; + let statepoint = "@llvm.experimental.gc.statepoint"; + let unmarked = statepoint_rewritten_ir_with_passes( + GC_IR, + target, + "unmarked", + STATEPOINT_REWRITE_PASSES, + ) + .unwrap(); + assert!( + !site_attrs(&unmarked, statepoint).contains("cold"), + "without the pass the statepoint does not say cold:\n{unmarked}" + ); + assert!(!unmarked.contains("!prof"), "{unmarked}"); + + let ctx = Context::create(); + let module = parse_ir_text(&ctx, GC_IR, "marked").unwrap(); + assert_eq!(mark(&module), (1, 1)); + let marked = statepoint_rewritten_ir_with_passes( + &module.print_to_string().to_string(), + target, + "marked", + STATEPOINT_REWRITE_PASSES, + ) + .unwrap(); + assert!( + site_attrs(&marked, statepoint).contains("cold"), + "the statepoint carries `cold`:\n{marked}" + ); + assert!( + marked.contains("!\"branch_weights\", i32 1, i32 2000"), + "{marked}" + ); + let relocations = |ir: &str| ir.matches("@llvm.experimental.gc.relocate").count(); + assert!(relocations(&marked) > 0, "{marked}"); + assert_eq!(relocations(&marked), relocations(&unmarked)); + assert_eq!( + marked.matches("\"gc-live\"").count(), + unmarked.matches("\"gc-live\"").count() + ); + } +} diff --git a/crates/perry-codegen/src/inprocess/optimize_emit.rs b/crates/perry-codegen/src/inprocess/optimize_emit.rs index 3664626f2d4..63f6563b365 100644 --- a/crates/perry-codegen/src/inprocess/optimize_emit.rs +++ b/crates/perry-codegen/src/inprocess/optimize_emit.rs @@ -128,6 +128,8 @@ pub(super) fn optimize_and_emit( module.set_triple(&triple); module.set_data_layout(&tm.get_target_data().get_data_layout()); super::coff::coalesce_helpers(module, effective_target); + // Before the statepoint rewrite hides the callees (see `cold_paths`). + super::cold_paths::apply(module); // RS4GC must run BEFORE the optimization pipeline, and — critically — in // this process, against this LLVM. diff --git a/crates/perry-codegen/src/module/linkage.rs b/crates/perry-codegen/src/module/linkage.rs index 9885d32f4da..914ef3c69f9 100644 --- a/crates/perry-codegen/src/module/linkage.rs +++ b/crates/perry-codegen/src/module/linkage.rs @@ -445,6 +445,49 @@ const COLD_HELPERS: &[&str] = &[ "js_stack_overflow", ]; +/// Helpers that exist only as the slow arm of an inline fast path: the loop +/// poll's call behind the arming word, the outlined element read behind the +/// plain-array guards, the F64-kind store's note behind its kind test (it +/// clears the kind, so it runs once per array). Declared `cold` like the +/// list above, under `PERRY_COLD_PATHS`. +/// +/// These, and only these, are also marked at their call sites +/// (`inprocess/cold_paths.rs`). The statepoint rewrite keeps a call site's +/// attributes and drops the callee's, so `COLD_HELPERS`' `cold` reaches LLVM +/// only where the call is not a statepoint. Marking those sites as well was +/// measured and left out: TypeScript compiled from source grew by 725 KB of +/// code (1.1%) for 0.04% fewer instructions. +const COLD_ARM_HELPERS: &[&str] = &[ + "js_array_note_numeric_write_value", + "js_gc_loop_safepoint", + "perry_array_read_cold_shared", +]; + +/// `PERRY_COLD_PATHS` (default on): cold arms are stated to LLVM where the +/// statement survives the statepoint rewrite (`inprocess/cold_paths.rs`). +/// `=0`/`off`/`false` restores the previous module exactly. +pub(crate) fn cold_paths_enabled() -> bool { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + cold_paths_enabled_from_env(std::env::var("PERRY_COLD_PATHS").ok().as_deref()) + }) +} + +pub(crate) fn cold_paths_enabled_from_env(value: Option<&str>) -> bool { + !matches!(value, Some("0" | "off" | "false")) +} + +/// Whether calls of `name` are marked cold at the call site. +pub(crate) fn is_cold_arm_helper(name: &str) -> bool { + COLD_ARM_HELPERS.contains(&name) +} + +/// Whether `name` is declared `cold`; `cold_arms` is [`cold_paths_enabled`]. +pub(crate) fn is_cold_helper(name: &str, cold_arms: bool) -> bool { + COLD_HELPERS.contains(&name) || (cold_arms && COLD_ARM_HELPERS.contains(&name)) +} + // The two memory-contract lists below are audited facts about what a // helper's body reads and writes, which no table carries. Whether a call can // unwind is not part of them: `nounwind` comes only from the table of the @@ -543,7 +586,7 @@ const READONLY_HELPERS: &[&str] = &[ /// `js_value_length_f64` (Buffer/TypedArray registry lookups take locks — /// a lock acquisition writes memory). fn helper_decl_contract(name: &str) -> HelperDeclContract { - if COLD_HELPERS.contains(&name) { + if is_cold_helper(name, cold_paths_enabled()) { HelperDeclContract::Cold } else if PURE_HELPERS.contains(&name) { HelperDeclContract::Pure diff --git a/crates/perry-codegen/src/module/tests.rs b/crates/perry-codegen/src/module/tests.rs index effbce96098..af3a1eab255 100644 --- a/crates/perry-codegen/src/module/tests.rs +++ b/crates/perry-codegen/src/module/tests.rs @@ -1135,3 +1135,56 @@ fn read_site_collecting_declarations_preserve_the_original_contract() { "{ir}" ); } + +/// N4: the slow-arm helpers are `cold` under `PERRY_COLD_PATHS` (default on) +/// and ordinary without it; the older list does not depend on the switch. +#[test] +fn cold_arm_helpers_follow_the_cold_path_switch() { + use super::{cold_paths_enabled_from_env, is_cold_helper}; + assert!(cold_paths_enabled_from_env(None)); + assert!(cold_paths_enabled_from_env(Some("1"))); + for off in ["0", "off", "false"] { + assert!(!cold_paths_enabled_from_env(Some(off)), "{off}"); + } + for name in [ + "js_gc_loop_safepoint", + "perry_array_read_cold_shared", + "js_array_note_numeric_write_value", + ] { + assert!(is_cold_helper(name, true), "{name}"); + assert!(!is_cold_helper(name, false), "{name} with the switch off"); + } + for on in [true, false] { + assert!(is_cold_helper("js_object_get_field_slow", on)); + // Negative controls: helpers that are also called on ordinary paths. + for name in [ + "js_dynamic_bitand", + "js_write_barrier_slot", + "js_array_push_f64", + ] { + assert!(!is_cold_helper(name, on), "{name}"); + } + } + + let mut m = module_under("js_gc_loop_safepoint\tReenters\tno\n"); + m.declare_function("js_gc_loop_safepoint", VOID, &[]); + m.declare_function( + "perry_array_read_cold_shared", + DOUBLE, + &[DOUBLE, DOUBLE, I64], + ); + let f = m.define_function("main", I32, vec![]); + f.create_block("entry").ret(I32, "0"); + let ir = m.to_ir(); + let cold = super::cold_paths_enabled(); + for (decl, attrs) in [ + ("declare void @js_gc_loop_safepoint()", " nounwind"), + ( + "declare double @perry_array_read_cold_shared(double, double, i64)", + "", + ), + ] { + let line = format!("{decl}{attrs}{}\n", if cold { " cold" } else { "" }); + assert!(ir.contains(&line), "missing {line:?} in:\n{ir}"); + } +} diff --git a/crates/perry/src/commands/compile/compile_env.rs b/crates/perry/src/commands/compile/compile_env.rs index 62368400e40..ba8db9a589e 100644 --- a/crates/perry/src/commands/compile/compile_env.rs +++ b/crates/perry/src/commands/compile/compile_env.rs @@ -38,6 +38,7 @@ pub(super) const COMPILE_ENV_VARS: &[&str] = &[ "PERRY_CODEGEN_UNITS", "PERRY_CODEGEN_UNIT_BYTES", "PERRY_CODEGEN_UNIT_SIZE", + "PERRY_COLD_PATHS", "PERRY_CONCAT_SITE_CACHE", "PERRY_CONSTFN_SHAPE", "PERRY_CONST_ARRAY_DESCRIPTOR", diff --git a/crates/perry/src/commands/compile/compile_env/tests.rs b/crates/perry/src/commands/compile/compile_env/tests.rs index 8dd323220cf..85292133aea 100644 --- a/crates/perry/src/commands/compile/compile_env/tests.rs +++ b/crates/perry/src/commands/compile/compile_env/tests.rs @@ -347,3 +347,13 @@ fn const_number_switch_is_a_cache_input() { "PERRY_FACTS_CONST_NUMBERS must key the compile caches" ); } + +/// N4: the cold-path switch changes helper declarations, call-site +/// attributes and branch weights, so `=0` must never reuse an `=1` object. +#[test] +fn cold_path_switch_is_a_cache_input() { + assert!( + COMPILE_ENV_VARS.contains(&"PERRY_COLD_PATHS"), + "PERRY_COLD_PATHS must key the compile caches" + ); +} diff --git a/tests/parity/language/control-flow/test_cold_paths.ts b/tests/parity/language/control-flow/test_cold_paths.ts new file mode 100644 index 00000000000..33b8798ce19 --- /dev/null +++ b/tests/parity/language/control-flow/test_cold_paths.ts @@ -0,0 +1,187 @@ +// Cold paths inside hot loops: the loop poll's slow call, the shared cold +// element read, store slow paths, a throw from a cold arm, the stack guard. +// Every loop keeps many numbers, integers and heap values alive across the +// cold arm, and every cold arm really runs (the negative controls of +// PERRY_COLD_PATHS: with the arm weighed as unlikely, a value that was not +// saved around it, or a root the collector did not see, changes a line). +// Every line is a value node prints too. + +class P { + x: number; + y: number; + tag: string; + constructor(x: number, y: number, tag: string) { + this.x = x; + this.y = y; + this.tag = tag; + } +} + +// 1. An allocating loop (it polls) with ten doubles, four integers and three +// heap values live across the back edge. +function poll(n: number): string { + let a = 0.5, b = 1.25, c = 2.75, d = 3.5, e = 4.125, f = 5.0625, g = 6.5, h = 7.75, i2 = 8.25, j = 9.5; + let k = 1, l = 2, m = 3, o = 4; + let keep = new P(1, 2, "keep"); + const list: P[] = []; + let text = ""; + for (let i = 0; i < n; i++) { + const p = new P(i, a + b, "p" + (i & 7)); + if ((i & 63) === 0) list.push(p); + a += b * 0.5; b += c * 0.25; c += d * 0.125; d += e * 0.5; e += f * 0.25; + f += g * 0.125; g += h * 0.5; h += i2 * 0.25; i2 += j * 0.125; j += a * 1e-9; + k = (k + l) | 0; l = (l ^ m) | 0; m = (m + o + i) | 0; o = (o * 3 + 1) & 0xffff; + if ((i & 255) === 0) { keep = new P(keep.x + p.x, keep.y + 1, keep.tag); text += p.tag; } + if (a > 1e12) { a = 0.5; b = 1.25; c = 2.75; d = 3.5; e = 4.125; f = 5.0625; g = 6.5; h = 7.75; i2 = 8.25; } + } + let sum = 0; + for (const p of list) sum += p.x + p.y; + return [a, b, c, d, e, f, g, h, i2, j].map((v) => v.toFixed(3)).join(" ") + + " | " + [k, l, m, o].join(" ") + " | " + keep.x + " " + keep.y + " " + keep.tag + + " | " + list.length + " " + sum.toFixed(3) + " " + text.length; +} +console.log("poll", poll(20000)); + +// 2. Element reads whose cold arm runs: a hole, past the end, a negative +// key, a fraction, a string key, a receiver that is not an array. +function reads(a: any, keys: any[], n: number): string { + let s = 0.5, t = 1.5, u = 2.5, w = 3.5; + let und = 0, str = 0; + for (let i = 0; i < n; i++) { + const key = keys[i % keys.length]; + const v = a[key]; + s += t * 0.5; t += u * 0.25; u += w * 0.125; w += 1; + if (v === undefined) und++; + else if (typeof v === "string") str += v.length; + else if (typeof v === "number") s += v; + else str += 100; + if (s > 1e9) { s = 0.5; t = 1.5; u = 2.5; } + } + return s.toFixed(3) + " " + t.toFixed(3) + " " + u.toFixed(3) + " " + w + " " + und + " " + str; +} +const holey: any[] = [1, 2, , 4, 5.5, "six", , 8]; +const KEYS: any[] = [0, 1, 2, 3, 4, 5, 6, 7, 8, -1, 0.5, "3", "length", 100]; +console.log("reads array", reads(holey, KEYS, 5000)); +console.log("reads string", reads("hello world", KEYS, 5000)); +console.log("reads object", reads({ 0: 7, 3: "x", length: 2 }, KEYS, 5000)); + +function pawnFiles(own: number[], n: number): number { + let passed = 0, acc = 0.25; + for (let i = 0; i < n; i++) { + const file = i & 7; + for (let f = file - 1; f <= file + 1; f++) { + const v = own[f]; + acc += 0.5; + if (v === undefined) passed += 1; + else passed += v; + } + } + return passed + acc; +} +console.log("pawn files", pawnFiles([0, 1, 0, 2, 0, 0, 3, 0], 4000)); + +// 3. Element stores whose slow path runs: growth, a hole, a kind change. +function stores(n: number): string { + const a: any[] = [1, 2, 3]; + let s = 0.5, t = 1.5, u = 2.5; + for (let i = 0; i < n; i++) { + s += t * 0.5; t += u * 0.25; u += 1; + if (i % 97 === 0) a[a.length + 2] = i; + else if (i % 89 === 0) a[i % a.length] = "s" + i; + else if (i % 83 === 0) a[i % a.length] = { i }; + else a[i % a.length] = s; + if (s > 1e9) { s = 0.5; t = 1.5; } + } + let holes = 0, strings = 0, objects = 0, numbers = 0; + for (let i = 0; i < a.length; i++) { + const v = a[i]; + if (!(i in a)) holes++; + else if (typeof v === "string") strings++; + else if (typeof v === "object") objects++; + else numbers++; + } + return a.length + " " + holes + " " + strings + " " + objects + " " + numbers + " " + s.toFixed(3) + " " + u; +} +console.log("stores", stores(6000)); + +// 4. A throw out of a cold arm, caught in the loop, with live values used in +// the handler and after it. +function throws(rows: any[], n: number): string { + let s = 0.5, t = 1.5, u = 2.5, caught = 0, last = ""; + for (let i = 0; i < n; i++) { + s += t * 0.5; t += u * 0.25; u += 1; + try { + const row = rows[i % rows.length]; + s += row.cells[i & 3]; + } catch (err) { + caught++; + last = (err as Error).constructor.name + ":" + i + ":" + t.toFixed(2); + s -= 1; + } + if (s > 1e9) { s = 0.5; t = 1.5; } + } + return s.toFixed(3) + " " + t.toFixed(3) + " " + u + " " + caught + " " + last; +} +const ROWS: any[] = [{ cells: [1, 2, 3, 4] }, { cells: [5, 6, 7, 8] }, undefined, { cells: [9, 10, 11, 12] }, null]; +console.log("throws", throws(ROWS, 3000)); + +function thrower(a: number[], i: number): number { + if (i > a.length) throw new RangeError("past " + i); + return a[i]; +} +function throwsFromCallee(n: number): string { + const a = [1.5, 2.5, 3.5]; + let s = 0.25, t = 0.75, caught = 0, und = 0; + for (let i = 0; i < n; i++) { + t += 0.5; + try { + const v = thrower(a, i % 6); + if (v === undefined) und++; else s += v; + } catch (err) { + caught++; + s += (err as Error).message.length + t; + } + } + return s.toFixed(3) + " " + t + " " + caught + " " + und; +} +console.log("throws from callee", throwsFromCallee(3000)); + +// 5. The stack guard's arm: locals of every frame stay right after the +// RangeError unwinds through them. +function deep(n: number, a: number, b: number): number { + const local = a * 0.5 + b * 0.25 + 1; + if (n === 0) return local; + return deep(n - 1, b, local) + 1; +} +console.log("deep ok", deep(2000, 1, 2).toFixed(3)); +function overflow(a: number): number { + const local = a + 0.5; + return overflow(local) + local; +} +let guard = "none", before = 1.5, after = 2.5; +try { + overflow(1); +} catch (err) { + guard = (err as Error).constructor.name; +} +console.log("deep overflow", guard, before + after, deep(100, 1, 2).toFixed(3)); + +// 6. Heap values in locals across the poll and across a cold read, while +// the loop allocates: moved objects are read through their new address. +function moved(n: number): string { + const board: any[] = []; + for (let i = 0; i < 64; i++) board.push(i % 5 === 0 ? undefined : new P(i, i * 2, "sq" + i)); + let first = board[1] as P, s = 0.5, missing = 0; + let name = ""; + for (let i = 0; i < n; i++) { + const sq = board[(i * 7) % 70]; + const fresh = new P(i, s, "f" + (i % 3)); + if (sq === undefined) { missing++; if (i % 11 === 0) board[(i * 7) % 64] = fresh; } + else { s += sq.x + sq.y * 0.5; name = sq.tag; } + if (i % 13 === 0) first = fresh; + s += first.x * 0.001; + if (s > 1e9) s = 0.5; + } + return s.toFixed(3) + " " + missing + " " + name + " " + first.x + " " + first.tag; +} +console.log("moved", moved(20000));