From 5c1c6be5533a2d5fe07646715bea7f6c001843ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 23 Oct 2026 10:54:21 +0000 Subject: [PATCH 01/12] perf(runtime): memoize getter read sites with live shape-slot proofs --- .../12986-prototype-getter-site-memo.md | 1 + crates/perry-codegen/src/codegen/closure.rs | 1 + crates/perry-codegen/src/codegen/entry.rs | 1 + .../src/codegen/entry/module_init.rs | 1 + crates/perry-codegen/src/codegen/function.rs | 1 + crates/perry-codegen/src/codegen/method.rs | 1 + .../src/codegen/method_static.rs | 1 + crates/perry-codegen/src/codegen/mod.rs | 1 + crates/perry-codegen/src/codegen/opts.rs | 23 +- .../src/codegen/opts/computed_names.rs | 19 +- .../src/expr/barrier_stem_census_tests.rs | 10 +- .../perry-codegen/src/expr/ic_fast_split.rs | 113 +++++ crates/perry-codegen/src/expr/index_get.rs | 15 +- .../src/expr/index_get/symbol_ic.rs | 41 +- crates/perry-codegen/src/expr/mod.rs | 5 + .../expr/property_get/front_contract_tests.rs | 55 +- .../src/expr/property_get/generic_dispatch.rs | 101 +++- .../expr/property_get/refusal_join_tests.rs | 4 + .../src/expr/property_get/tests.rs | 39 +- .../src/gc_effects_test_world.tsv | 2 + crates/perry-codegen/src/module/linkage.rs | 2 + crates/perry-codegen/src/module/tests.rs | 26 + .../src/native_root_coverage/ic_fast_split.rs | 77 +++ .../src/runtime_decls/objects.rs | 17 + .../perry-codegen/src/wasm32/runtime_abi.tsv | 5 + .../perry-runtime/src/object/field_get_set.rs | 3 +- .../src/object/field_get_set/get_slow.rs | 122 ++++- .../object/field_get_set/runtime_read_site.rs | 46 +- .../perry-runtime/src/object/layout_probe.rs | 41 +- .../src/object/layout_probe_site_memo.rs | 471 ++++++++++++++++++ .../object/layout_probe_site_memo_tests.rs | 348 +++++++++++++ .../src/object/node_http_prototypes.rs | 4 + crates/perry-runtime/src/runtime_state_key.rs | 49 +- ...est_gap_tsp1_prototype_getter_site_memo.ts | 67 +++ 34 files changed, 1631 insertions(+), 82 deletions(-) create mode 100644 changelog.d/12986-prototype-getter-site-memo.md create mode 100644 crates/perry-runtime/src/object/layout_probe_site_memo.rs create mode 100644 crates/perry-runtime/src/object/layout_probe_site_memo_tests.rs create mode 100644 test-files/test_gap_tsp1_prototype_getter_site_memo.ts diff --git a/changelog.d/12986-prototype-getter-site-memo.md b/changelog.d/12986-prototype-getter-site-memo.md new file mode 100644 index 00000000000..915202e0abd --- /dev/null +++ b/changelog.d/12986-prototype-getter-site-memo.md @@ -0,0 +1 @@ +Repeated property reads can reuse a per-site shape/slot answer for own and inherited getters or data properties, including fixed-name Rust runtime call sites and short computed string keys. Each use checks the live receiver and prototype shapes and loads the current holder slot; getter redefinition, shadowing, prototype swaps and Proxy chains retain ordinary property semantics without epochs or cached getter values. diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index 2cf93eb69c6..f70d50bb1d3 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -1151,6 +1151,7 @@ pub(super) fn compile_closure( let mut ctx = FnCtx { program_has_worker: cross_module.program_has_worker, program_has_thread_agents: cross_module.program_has_thread_agents, + program_class_accessor_names: cross_module.program_class_accessor_names.clone(), func: lf, module_slug: crate::expr::native_region_slug(strings.module_prefix()), source_function: format!("closure_{}", func_id), diff --git a/crates/perry-codegen/src/codegen/entry.rs b/crates/perry-codegen/src/codegen/entry.rs index 00daa61bd6c..2e385ea2143 100644 --- a/crates/perry-codegen/src/codegen/entry.rs +++ b/crates/perry-codegen/src/codegen/entry.rs @@ -734,6 +734,7 @@ pub(super) fn compile_module_entry( let mut ctx = FnCtx { program_has_worker: cross_module.program_has_worker, program_has_thread_agents: cross_module.program_has_thread_agents, + program_class_accessor_names: cross_module.program_class_accessor_names.clone(), func: main, module_slug: crate::expr::native_region_slug(strings.module_prefix()), source_function: "module_init".to_string(), diff --git a/crates/perry-codegen/src/codegen/entry/module_init.rs b/crates/perry-codegen/src/codegen/entry/module_init.rs index 81a946076e4..0102d54de2e 100644 --- a/crates/perry-codegen/src/codegen/entry/module_init.rs +++ b/crates/perry-codegen/src/codegen/entry/module_init.rs @@ -272,6 +272,7 @@ pub(super) fn compile_module_init( let mut ctx = FnCtx { program_has_worker: cross_module.program_has_worker, program_has_thread_agents: cross_module.program_has_thread_agents, + program_class_accessor_names: cross_module.program_class_accessor_names.clone(), func: init_fn, module_slug: crate::expr::native_region_slug(strings.module_prefix()), source_function: "module_init".to_string(), diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index 8d902d3ba53..ef84fc5c0de 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -1273,6 +1273,7 @@ pub(super) fn compile_function( let mut ctx = FnCtx { program_has_worker: cross_module.program_has_worker, program_has_thread_agents: cross_module.program_has_thread_agents, + program_class_accessor_names: cross_module.program_class_accessor_names.clone(), func: lf, module_slug: crate::expr::native_region_slug(strings.module_prefix()), source_function: f.name.clone(), diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index 1d493367d28..0702a05ceed 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -572,6 +572,7 @@ pub(super) fn compile_method( let mut ctx = FnCtx { program_has_worker: cross_module.program_has_worker, program_has_thread_agents: cross_module.program_has_thread_agents, + program_class_accessor_names: cross_module.program_class_accessor_names.clone(), func: lf, module_slug: crate::expr::native_region_slug(strings.module_prefix()), source_function: format!("{}.{}", class.name, method.name), diff --git a/crates/perry-codegen/src/codegen/method_static.rs b/crates/perry-codegen/src/codegen/method_static.rs index 710ee0a24f8..8b4c1ac1f3f 100644 --- a/crates/perry-codegen/src/codegen/method_static.rs +++ b/crates/perry-codegen/src/codegen/method_static.rs @@ -244,6 +244,7 @@ pub(in crate::codegen) fn compile_static_method( let mut ctx = FnCtx { program_has_worker: cross_module.program_has_worker, program_has_thread_agents: cross_module.program_has_thread_agents, + program_class_accessor_names: cross_module.program_class_accessor_names.clone(), func: lf, module_slug: crate::expr::native_region_slug(strings.module_prefix()), source_function: format!("{}.{}", class.name, f.name), diff --git a/crates/perry-codegen/src/codegen/mod.rs b/crates/perry-codegen/src/codegen/mod.rs index 96bde75eac6..917a7299f4c 100644 --- a/crates/perry-codegen/src/codegen/mod.rs +++ b/crates/perry-codegen/src/codegen/mod.rs @@ -2930,6 +2930,7 @@ fn lower_module( imported_async_funcs: opts.imported_async_funcs, short_spread_method_candidates: Arc::clone(&opts.short_spread_method_candidates), object_literal_method_candidates: Arc::clone(&opts.object_literal_method_candidates), + program_class_accessor_names: opts.program_class_accessor_names.clone(), local_async_funcs, local_generator_funcs, fixed_generator_funcs, diff --git a/crates/perry-codegen/src/codegen/opts.rs b/crates/perry-codegen/src/codegen/opts.rs index 1e491effda7..45afb85d6c8 100644 --- a/crates/perry-codegen/src/codegen/opts.rs +++ b/crates/perry-codegen/src/codegen/opts.rs @@ -128,6 +128,10 @@ pub fn namespace_member_func_key(namespace: &str, member: &str) -> String { pub struct ClassAccessorNames { getters: std::collections::HashSet, setters: std::collections::HashSet, + /// Getter names an object literal or a literal-keyed + /// `Object.defineProperty` declares. They never widen [`Self::may_get`]; + /// they only give a read of that name its shape/slot memo. + literal_getters: std::collections::HashSet, } mod computed_names; @@ -157,6 +161,14 @@ impl ClassAccessorNames { self.getters.contains(name) } + /// May any getter of the program be declared under `name` (a class, an + /// object literal or a literal-keyed `Object.defineProperty`)? Only such + /// a fixed-name read owns a shape/slot memo; every other read keeps the + /// stateless probe and complete read. + pub fn may_read_through_getter(&self, name: &str) -> bool { + self.getters.contains(name) || self.literal_getters.contains(name) + } + /// May a compiled class declare a setter named `name`? pub fn may_set(&self, name: &str) -> bool { self.setters.contains(name) @@ -166,9 +178,16 @@ impl ClassAccessorNames { pub fn cache_key(&self) -> String { let mut getters: Vec<&str> = self.getters.iter().map(String::as_str).collect(); let mut setters: Vec<&str> = self.setters.iter().map(String::as_str).collect(); + let mut literal: Vec<&str> = self.literal_getters.iter().map(String::as_str).collect(); getters.sort_unstable(); setters.sort_unstable(); - format!("get:{}|set:{}", getters.join(","), setters.join(",")) + literal.sort_unstable(); + format!( + "get:{}|set:{}|lit:{}", + getters.join(","), + setters.join(","), + literal.join(",") + ) } } @@ -970,6 +989,8 @@ pub(crate) struct CrossModuleCtx { std::sync::Arc>>, pub object_literal_method_candidates: std::sync::Arc>>, + /// The whole program class accessor names, carried from the compile options. + pub program_class_accessor_names: Option>, /// FuncIds of locally-defined async functions in this module. Populated /// from `hir.functions.is_async`. Used by `is_promise_expr` to refine /// `let p = asyncFn();` to `Promise(_)` so subsequent `p.then(cb)` diff --git a/crates/perry-codegen/src/codegen/opts/computed_names.rs b/crates/perry-codegen/src/codegen/opts/computed_names.rs index 9d522a1728a..78c9907a24f 100644 --- a/crates/perry-codegen/src/codegen/opts/computed_names.rs +++ b/crates/perry-codegen/src/codegen/opts/computed_names.rs @@ -11,6 +11,7 @@ struct Facts { returns: HashMap>, properties: Vec<(String, Expr)>, accessors: Vec<(Expr, bool, bool)>, + literal_getters: Vec, class_fields: HashMap>, visited_functions: HashSet, } @@ -120,6 +121,15 @@ impl Facts { .extend(fields.iter().cloned().zip(args.iter().cloned())); } } + Expr::ObjectDefineProperty(_, key, descriptor) => { + if let (Expr::String(key), Expr::Object(fields)) = + (key.as_ref(), descriptor.as_ref()) + { + if fields.iter().any(|(field, _)| field == "get") { + self.literal_getters.push(key.clone()); + } + } + } Expr::Call { callee, args, .. } => { if let Expr::ExternFuncRef { name, .. } = callee.as_ref() { if name == "js_object_define_accessor" && args.len() == 4 { @@ -127,7 +137,11 @@ impl Facts { // path. Extend admission for computed expressions; // widening unrelated named object reads adds miss // guards without helping the computed-key workload. - if !matches!(args[1], Expr::String(_)) { + if let Expr::String(key) = &args[1] { + if !matches!(args[2], Expr::Undefined) { + self.literal_getters.push(key.clone()); + } + } else { self.accessors.push(( args[1].clone(), !matches!(args[2], Expr::Undefined), @@ -248,6 +262,9 @@ pub(super) fn collect<'m>(modules: impl IntoIterator) -> Clas properties = next; } for f in &facts { + names + .literal_getters + .extend(f.literal_getters.iter().cloned()); for (key, get, set) in &f.accessors { let candidates = f.names(key, &properties, 0); if *get { diff --git a/crates/perry-codegen/src/expr/barrier_stem_census_tests.rs b/crates/perry-codegen/src/expr/barrier_stem_census_tests.rs index 075a87ba504..d63a0951e50 100644 --- a/crates/perry-codegen/src/expr/barrier_stem_census_tests.rs +++ b/crates/perry-codegen/src/expr/barrier_stem_census_tests.rs @@ -1402,8 +1402,14 @@ fn generator_fixed_slot_store_uses_the_shared_barrier_operation() { "no per-site barrier scaffold" ); assert!( - (ir.contains("call double @js_object_get_field_value(") - || ir.contains("call double @js_object_get_field_slow(")), + [ + "js_object_get_field_value", + "js_object_get_field_slow", + "js_object_get_field_value_site", + "js_object_get_field_site_slow", + ] + .iter() + .any(|exit| ir.contains(&format!("call double @{exit}("))), "generic continuation reads call the complete collecting operation" ); assert!( diff --git a/crates/perry-codegen/src/expr/ic_fast_split.rs b/crates/perry-codegen/src/expr/ic_fast_split.rs index a975b996afb..619699cdd8f 100644 --- a/crates/perry-codegen/src/expr/ic_fast_split.rs +++ b/crates/perry-codegen/src/expr/ic_fast_split.rs @@ -144,3 +144,116 @@ pub(crate) fn emit_checked_callee_unbox(ctx: &mut FnCtx<'_>, callee: &str) -> St ], ) } + +/// Does a fixed-name read of `property` own a shape/slot memo? Only a name +/// some getter of the program is declared under can reach the collecting +/// exit the memo serves; any other read keeps the stateless probe and owns +/// no storage. A compile without the program's accessor names gives every +/// read its memo. +pub(crate) fn read_may_reach_getter(ctx: &FnCtx<'_>, property: &str) -> bool { + ctx.program_class_accessor_names + .as_ref() + .map_or(true, |names| names.may_read_through_getter(property)) +} + +/// Agent-local storage for a site's shape/slot answers. No heap pointer or +/// validity state is stored, so these zero-filled integer words need no root. +pub(crate) fn emit_read_site_memo(ctx: &mut FnCtx<'_>, words: usize) -> String { + let site = ctx.ic_site_counter; + ctx.ic_site_counter += 1; + let name = format!( + "@{}_read_shape_slots", + crate::expr::inline_cache_global_name(ctx, site) + ); + let tls = if ctx.program_has_worker || ctx.program_has_thread_agents { + "thread_local " + } else { + "" + }; + ctx.typed_parse_rodata.push(format!( + "{name} = private {tls}global [{words} x i64] zeroinitializer, align 8" + )); + name +} + +#[cfg(test)] +mod tests { + #[test] + fn only_a_declared_getter_name_owns_read_site_storage() { + use perry_hir::{Expr, Module, Stmt}; + for (getter, memo) in [("width", true), ("height", false)] { + let mut module = Module::new("read_site_names.ts"); + module.init.push(Stmt::Expr(Expr::PropertyGet { + object: Box::new(Expr::GlobalGet(100)), + property: "width".into(), + byte_offset: 0, + })); + let names = crate::codegen::ClassAccessorNames::from_names( + [getter.to_string()], + std::iter::empty::(), + ); + let ir = String::from_utf8( + crate::compile_module( + &module, + crate::CompileOptions { + emit_ir_only: true, + program_class_accessor_names: Some(std::sync::Arc::new(names)), + ..Default::default() + }, + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(ir.contains("_read_shape_slots = private"), memo, "{ir}"); + assert_eq!( + ir.contains("call double @perry_probe_get_site("), + memo, + "{ir}" + ); + // Declarations name every entrance; only the calls differ. + let calls = |callee: &str| { + ir.lines() + .any(|l| !l.starts_with("declare") && l.contains(&format!("@{callee}("))) + }; + assert_eq!(calls("js_object_get_field_value_site"), memo, "{ir}"); + assert_eq!(calls("perry_probe_get"), !memo, "{ir}"); + assert_eq!(calls("js_object_get_field_value"), !memo, "{ir}"); + } + } + + #[test] + fn read_site_storage_respects_agent_multiplicity() { + use perry_hir::{Expr, Module, Stmt}; + for (worker, thread_agent) in [(false, false), (true, false), (false, true), (true, true)] { + let mut module = Module::new("read_site_agents.ts"); + module.init.push(Stmt::Expr(Expr::PropertyGet { + object: Box::new(Expr::GlobalGet(100)), + property: "width".into(), + byte_offset: 0, + })); + let ir = String::from_utf8( + crate::compile_module( + &module, + crate::CompileOptions { + emit_ir_only: true, + program_has_worker: worker, + program_has_thread_agents: thread_agent, + ..Default::default() + }, + ) + .unwrap(), + ) + .unwrap(); + assert!(ir.contains("call double @perry_probe_get_site("), "{ir}"); + let declaration = ir + .lines() + .find(|line| line.contains("_read_shape_slots = private")) + .expect("a generic read must own its shape/slot storage"); + assert_eq!( + declaration.contains("thread_local"), + worker || thread_agent, + "{declaration}" + ); + } + } +} diff --git a/crates/perry-codegen/src/expr/index_get.rs b/crates/perry-codegen/src/expr/index_get.rs index 4cb561bc8a5..ae8df03fd88 100644 --- a/crates/perry-codegen/src/expr/index_get.rs +++ b/crates/perry-codegen/src/expr/index_get.rs @@ -1628,9 +1628,14 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // layout and its chain answer the key's atom // (`perry_probe_get_keyed`, a GC leaf); only a decline // takes the complete keyed read. - Ok(symbol_ic::emit_keyed_probe(ctx, obj_box, key_box, |ctx| { - symbol_ic::keyed_slow(ctx, obj_box, key_box, &site_id, false) - })) + Ok(symbol_ic::emit_keyed_probe( + ctx, + obj_box, + key_box, + |ctx, memo| { + symbol_ic::keyed_site_slow(ctx, obj_box, key_box, &site_id, false, memo) + }, + )) }); } // Last-resort fallback with runtime tag checks on the index. @@ -1737,8 +1742,8 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // takes the by-value read (see the dynamic-string arm above). let _ = str_obj_handle; // The probe first (§3.7 case 3), as the dynamic-string arm. - let v_str = symbol_ic::emit_keyed_probe(ctx, &obj_box, &idx_box, |ctx| { - symbol_ic::keyed_slow(ctx, &obj_box, &idx_box, &site_id, false) + let v_str = symbol_ic::emit_keyed_probe(ctx, &obj_box, &idx_box, |ctx, memo| { + symbol_ic::keyed_site_slow(ctx, &obj_box, &idx_box, &site_id, false, memo) }); let str_end_lbl = ctx.block().label.clone(); ctx.block().br(&merge_lbl); diff --git a/crates/perry-codegen/src/expr/index_get/symbol_ic.rs b/crates/perry-codegen/src/expr/index_get/symbol_ic.rs index 4b5c8c1d8e4..2ded85ec050 100644 --- a/crates/perry-codegen/src/expr/index_get/symbol_ic.rs +++ b/crates/perry-codegen/src/expr/index_get/symbol_ic.rs @@ -16,8 +16,8 @@ pub(crate) fn lower_symbol_property_get_ic( // Object model step 5 (§3.7 case 3): the receiver's own layout answers // the symbol key (`perry_probe_get_keyed`); a decline takes the complete // keyed read. - emit_keyed_probe(ctx, obj_box, sym_box, |ctx| { - keyed_slow(ctx, obj_box, sym_box, "0", false) + emit_keyed_probe(ctx, obj_box, sym_box, |ctx, memo| { + keyed_site_slow(ctx, obj_box, sym_box, "0", false, memo) }) } @@ -42,7 +42,9 @@ pub(super) fn lower_unknown_key_get(ctx: &mut FnCtx<'_>, obj: &str, key: &str) - // Object model step 5 (§3.7 case 3): the receiver's own layout answers a // string or symbol key through the key's atom (`perry_probe_get_keyed`); // only a decline calls the complete keyed read. - let named_value = emit_keyed_probe(ctx, obj, key, |ctx| keyed_slow(ctx, obj, key, "0", true)); + let named_value = emit_keyed_probe(ctx, obj, key, |ctx, memo| { + keyed_site_slow(ctx, obj, key, "0", true, memo) + }); let named_end = ctx.block().label.clone(); ctx.block().br(&merge_l); ctx.current_block = numeric; @@ -62,8 +64,11 @@ pub(crate) fn emit_keyed_probe( ctx: &mut FnCtx<'_>, obj: &str, key: &str, - slow: impl FnOnce(&mut FnCtx<'_>) -> String, + slow: impl FnOnce(&mut FnCtx<'_>, &str) -> String, ) -> String { + if crate::target_layout::target_is_ilp32(ctx.target_triple) { + return slow(ctx, "null"); + } let dir = crate::expr::agent_ptr::emit_agent_ptr_or( ctx, crate::runtime_abi::AGENT_PTR_SHAPE_DIR, @@ -71,6 +76,8 @@ pub(crate) fn emit_keyed_probe( ); let obj_bits = ctx.block().bitcast_double_to_i64(obj); let key_bits = ctx.block().bitcast_double_to_i64(key); + // The leaf stays stateless; only its decline consults the site memo. + let memo = crate::expr::ic_fast_split::emit_read_site_memo(ctx, 24); let probed = ctx.block().call( DOUBLE, "perry_probe_get_keyed", @@ -87,7 +94,7 @@ pub(crate) fn emit_keyed_probe( let probe_end = ctx.block().label.clone(); ctx.block().cond_br(&served, &merge_l, &slow_l); ctx.current_block = slow_idx; - let slow_value = slow(ctx); + let slow_value = slow(ctx, &memo); let slow_end = ctx.block().label.clone(); ctx.block().br(&merge_l); ctx.current_block = merge_idx; @@ -118,3 +125,27 @@ pub(crate) fn keyed_slow( ], ) } + +pub(crate) fn keyed_site_slow( + ctx: &mut FnCtx<'_>, + obj: &str, + key: &str, + site_id: &str, + dynamic: bool, + memo: &str, +) -> String { + if memo == "null" { + return keyed_slow(ctx, obj, key, site_id, dynamic); + } + ctx.block().call( + DOUBLE, + "js_object_get_keyed_site_slow", + &[ + (DOUBLE, obj), + (DOUBLE, key), + (I64, site_id), + (crate::types::I32, if dynamic { "1" } else { "0" }), + (PTR, memo), + ], + ) +} diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index 595f668de78..65698e8257b 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -398,6 +398,11 @@ pub(crate) struct FnCtx<'a> { pub wpfacts_off: bool, pub program_has_worker: bool, pub program_has_thread_agents: bool, + /// The program's accessor names (`ClassAccessorNames`): a fixed-name read + /// owns a shape/slot memo only for a name some getter of the program is + /// declared under. `None` (a compile that did not collect them) gives + /// every fixed-name read its memo. + pub program_class_accessor_names: Option>, /// Function being built (blocks, params, registers). pub func: &'a mut LlFunction, /// Stable slug for native-region ids derived from this module. diff --git a/crates/perry-codegen/src/expr/property_get/front_contract_tests.rs b/crates/perry-codegen/src/expr/property_get/front_contract_tests.rs index 741b8a3fc3d..45685bdb191 100644 --- a/crates/perry-codegen/src/expr/property_get/front_contract_tests.rs +++ b/crates/perry-codegen/src/expr/property_get/front_contract_tests.rs @@ -1,4 +1,4 @@ -//! The generic read's shape probe (`perry_probe_get`) takes the agent's shape +//! The generic read's shape probe (`perry_probe_get_site`) takes the agent's shape //! directory, whose lookup is target-specific (extra blocks on Apple aarch64, //! a TEB-derived chain on Windows). Follow the CFG and the call operand //! rather than assuming a single block or ELF. @@ -28,7 +28,7 @@ fn verify_front_flow(blocks: &Blocks) -> Result { .enumerate() .filter(|(_, (_, body))| { body.iter() - .any(|line| line.contains("call double @perry_probe_get(")) + .any(|line| line.contains("call double @perry_probe_get_site(")) }) .collect(); let [(index, (call_label, call_body))] = calls.as_slice() else { @@ -99,7 +99,7 @@ fn verify_front_flow(blocks: &Blocks) -> Result { let (cond, served, declined) = tower_cond_br(call_body); let call = call_body .iter() - .find(|line| line.contains("call double @perry_probe_get(")) + .find(|line| line.contains("call double @perry_probe_get_site(")) .unwrap(); let answer = call.split_once(" = ").unwrap().0; let bits = call_body @@ -140,6 +140,37 @@ fn verify_front_flow(blocks: &Blocks) -> Result { Ok(*index) } +/// Both halves of a fixed read must use this site's integer-only storage. +pub(super) fn verify_front_site_memo(blocks: &Blocks) -> Result { + let index = verify_front_flow(blocks)?; + let call = blocks[index] + .1 + .iter() + .find(|line| line.contains("call double @perry_probe_get_site(")) + .unwrap(); + let memo = call + .rsplit_once(", ptr ") + .and_then(|(_, tail)| tail.split_once(')')) + .map(|(memo, _)| memo) + .ok_or("the probe must receive its site storage")?; + if !memo.starts_with("@perry_ic_") || !memo.ends_with("_read_shape_slots") { + return Err(format!( + "the memo must be this site's shape/slot storage: {memo}" + )); + } + let slow: Vec<_> = blocks + .iter() + .flat_map(|(_, body)| body) + .filter(|line| line.contains("@js_object_get_field_value_site(")) + .collect(); + if slow.len() != 1 || !slow[0].contains(&format!(", ptr {memo})")) { + return Err(format!( + "the leaf and collecting exit must share one memo: {slow:?}" + )); + } + Ok(memo.to_string()) +} + pub(super) fn front_call_block(blocks: &Blocks) -> (&str, &[String]) { let index = verify_front_flow(blocks).unwrap_or_else(|e| panic!("{e}: {blocks:?}")); (&blocks[index].0, &blocks[index].1) @@ -164,7 +195,7 @@ pub(super) fn verify_front_directory(blocks: &Blocks) -> Result<(), String> { let call = blocks[index] .1 .iter() - .find(|line| line.contains("call double @perry_probe_get(")) + .find(|line| line.contains("call double @perry_probe_get_site(")) .unwrap(); let dir = call .split_once("(ptr ") @@ -315,6 +346,20 @@ fn front_contract_rejects_lookup_bypasses_wrong_slots_and_wrong_declines() { let blocks = tower_blocks(&ir); let index = verify_front_flow(&blocks).unwrap(); verify_front_directory(&blocks).unwrap(); + let memo = verify_front_site_memo(&blocks).unwrap(); + let mut wrong_memo = blocks.clone(); + for (_, body) in &mut wrong_memo { + for line in body { + if line.contains("@js_object_get_field_value_site(") { + *line = line.replace(&format!(", ptr {memo})"), ", ptr null)"); + } + } + } + assert_ne!(wrong_memo, blocks, "{target}: memo sabotage must alter IR"); + assert!( + verify_front_site_memo(&wrong_memo).is_err(), + "{target}: disconnected slow memo" + ); let mut wrong = blocks.clone(); let cond = tower_cond_br(&wrong[index].1).0; let term = wrong[index].1.last_mut().unwrap(); @@ -340,7 +385,7 @@ fn front_contract_rejects_lookup_bypasses_wrong_slots_and_wrong_declines() { let call = wrong[index] .1 .iter_mut() - .find(|l| l.contains("call double @perry_probe_get(")) + .find(|l| l.contains("call double @perry_probe_get_site(")) .unwrap(); let operand = call .split_once("(ptr ") diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 173ef13ca99..819d0489d1f 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -1,6 +1,10 @@ //! Ordinary property-get dispatch: static family slots, coloured layout //! slots, then the receiver layout's probe and complete property semantics. -//! These read paths allocate no learned per-site state. +//! A fixed-name read of a name some getter of the program is declared under +//! owns bounded shape/slot answers: its leaf checks them only after an own +//! miss, and its collecting exit derives them. Every other read keeps the +//! stateless probe. Static family and colour slots retain their direct +//! layout checks. use super::*; @@ -626,19 +630,38 @@ fn lower_property_site_tail( let key_box = ctx.block().load(DOUBLE, &key_handle_global); let key_bits = ctx.block().bitcast_double_to_i64(&key_box); let hash = (crate::layout_desc::atom_hash(property) as i64).to_string(); + let memo = (!crate::target_layout::target_is_ilp32(ctx.target_triple) + && !property.starts_with('#') + && property != "__proto__" + && crate::expr::ic_fast_split::read_may_reach_getter(ctx, property)) + .then(|| crate::expr::ic_fast_split::emit_read_site_memo(ctx, 5)); + let mut fast_args = vec![ + (PTR, dir.as_str()), + (I64, obj_bits.as_str()), + (I64, key_bits.as_str()), + (I64, hash.as_str()), + ]; + let mut slow_args = entry_args.to_vec(); + if let Some(memo) = memo.as_ref() { + fast_args.push((PTR, memo.as_str())); + slow_args.push((PTR, memo.as_str())); + } crate::expr::note_site("probe-outlined"); return Ok(crate::expr::ic_fast_split::emit_hole_declining_split( ctx, "pget.outline", - "perry_probe_get_value", - &[ - (PTR, &dir), - (I64, &obj_bits), - (I64, &key_bits), - (I64, &hash), - ], - "js_object_get_field_value", - &entry_args, + if memo.is_some() { + "perry_probe_get_value_site" + } else { + "perry_probe_get_value" + }, + &fast_args, + if memo.is_some() { + "js_object_get_field_value_site" + } else { + "js_object_get_field_value" + }, + &slow_args, )); } @@ -646,7 +669,8 @@ fn lower_property_site_tail( // // receiver test -> [`.size` on a native Map/Set] -> [`.length` on a plain // Array or a heap string] -> the unknown-receiver probe -> on a decline, - // the complete read entry. Nothing at the site is learned or remembered. + // the complete read entry. The per-site shape/slot proof is shared by + // both halves; fixed layout arms above remain direct. // // `.length` admits both pointer-ish tags (POINTER and STRING) because it // has a heap-string arm; every other key takes the fused POINTER range @@ -843,6 +867,23 @@ fn lower_property_site_tail( // only answers the probe finds, never absence. let index_name = !property.is_empty() && property.bytes().all(|b| b.is_ascii_digit()); let private_name = property.starts_with('#'); + let memo = (!crate::target_layout::target_is_ilp32(ctx.target_triple) + && !global_binding + && !index_name + && !private_name + && !inline_string_length + && property != "__proto__" + && crate::expr::ic_fast_split::read_may_reach_getter(ctx, property)) + .then(|| crate::expr::ic_fast_split::emit_read_site_memo(ctx, 5)); + let mut probe_args = vec![ + (PTR, dir.as_str()), + (I64, probe_recv.as_str()), + (I64, key_bits.as_str()), + (I64, hash.as_str()), + ]; + if let Some(memo) = memo.as_ref() { + probe_args.push((PTR, memo.as_str())); + } crate::expr::note_site("probe"); let answered = ctx.block().call( DOUBLE, @@ -851,15 +892,12 @@ fn lower_property_site_tail( "perry_probe_get_binding" } else if inline_string_length { "perry_probe_get_length" + } else if memo.is_some() { + "perry_probe_get_site" } else { "perry_probe_get" }, - &[ - (PTR, &dir), - (I64, &probe_recv), - (I64, &key_bits), - (I64, &hash), - ], + &probe_args, ); let answered_bits = ctx.block().bitcast_double_to_i64(&answered); let served = ctx @@ -877,25 +915,40 @@ fn lower_property_site_tail( let val_miss = if shared_refusal { // Preserve the original boxed receiver: the shared entry classifies // primitives and native handles as well as declined object pointers. + // A site that owns a memo takes the same entry's memo-aware form. + let mut args = vec![ + (I64, obj_bits.as_str()), + (I64, miss_key_handle.as_str()), + (I64, feedback_site_id.as_str()), + ]; + if let Some(memo) = memo.as_ref() { + args.push((PTR, memo.as_str())); + } ctx.block().call( DOUBLE, - "js_object_get_field_value", - &[ - (I64, &obj_bits), - (I64, &miss_key_handle), - (I64, &feedback_site_id), - ], + if memo.is_some() { + "js_object_get_field_value_site" + } else { + "js_object_get_field_value" + }, + &args, ) } else { let miss_handle = recv_handle(ctx, fused_recv.as_ref(), &entry_handle); + let mut args = vec![(I64, miss_handle.as_str()), (I64, miss_key_handle.as_str())]; + if let Some(memo) = memo.as_ref() { + args.push((PTR, memo.as_str())); + } ctx.block().call( DOUBLE, if global_binding { "js_global_get_field_slow" + } else if memo.is_some() { + "js_object_get_field_site_slow" } else { "js_object_get_field_slow" }, - &[(I64, &miss_handle), (I64, &miss_key_handle)], + &args, ) }; let miss_end_label = ctx.block().label.clone(); diff --git a/crates/perry-codegen/src/expr/property_get/refusal_join_tests.rs b/crates/perry-codegen/src/expr/property_get/refusal_join_tests.rs index 115e0627cef..9170c67e8f5 100644 --- a/crates/perry-codegen/src/expr/property_get/refusal_join_tests.rs +++ b/crates/perry-codegen/src/expr/property_get/refusal_join_tests.rs @@ -21,6 +21,10 @@ fn emit(properties: &[&str], colored: bool, target: &str) -> String { } let mut opts = ir_opts(false, None); opts.target = Some(target.into()); + // No getter of the program is declared under these names: the reads + // keep the stateless probe and own no shape/slot memo. + opts.program_class_accessor_names = + Some(std::sync::Arc::new(crate::ClassAccessorNames::default())); if colored { let mut layouts = crate::ProgramLayouts::default(); layouts.colored.insert(properties[0].into(), (0, 0)); diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index e0ecb74c851..19271fe97fc 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -171,7 +171,7 @@ fn imported_variable_read_preserves_class_tags_and_calls_the_live_getter_once() // would lose the 0x7FFE tag and the class would dispatch as an object. assert!( ir.lines().any(|line| { - line.contains("call double @js_object_get_field_value(") + line.contains("call double @js_object_get_field_value_site(") && bits.iter().any(|b| line.contains(&format!("(i64 {b},"))) }), "class dispatch must receive the getter's unmasked value bits:\n{ir}" @@ -223,11 +223,11 @@ fn no_call_location_without_debug_symbols() { ); } -/// The generic property read holds no per-site state: the hit is ONE leaf -/// call of the receiver layout's probe (`perry_probe_get`) over the agent's -/// shape directory, the key and the receiver's handle. There is no inline -/// ShapeId compare against a site cache, no way token, no epoch and no -/// `@perry_ic_*` site global; the receiver's own shape is the authority. +/// The generic property read's hit is ONE leaf call of the receiver layout's +/// probe (`perry_probe_get_site`) over the agent's shape directory, the key, +/// the receiver's handle and the site's five integer shape/slot pairs. There +/// is no inline ShapeId compare, no way token and no epoch; the receiver's +/// own shape is the authority. #[test] fn generic_property_get_hit_path_is_shape_id_only() { let ir = emit(false, None); @@ -235,7 +235,7 @@ fn generic_property_get_hit_path_is_shape_id_only() { let (_, ok) = tower_block(&blocks, "pget.recv_ok"); let probes: Vec<&String> = ok .iter() - .filter(|l| l.contains("call double @perry_probe_get(")) + .filter(|l| l.contains("call double @perry_probe_get_site(")) .collect(); assert_eq!(probes.len(), 1, "the hit is one probe call: {ok:?}"); assert!( @@ -245,8 +245,11 @@ fn generic_property_get_hit_path_is_shape_id_only() { ShapeId: {ok:?}" ); assert!( - !ir.contains("@perry_ic_"), - "a generic read must not own a site cache global:\n{ir}" + ir.contains(&format!( + "{} = private global [5 x i64] zeroinitializer, align 8", + front_contract::verify_front_site_memo(&blocks).unwrap() + )), + "the read must own exactly five zero-initialized shape/slot pairs:\n{ir}" ); assert!( !ir.contains("4611686018427387904"), @@ -452,13 +455,13 @@ fn generic_property_get_tries_ways_before_calling_the_miss_handler() { assert!( probe_block .iter() - .any(|l| l.contains("call double @perry_probe_get(")), + .any(|l| l.contains("call double @perry_probe_get_site(")), "{probe_block:?}" ); assert!( !probe_block .iter() - .any(|l| l.contains("@js_object_get_field_value(")), + .any(|l| l.contains("@js_object_get_field_value_site(")), "the slow call must not sit inside the probe block: {probe_block:?}" ); let (_, served, declined) = tower_cond_br(probe_block); @@ -495,7 +498,7 @@ fn generic_refusal_keeps_boxed_receiver_and_reloads_key_on_the_join() { assert_eq!(slow.iter().filter(|l| l.contains(" call ")).count(), 1); let call = slow .iter() - .find(|l| l.contains("@js_object_get_field_value(")) + .find(|l| l.contains("@js_object_get_field_value_site(")) .unwrap(); let bits = call .split_once("(i64 ") @@ -524,8 +527,8 @@ fn generic_refusal_keeps_boxed_receiver_and_reloads_key_on_the_join() { "the removed keys-pointer epoch global must not appear" ); assert!( - !all.iter().any(|l| l.contains("@perry_ic_")), - "the read holds no site state" + front_contract::verify_front_site_memo(&blocks).is_ok(), + "the refusal must pass the leaf's own shape/slot storage" ); } @@ -541,7 +544,7 @@ fn a_spill_entry_is_served_by_the_leaf_front_before_the_slow_call() { let (probe_label, probe_block) = front_call_block(&blocks); let call = probe_block .iter() - .find(|l| l.contains("@perry_probe_get(")) + .find(|l| l.contains("@perry_probe_get_site(")) .unwrap_or_else(|| panic!("the probe call: {probe_block:?}")); let answer = call.split_once(" = ").map(|(r, _)| r).unwrap(); let (_, merge) = tower_block(&blocks, "pget.recv_merge"); @@ -798,13 +801,13 @@ fn the_generic_tower_is_one_leaf_call_one_exit_and_a_bounded_number_of_blocks() sorted.sort(); assert_eq!( sorted, - vec!["js_object_get_field_value", "perry_probe_get"], + vec!["js_object_get_field_value_site", "perry_probe_get_site"], "the read must expand the probe and exactly one collecting exit:\n{func}" ); // The probe is nounwind and GC-leaf: a plain call, never an invoke. let probes: Vec<&str> = func .lines() - .filter(|l| l.contains("@perry_probe_get(")) + .filter(|l| l.contains("@perry_probe_get_site(")) .collect(); assert_eq!(probes.len(), 1, "one probe call per read:\n{func}"); assert!( @@ -865,7 +868,7 @@ fn the_generic_shared_read_receives_boxed_receiver_key_and_feedback() { .iter() .flat_map(|(l, body)| { body.iter() - .filter(|t| t.contains("@js_object_get_field_value(")) + .filter(|t| t.contains("@js_object_get_field_value_site(")) .map(move |t| (l.as_str(), t)) }) .collect(); diff --git a/crates/perry-codegen/src/gc_effects_test_world.tsv b/crates/perry-codegen/src/gc_effects_test_world.tsv index f5536035097..da6c73ffe56 100644 --- a/crates/perry-codegen/src/gc_effects_test_world.tsv +++ b/crates/perry-codegen/src/gc_effects_test_world.tsv @@ -67,7 +67,9 @@ perry_array_iteration_init Leaf no perry_class_value_dir_cell Leaf no perry_intrinsic_proto Leaf no perry_probe_get Leaf no +perry_probe_get_site Leaf no perry_probe_get_keyed Leaf no perry_probe_get_length Leaf no perry_probe_get_value Leaf no +perry_probe_get_value_site Leaf no perry_probe_set Leaf no diff --git a/crates/perry-codegen/src/module/linkage.rs b/crates/perry-codegen/src/module/linkage.rs index 968aaa633f4..533cb9029a4 100644 --- a/crates/perry-codegen/src/module/linkage.rs +++ b/crates/perry-codegen/src/module/linkage.rs @@ -402,6 +402,8 @@ const COLD_HELPERS: &[&str] = &[ // `cold` is only a profitability hint: both calls remain fully // memory-clobbering and the cache miss remains GC-capable/throwing. "js_object_get_field_slow", + "js_object_get_field_site_slow", + "js_object_get_field_value_site", "js_object_get_field_nonptr", "js_object_get_field_value", "js_global_get_field_slow", diff --git a/crates/perry-codegen/src/module/tests.rs b/crates/perry-codegen/src/module/tests.rs index 7685b77d691..effbce96098 100644 --- a/crates/perry-codegen/src/module/tests.rs +++ b/crates/perry-codegen/src/module/tests.rs @@ -1109,3 +1109,29 @@ fn unicode_symbol_names_survive_ir_reference_parsing() { assert!(refs.contains(&reference)); assert!(refs.contains(&format!("{reference}$generic"))); } + +#[test] +fn read_site_collecting_declarations_preserve_the_original_contract() { + use crate::types::PTR; + let mut m = module_under( + "js_object_get_field_site_slow\tReenters\tyes\n\ + js_object_get_field_value_site\tReenters\tyes\n", + ); + m.declare_function("js_object_get_field_site_slow", DOUBLE, &[I64, I64, PTR]); + m.declare_function( + "js_object_get_field_value_site", + DOUBLE, + &[I64, I64, I64, PTR], + ); + let f = m.define_function("main", I32, vec![]); + f.create_block("entry").ret(I32, "0"); + let ir = m.to_ir(); + assert!( + ir.contains("declare double @js_object_get_field_site_slow(i64, i64, ptr) cold\n"), + "{ir}" + ); + assert!( + ir.contains("declare double @js_object_get_field_value_site(i64, i64, i64, ptr) cold\n"), + "{ir}" + ); +} diff --git a/crates/perry-codegen/src/native_root_coverage/ic_fast_split.rs b/crates/perry-codegen/src/native_root_coverage/ic_fast_split.rs index 2fdc5125e3b..c7ce08937dc 100644 --- a/crates/perry-codegen/src/native_root_coverage/ic_fast_split.rs +++ b/crates/perry-codegen/src/native_root_coverage/ic_fast_split.rs @@ -261,3 +261,80 @@ fn inline_splits_call_their_helper_only_from_the_cold_block() { } } } + +#[test] +fn read_site_memos_preserve_leaf_and_collecting_root_contracts() { + for outlined in [false, true] { + let mut module = split_module(); + if !outlined { + module.functions.retain(|f| f.id == 1); + } + let function = module.functions.iter_mut().find(|f| f.id == 1).unwrap(); + function.params.push(param(104, "key", Type::String)); + function.body.insert( + 2, + let_stmt( + 25, + "keyed", + Expr::IndexGet { + object: Box::new(Expr::LocalGet(100)), + index: Box::new(Expr::LocalGet(104)), + }, + ), + ); + let Some(Stmt::Return(Some(Expr::Array(values, _)))) = function.body.last_mut() else { + panic!("fixture must retain the lookup result"); + }; + values.push(Expr::LocalGet(25)); + assert_eq!( + crate::codegen::decide_full_outline_ic(crate::codegen::module_callable_count(&module)), + outlined, + ); + for target in NATIVE_TARGETS { + let ir = native_ir(&module, target, false); + let probe = probe_body_symbol(&ir, "ic_fast_split.ts"); + let body = function_slice(&ir, &probe); + let points = statepoints_of(&ir, target, &probe); + // A fixed-name leaf checks its memo after an own miss; the keyed + // leaf stays stateless. Both collecting exits take the memo. + let fixed = if outlined { + ( + "perry_probe_get_value_site", + "js_object_get_field_value_site", + ) + } else { + ("perry_probe_get_site", "js_object_get_field_value_site") + }; + for (fast, slow) in [ + fixed, + ("perry_probe_get_keyed", "js_object_get_keyed_site_slow"), + ] { + let calls: Vec<_> = body + .lines() + .filter(|line| line.contains("call ") && line.contains(&format!("@{fast}("))) + .collect(); + assert!( + !calls.is_empty(), + "[{target}] missing {fast} in outlined={outlined}: {body}" + ); + assert!(calls + .iter() + .all(|line| line.contains("\"gc-leaf-function\""))); + assert!(points.iter().all(|sp| sp.callee != fast)); + let collecting = points.at(slow); + assert!( + !collecting.is_empty(), + "[{target}] missing collecting {slow}" + ); + for sp in collecting { + assert!( + !sp.live.is_empty(), + "[{target}] {slow} must retain live roots: {sp:?}" + ); + } + let control = statepoints_of(&strip_leaf(&ir, fast), target, &probe); + assert!(!control.at(fast).is_empty()); + } + } + } +} diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index 7f05ffc1335..d114a95ac8c 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -407,6 +407,23 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // A GC leaf; `TAG_HOLE` = declined. module.declare_function("perry_probe_family_method", I64, &[PTR, I64, I64, I64, I32]); module.declare_function("perry_probe_get", DOUBLE, &[PTR, I64, I64, I64]); + module.declare_function("perry_probe_get_site", DOUBLE, &[PTR, I64, I64, I64, PTR]); + module.declare_function("js_object_get_field_site_slow", DOUBLE, &[I64, I64, PTR]); + module.declare_function( + "perry_probe_get_value_site", + DOUBLE, + &[PTR, I64, I64, I64, PTR], + ); + module.declare_function( + "js_object_get_field_value_site", + DOUBLE, + &[I64, I64, I64, PTR], + ); + module.declare_function( + "js_object_get_keyed_site_slow", + DOUBLE, + &[DOUBLE, DOUBLE, I64, I32, PTR], + ); // The store probe (`expr/put_value_store_ic.rs`): 1 when it stored. module.declare_function("perry_probe_set", I32, &[PTR, I64, I64, I64, DOUBLE]); module.declare_function( diff --git a/crates/perry-codegen/src/wasm32/runtime_abi.tsv b/crates/perry-codegen/src/wasm32/runtime_abi.tsv index 67a78b5e232..c49d253842a 100644 --- a/crates/perry-codegen/src/wasm32/runtime_abi.tsv +++ b/crates/perry-codegen/src/wasm32/runtime_abi.tsv @@ -2742,9 +2742,12 @@ js_object_get_field_by_name_f64 f64 ptr,ptr js_object_get_field_by_property_id_f64 f64 ptr,i64 js_object_get_field_f64 f64 ptr,i32u js_object_get_field_nonptr f64 i64,ptr,i64 +js_object_get_field_site_slow f64 i64,ptr,ptr js_object_get_field_slow f64 i64,ptr js_object_get_field_value f64 i64,ptr,i64 +js_object_get_field_value_site f64 i64,ptr,i64,ptr js_object_get_index_polymorphic f64 i64,f64 +js_object_get_keyed_site_slow f64 f64,f64,i64,i32u,ptr js_object_get_keyed_slow f64 f64,f64,i64,i32u js_object_get_own_field_or_undef f64 f64,ptr,usize js_object_get_own_property_descriptor f64 f64,f64 @@ -5051,7 +5054,9 @@ perry_probe_get f64 ptr,i64,i64,i64 perry_probe_get_binding f64 ptr,i64,i64,i64 perry_probe_get_keyed f64 ptr,i64,i64 perry_probe_get_length f64 ptr,i64,i64,i64 +perry_probe_get_site f64 ptr,i64,i64,i64,ptr perry_probe_get_value f64 ptr,i64,i64,i64 +perry_probe_get_value_site f64 ptr,i64,i64,i64,ptr perry_probe_own_method f64 ptr,f64,i64,i64 perry_probe_set i32s ptr,i64,i64,i64,f64 perry_register_static_plugin void ptr,f64 diff --git a/crates/perry-runtime/src/object/field_get_set.rs b/crates/perry-runtime/src/object/field_get_set.rs index 00d5f5f9683..03710f8bc83 100644 --- a/crates/perry-runtime/src/object/field_get_set.rs +++ b/crates/perry-runtime/src/object/field_get_set.rs @@ -229,7 +229,8 @@ mod for_in_stable; mod get_field_by_name; mod get_slow; pub use get_slow::{ - js_object_get_field_nonptr, js_object_get_field_slow, js_object_get_field_value, + js_object_get_field_nonptr, js_object_get_field_site_slow, js_object_get_field_slow, + js_object_get_field_value, js_object_get_field_value_site, js_object_get_keyed_site_slow, js_object_get_keyed_slow, }; #[cfg(test)] diff --git a/crates/perry-runtime/src/object/field_get_set/get_slow.rs b/crates/perry-runtime/src/object/field_get_set/get_slow.rs index b8274a40d15..de3c5239f28 100644 --- a/crates/perry-runtime/src/object/field_get_set/get_slow.rs +++ b/crates/perry-runtime/src/object/field_get_set/get_slow.rs @@ -3,18 +3,19 @@ //! //! A site reads a coloured key through its 2c arm, and everything else //! through the unknown-receiver probe (`layout_probe.rs`). What the probe -//! declines reaches one of these entries, which hold no site state: an +//! declines reaches one of these entries: an //! accessor the probe's walk handed back is called; every exotic receiver //! (a closure, an array, a typed array, a buffer, a native handle, a Proxy, //! a namespace, a string wrapper, ...) takes the complete by-name operation, -//! which is its spec path. +//! which is its spec path. The site-specific counterparts accept caller-owned +//! shape/slot storage and revalidate it before calling a live getter. use crate::object::ObjectHeader; /// `[[Get]]` through the accessor `acc` that the layout walk found for a key /// of `obj` (on `obj` or on its chain), with `obj` as `this`. `None` for a /// static class accessor, which the complete read calls. #[inline] -unsafe fn call_getter( +pub(super) unsafe fn call_getter( obj: *const ObjectHeader, acc: crate::object::accessor_pair::Accessor, ) -> Option { @@ -210,3 +211,118 @@ pub unsafe extern "C-unwind" fn js_object_get_keyed_slow( _ => crate::value::js_dyn_index_get(obj_box, key), } } + +/// Collecting half of the same fixed-name site. Revalidate shapes, then +/// load and call the holder's CURRENT accessor; no callable is remembered. +/// # Safety +/// memo is the agent-local five-pair storage belonging to this fixed key. +#[no_mangle] +pub unsafe extern "C-unwind" fn js_object_get_field_site_slow( + obj_handle: i64, + key: *const crate::StringHeader, + memo: *mut crate::object::layout_probe::site_memo::ReadSiteMemo, +) -> f64 { + use crate::object::layout_probe::{ + self, + site_memo::{self, Answer}, + }; + let obj = obj_handle as *const ObjectHeader; + if crate::value::addr_class::is_above_handle_band(obj as usize) + && !key.is_null() + && crate::string::is_property_atom(key) + { + let key_bits = crate::value::STRING_TAG | key as u64; + let dir = crate::object::shapes::ordinary_dir_addr(); + layout_probe::verify::check(obj as usize, key_bits); + let answer = site_memo::collect( + dir, + obj, + key_bits, + crate::string::property_atom_hash(key), + &mut *memo, + ); + match answer { + Some(Answer::Data(bits)) => return f64::from_bits(bits), + Some(Answer::Getter(pair)) => { + if let Some(acc) = crate::object::accessor_pair::pair_of_value(pair) { + if layout_probe::verify::on() { + layout_probe::verify::accessor(obj as usize, key_bits, &acc); + } + if let Some(value) = call_getter(obj, acc) { + return value; + } + } + } + None => {} + } + } + js_object_get_field_slow(obj_handle, key) +} + +/// Boxed collecting entrance for the same fixed-name site. Keep the +/// original representation migration and non-pointer dispatch unchanged. +/// # Safety +/// memo belongs to this fixed key and this agent, as for the raw entrance. +#[no_mangle] +pub unsafe extern "C-unwind" fn js_object_get_field_value_site( + obj_bits: i64, + key: *const crate::StringHeader, + site_id: u64, + memo: *mut crate::object::layout_probe::site_memo::ReadSiteMemo, +) -> f64 { + let bits = obj_bits as u64; + if bits >> 48 == crate::value::POINTER_TAG >> 48 + && crate::value::addr_class::is_above_handle_band( + (bits & crate::value::POINTER_MASK) as usize, + ) + { + crate::object::field_rep_store::migrate_on_miss_value(bits); + return js_object_get_field_site_slow( + (bits & crate::value::POINTER_MASK) as i64, + key, + memo, + ); + } + js_object_get_field_nonptr(obj_bits, key, site_id) +} + +/// Collecting computed-key half. An accessor is read from its holder's +/// live slot after the same shape proof used by the fixed-name entrance. +/// # Safety +/// memo belongs to this computed read site and this agent. +#[no_mangle] +pub unsafe extern "C-unwind" fn js_object_get_keyed_site_slow( + obj_box: f64, + key: f64, + site_id: u64, + dynamic: u32, + memo: *mut crate::object::layout_probe::site_memo::KeyedSiteMemo, +) -> f64 { + use crate::object::layout_probe::{ + self, + site_memo::{self, Answer}, + }; + let addr = (obj_box.to_bits() & crate::value::POINTER_MASK) as usize; + layout_probe::verify::check(addr, key.to_bits()); + let answer = site_memo::keyed_answer( + crate::object::shapes::ordinary_dir_addr(), + obj_box.to_bits(), + key.to_bits(), + &mut *memo, + ); + match answer { + Some(Answer::Data(bits)) => return f64::from_bits(bits), + Some(Answer::Getter(pair)) => { + if let Some(acc) = crate::object::accessor_pair::pair_of_value(pair) { + if layout_probe::verify::on() { + layout_probe::verify::accessor(addr, key.to_bits(), &acc); + } + if let Some(v) = call_getter(addr as *const ObjectHeader, acc) { + return v; + } + } + } + None => {} + } + js_object_get_keyed_slow(obj_box, key, site_id, dynamic) +} diff --git a/crates/perry-runtime/src/object/field_get_set/runtime_read_site.rs b/crates/perry-runtime/src/object/field_get_set/runtime_read_site.rs index 5ec249fdb80..c1638e663f4 100644 --- a/crates/perry-runtime/src/object/field_get_set/runtime_read_site.rs +++ b/crates/perry-runtime/src/object/field_get_set/runtime_read_site.rs @@ -6,9 +6,11 @@ //! emitted `o.key` site whose receiver no compile-time fact names: the //! unknown-receiver probe (`layout_probe::perry_probe_get`) with the agent's //! directory, and, when the probe declines, the cacheless complete read -//! (`js_object_get_field_slow`). Nothing is remembered between reads: every -//! word the probe reads is the receiver's own data or immutable layout data, -//! so there is nothing to prime, validate or root. +//! (`js_object_get_field_slow`). Fixed-name Rust sites use `get_with_memo`: +//! the same leaf and collecting entrance as an emitted fixed-name read that +//! owns a shape/slot memo (`perry_probe_get_site`, which asks the memo only +//! after an own miss, then `js_object_get_field_site_slow`). The memo retains +//! no heap addresses or getter values. //! //! # The key //! @@ -110,6 +112,20 @@ pub(crate) unsafe fn probe(obj: *const ObjectHeader, key: Key<'_>) -> Option f64 { + get_with_memo(obj, key, std::ptr::null_mut()) +} + +/// [`get`] for a fixed-key Rust site that owns a shape/slot memo, read as an +/// emitted site that owns one: an own hit never reads the memo; an own miss +/// asks it; only the collecting entrance derives it. +/// +/// # Safety +/// As [`get`]; `memo` is null or this site's agent-local storage. +pub(crate) unsafe fn get_with_memo( + obj: *mut ObjectHeader, + key: &[u8], + memo: *mut crate::object::layout_probe::site_memo::ReadSiteMemo, +) -> f64 { let hash = crate::object::key_bytes_hash(key.as_ptr(), key.len()); let (atom, obj) = match crate::string::atom_lookup(key, hash) { Some(atom) if crate::string::is_property_atom(atom) => (atom, obj), @@ -124,15 +140,27 @@ pub(crate) unsafe fn get(obj: *mut ObjectHeader, key: &[u8]) -> f64 { }) } }; - let v = probe_word( - obj, - crate::value::STRING_TAG | atom as u64, - crate::string::property_atom_hash(atom), - ); + let key_bits = crate::value::STRING_TAG | atom as u64; + let hash = crate::string::property_atom_hash(atom); + let v = if memo.is_null() { + probe_word(obj, key_bits, hash) + } else { + crate::object::layout_probe::site_memo::perry_probe_get_site( + crate::object::shapes::ordinary_dir_addr(), + (obj as usize).wrapping_sub(perry_abi::RECEIVER_HANDLE_FLOOR) as i64, + key_bits, + hash, + memo, + ) + }; if v.to_bits() != crate::value::TAG_HOLE { return v; } - super::js_object_get_field_slow(obj as i64, atom) + if memo.is_null() { + super::js_object_get_field_slow(obj as i64, atom) + } else { + super::js_object_get_field_site_slow(obj as i64, atom, memo) + } } /// The integer a Number property key names, when its canonical text is a diff --git a/crates/perry-runtime/src/object/layout_probe.rs b/crates/perry-runtime/src/object/layout_probe.rs index 387567185d8..26053d6fad9 100644 --- a/crates/perry-runtime/src/object/layout_probe.rs +++ b/crates/perry-runtime/src/object/layout_probe.rs @@ -36,6 +36,9 @@ //! absent. Everything else (an accessor, an exotic or per-object receiver, a //! chain that ends without the key, a builtin prototype that misses) is //! `TAG_HOLE`, and the site takes its complete slow operation. +#[path = "layout_probe_site_memo.rs"] +pub(crate) mod site_memo; + use super::shapes_store::{ShapeRecord, FLAT_BASE_OFFSET}; use crate::object::layout_desc::note_run; use crate::object::ObjectHeader; @@ -918,6 +921,20 @@ unsafe fn probe_get( obj_biased: i64, key_bits: u64, hash: u64, +) -> f64 { + probe_get_at::(dir, obj_biased, key_bits, hash, std::ptr::null()) +} + +/// The own-data front. With `SITE` (a compile-time choice) every own miss +/// first asks the site's shape/slot memo (`site_memo`), then continues +/// exactly as the stateless probe; an own hit never reads the memo. +#[inline(always)] +unsafe fn probe_get_at( + dir: *const u8, + obj_biased: i64, + key_bits: u64, + hash: u64, + memo: *const site_memo::ReadSiteMemo, ) -> f64 { // The hit path keeps to caller-saved registers: every other outcome is a // tail call, so the leaf saves nothing on a hit. @@ -930,11 +947,18 @@ unsafe fn probe_get( } let probe = (*r).probe; if probe & (PROBE_READY | PROBE_DECLINE | PROBE_CHECKED) != PROBE_READY { + if SITE { + return site_memo::miss_underived::(dir, obj, key_bits, hash, memo); + } return probe_get_underived::(dir, obj, word, r, key_bits, hash); } let count = (*r).logical_key_count; if count == 0 { - return absent_from::(dir, obj, key_bits, hash); + return if SITE { + site_memo::miss_absent::(dir, obj, key_bits, hash, memo) + } else { + absent_from::(dir, obj, key_bits, hash) + }; } let table = (probe & !PROBE_FLAGS) as *const KeyIndex; let (pos, load) = if table.is_null() { @@ -954,7 +978,11 @@ unsafe fn probe_get( } i += 1; if i == count { - return absent_from::(dir, obj, key_bits, hash); + return if SITE { + site_memo::miss_absent::(dir, obj, key_bits, hash, memo) + } else { + absent_from::(dir, obj, key_bits, hash) + }; } } } else { @@ -965,7 +993,11 @@ unsafe fn probe_get( loop { let pos = *slots.add(b); if pos == u32::MAX { - return absent_from::(dir, obj, key_bits, hash); + return if SITE { + site_memo::miss_absent::(dir, obj, key_bits, hash, memo) + } else { + absent_from::(dir, obj, key_bits, hash) + }; } let i = pos & POS_BITS; if pos & TEXT_ENTRY != 0 { @@ -995,6 +1027,9 @@ unsafe fn probe_get( if v.to_bits() != TAG_HOLE { return v; } + if SITE { + return site_memo::miss_hole::(dir, obj, pos, key_bits, hash, memo); + } own_hole::(dir, obj, r, pos, key_bits, hash) } diff --git a/crates/perry-runtime/src/object/layout_probe_site_memo.rs b/crates/perry-runtime/src/object/layout_probe_site_memo.rs new file mode 100644 index 00000000000..fa296e40871 --- /dev/null +++ b/crates/perry-runtime/src/object/layout_probe_site_memo.rs @@ -0,0 +1,471 @@ +//! Per-site answers derived exclusively from the shapes on a bounded chain. +//! The entries contain no heap addresses: each use follows the live shape +//! links and reads the current holder slot, including its current getter pair. +//! +//! Only a collecting entrance writes a memo: a read the stateless probe +//! declined (a getter on the chain, a computed key a prototype holds) proves +//! the memo's shapes, and otherwise derives it from the live chain before the +//! complete read. A fixed-name site's leaf (`perry_probe_get_site`) is the +//! stateless own-data front; only after an own miss does it compare the +//! memo's first ShapeId with the receiver's and, on a match, answer from the +//! memo (a getter still declines to the collecting entrance). An own hit +//! never reads the memo, and no leaf ever writes it. Computed-key leaves stay +//! stateless (`perry_probe_get_keyed`). +use super::*; + +const PAIRS: usize = 5; +const NEXT: u32 = u32::MAX; +const ABSENT: u32 = u32::MAX - 1; +/// The receiver's shape has no memo answer at this site (its chain is not +/// one the shapes prove): the collecting entrance takes the complete read +/// without deriving again. A negative answer, checked by the same ShapeId. +const DECLINE: u32 = u32::MAX - 2; +const ACCESSOR: u32 = 1 << 31; +const SPILL: u32 = 1 << 30; +const SLOT_MASK: u32 = SPILL - 1; + +/// Exactly five (ShapeId, slot) pairs. Zero shape means an empty answer. +/// Kept in the site's own TLS storage, never indexed through a registry. +#[derive(Clone, Copy)] +#[repr(C)] +pub struct ReadSiteMemo { + pairs: [u64; PAIRS], +} +impl ReadSiteMemo { + pub(crate) const EMPTY: Self = Self { pairs: [0; PAIRS] }; +} + +#[derive(Clone, Copy)] +pub(crate) enum Answer { + Data(u64), + // The live pair word, decoded only by the collecting entrance. + // Keeping a full Accessor here made even data hits return 56 bytes + // through a hidden sret argument and a stack frame. + Getter(u64), +} + +#[inline(always)] +unsafe fn ordinary_word(obj: *const ObjectHeader) -> Option { + let h = crate::gc::header_from_trusted_user_ptr(obj.cast()); + if (*h).obj_type != crate::gc::GC_TYPE_OBJECT + || (*h).gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + || (*h)._reserved & crate::gc::OBJ_FLAG_TYPED_ARRAY_PROTO != 0 + { + return None; + } + let meta = (*obj).meta; + if !meta.is_null() + && (*meta).flags + & (crate::object::OBJECT_META_FLAG_EXOTIC_READ_RECEIVER + | crate::object::OBJECT_META_FLAG_NATIVE_ALIAS) + != 0 + { + return None; + } + let word = (*obj).parent_class_id; + // An overflow's SHADOW_BIT deliberately prevents a site comparison. + crate::object::shapes::is_site_matchable_shape_id(word).then_some(word) +} + +#[inline(always)] +unsafe fn slot_answer(obj: *const ObjectHeader, slot: u32) -> Option { + if slot == ABSENT { + return Some(Answer::Data(crate::value::TAG_UNDEFINED)); + } + let index = slot & SLOT_MASK; + let bits = if slot & SPILL != 0 { + crate::object::spill::spill_get_present(obj as usize, index as usize)? + } else { + *((obj as *const u8).add(std::mem::size_of::() + index as usize * 8) + as *const u64) + }; + if bits == TAG_HOLE { + return None; + } + if slot & ACCESSOR != 0 { + Some(Answer::Getter(bits)) + } else { + Some(Answer::Data(bits)) + } +} + +/// The complete O(1) proof on every use. No key lookup, allocation or user +/// code. Each continuation's shape proves absence and supplies its live link. +#[inline] +pub(crate) unsafe fn lookup( + dir: *const u8, + mut obj: *const ObjectHeader, + memo: &ReadSiteMemo, +) -> Option { + for &pair in &memo.pairs { + let shape = (pair >> 32) as u32; + if shape == 0 || (*obj).parent_class_id != shape || ordinary_word(obj)? != shape { + return None; + } + let slot = pair as u32; + if slot == DECLINE { + return None; + } + if slot != NEXT && slot != ABSENT { + if let Some(answer) = slot_answer(obj, slot) { + return Some(answer); + } + // A data hole is absence on this object, not absence proved by + // the shape. Continue only through the live link and verified + // tail. An accessor hole retains the complete getter's rules. + if slot & ACCESSOR != 0 { + return None; + } + } + let r = record_of(dir, shape); + if r.is_null() { + return None; + } + if slot == ABSENT { + return matches!(link_of(dir, obj, r), Next::End) + .then_some(Answer::Data(crate::value::TAG_UNDEFINED)); + } + obj = match link_of(dir, obj, r) { + Next::Object(next) => next, + Next::End => return Some(Answer::Data(crate::value::TAG_UNDEFINED)), + Next::Unknown => return None, + }; + } + None +} + +/// Has this memo found that a receiver of `obj`'s shape has no memo answer? +#[inline(always)] +unsafe fn declined(obj: *const ObjectHeader, memo: &ReadSiteMemo) -> bool { + memo.pairs[0] == (u64::from((*obj).parent_class_id) << 32) | u64::from(DECLINE) +} + +/// Remember that a receiver of `obj`'s shape has no memo answer, unless the +/// memo already holds a positive answer for another shape. +#[inline] +unsafe fn decline(obj: *const ObjectHeader, memo: &mut ReadSiteMemo) { + let word = (*obj).parent_class_id; + let held = memo.pairs[0]; + if crate::object::shapes::is_site_matchable_shape_id(word) + && (held == 0 || held as u32 == DECLINE) + { + *memo = ReadSiteMemo::EMPTY; + memo.pairs[0] = (u64::from(word) << 32) | u64::from(DECLINE); + } +} + +/// A collecting entrance's answer for a fixed key: the memo's proof, else a +/// derivation from the live chain, unless this receiver shape already +/// declined. A derivation that declines is remembered as [`DECLINE`]. +#[inline(always)] +pub(crate) unsafe fn collect( + dir: *const u8, + obj: *const ObjectHeader, + key_bits: u64, + hash: u64, + memo: &mut ReadSiteMemo, +) -> Option { + if let Some(answer) = lookup(dir, obj, memo) { + return Some(answer); + } + if declined(obj, memo) { + return None; + } + let answer = prime(dir, obj, key_bits, hash, memo); + if answer.is_none() { + decline(obj, memo); + } + answer +} + +/// Derive the memo without running user code. Data holes retain a guarded +/// prototype tail; overflow, +/// dictionaries, exotics and chains longer than the fixed bound decline. +#[inline(never)] +pub(crate) unsafe fn prime( + dir: *const u8, + mut obj: *const ObjectHeader, + key_bits: u64, + hash: u64, + memo: &mut ReadSiteMemo, +) -> Option { + let mut learned = ReadSiteMemo::EMPTY; + for i in 0..PAIRS { + let shape = ordinary_word(obj)?; + let r = record_of(dir, shape); + if r.is_null() + || !(*r).present() + || !positional_layout(&*r) + || !matches!( + (*r).object_kind(), + super::super::ShapeObjectKind::Ordinary + | super::super::ShapeObjectKind::OrdinaryUnmarked + ) + || !walks_from(obj, r) + { + return None; + } + let prefix = u64::from(shape) << 32; + match own_entry(r, key_bits, hash) { + Ok((pos, _)) => { + let keys = (*r).keys as usize as *const crate::array::ArrayHeader; + let attrs = crate::object::key_attrs::keys_entry(keys, pos); + if attrs & crate::object::key_attrs::ENTRY_PRIVATE != 0 { + return None; + } + let slot = crate::object::key_attrs::key_storage_slot(keys, pos); + if slot >= SLOT_MASK - 1 { + return None; + } + let slot = + slot | if slot < (*r).live_inline_slot_count { + 0 + } else { + SPILL + } | if attrs & crate::object::key_attrs::ENTRY_ACCESSOR_MASK != 0 { + ACCESSOR + } else { + 0 + }; + learned.pairs[i] = prefix | u64::from(slot); + if let Some(answer) = slot_answer(obj, slot) { + *memo = learned; + return Some(answer); + } + if slot & ACCESSOR != 0 { + return None; + } + // Keep the data slot itself in the entry: every use must + // re-read its live presence before trusting the tail. + } + Err(false) => return None, + Err(true) => {} + } + match link_of(dir, obj, r) { + Next::End => { + if learned.pairs[i] == 0 { + learned.pairs[i] = prefix | u64::from(ABSENT); + } + *memo = learned; + return Some(Answer::Data(crate::value::TAG_UNDEFINED)); + } + Next::Object(next) => { + if learned.pairs[i] == 0 { + learned.pairs[i] = prefix | u64::from(NEXT); + } + obj = next; + } + Next::Unknown => return None, + } + } + None +} + +/// The leaf's question after an own miss: does the memo, whose first +/// ShapeId is the receiver's, answer? `None` continues the stateless probe. +#[inline(always)] +unsafe fn leaf_answer( + dir: *const u8, + obj: *const ObjectHeader, + key_bits: u64, + memo: *const ReadSiteMemo, +) -> Option { + if ((*memo).pairs[0] >> 32) as u32 != (*obj).parent_class_id { + return None; + } + Some(match lookup(dir, obj, &*memo)? { + Answer::Data(bits) => verify::answer(obj as usize, key_bits, f64::from_bits(bits)), + // The collecting entrance reloads and calls the live getter. + Answer::Getter(_) => hole(), + }) +} + +/// The own list proved the key absent. +#[inline(never)] +pub(super) unsafe fn miss_absent( + dir: *const u8, + obj: *const ObjectHeader, + key_bits: u64, + hash: u64, + memo: *const ReadSiteMemo, +) -> f64 { + if let Some(v) = leaf_answer(dir, obj, key_bits, memo) { + return v; + } + absent_from::(dir, obj, key_bits, hash) +} + +/// The receiver's record is not derived, or is checked (accessor/private). +#[inline(never)] +pub(super) unsafe fn miss_underived( + dir: *const u8, + obj: *const ObjectHeader, + key_bits: u64, + hash: u64, + memo: *const ReadSiteMemo, +) -> f64 { + if let Some(v) = leaf_answer(dir, obj, key_bits, memo) { + return v; + } + let word = (*obj).parent_class_id; + probe_get_underived::(dir, obj, word, record_of(dir, word), key_bits, hash) +} + +/// The own slot at position `pos` read a hole. +#[inline(never)] +pub(super) unsafe fn miss_hole( + dir: *const u8, + obj: *const ObjectHeader, + pos: u32, + key_bits: u64, + hash: u64, + memo: *const ReadSiteMemo, +) -> f64 { + if let Some(v) = leaf_answer(dir, obj, key_bits, memo) { + return v; + } + own_hole::( + dir, + obj, + record_of(dir, (*obj).parent_class_id), + pos, + key_bits, + hash, + ) +} + +/// The fixed-name leaf of a site that owns a memo: [`perry_probe_get`] with +/// the memo asked after an own miss. A GC leaf; it never writes the memo. +/// # Safety +/// As [`perry_probe_get`]; `memo` is this site's agent-local storage. +#[no_mangle] +pub unsafe extern "C" fn perry_probe_get_site( + dir: *const u8, + obj_biased: i64, + key_bits: u64, + hash: u64, + memo: *const ReadSiteMemo, +) -> f64 { + probe_get_at::(dir, obj_biased, key_bits, hash, memo) +} + +/// [`perry_probe_get_value`] for an outlined or generator site that owns a +/// memo. +/// # Safety +/// As [`perry_probe_get_value`]; `memo` is this site's agent-local storage. +#[no_mangle] +pub unsafe extern "C" fn perry_probe_get_value_site( + dir: *const u8, + obj_bits: u64, + key_bits: u64, + hash: u64, + memo: *const ReadSiteMemo, +) -> f64 { + if obj_bits & TAG_MASK != POINTER_TAG { + return hole(); + } + let payload = (obj_bits & POINTER_MASK) as usize; + if payload < perry_abi::RECEIVER_HANDLE_FLOOR { + return hole(); + } + perry_probe_get_site( + dir, + (payload - perry_abi::RECEIVER_HANDLE_FLOOR) as i64, + key_bits, + hash, + memo, + ) +} + +#[derive(Clone, Copy)] +#[repr(C)] +struct KeyedWay { + key: u64, + memo: ReadSiteMemo, +} + +/// Four shape answers for a small rotating set of short string keys. +/// The key is CONTENT bits, never a GC address or a cached program value. +#[repr(C)] +pub struct KeyedSiteMemo { + ways: [KeyedWay; 4], +} + +#[inline] +unsafe fn short_key(bits: u64) -> Option { + if bits & TAG_MASK == crate::value::SHORT_STRING_TAG { + return Some(bits); + } + if bits & TAG_MASK == STRING_TAG { + return crate::string::short_ascii_sso_bits( + (bits & POINTER_MASK) as *const crate::StringHeader, + ); + } + None +} + +pub(crate) unsafe fn keyed_answer( + dir: *const u8, + obj_bits: u64, + key_bits: u64, + memo: &mut KeyedSiteMemo, +) -> Option { + if obj_bits & TAG_MASK != POINTER_TAG { + return None; + } + let addr = (obj_bits & POINTER_MASK) as usize; + if addr < perry_abi::RECEIVER_HANDLE_FLOOR { + return None; + } + let key = short_key(key_bits)?; + let obj = addr as *const ObjectHeader; + for way in &memo.ways { + if way.key == key { + if let Some(answer) = lookup(dir, obj, &way.memo) { + return Some(answer); + } + if declined(obj, &way.memo) { + return None; + } + } + } + let mut learned = ReadSiteMemo::EMPTY; + let answer = match computed_key(key_bits) { + Some((canonical, hash)) => prime(dir, obj, canonical, hash, &mut learned), + None => None, + }; + let Some(answer) = answer else { + // This key on this receiver shape has no memo answer: remember the + // decline so its next read takes the complete read at once. + decline(obj, &mut learned); + if learned.pairs[0] != 0 { + remember(memo, KeyedWay { key, memo: learned }); + } + return None; + }; + // A computed key is retained only when it is the content of a real + // holder key at the recorded slot. Absence has no such key to derive + // from the shape. Return this fresh proof without remembering its key. + if learned + .pairs + .iter() + .any(|&pair| pair != 0 && pair as u32 == ABSENT) + { + return Some(answer); + } + remember(memo, KeyedWay { key, memo: learned }); + Some(answer) +} + +/// The newest way first; the oldest leaves. +#[inline] +fn remember(memo: &mut KeyedSiteMemo, way: KeyedWay) { + for i in (1..memo.ways.len()).rev() { + memo.ways[i] = memo.ways[i - 1]; + } + memo.ways[0] = way; +} + +#[cfg(test)] +#[path = "layout_probe_site_memo_tests.rs"] +mod tests; diff --git a/crates/perry-runtime/src/object/layout_probe_site_memo_tests.rs b/crates/perry-runtime/src/object/layout_probe_site_memo_tests.rs new file mode 100644 index 00000000000..9daf5fc6f60 --- /dev/null +++ b/crates/perry-runtime/src/object/layout_probe_site_memo_tests.rs @@ -0,0 +1,348 @@ +use super::*; + +fn atom(name: &[u8]) -> *const crate::StringHeader { + crate::string::js_string_pool_atom( + name.as_ptr(), + name.len() as u32, + unsafe { crate::object::key_bytes_hash(name.as_ptr(), name.len()) }, + 0, + ) as *const _ +} +unsafe fn plain(name: &[u8], value: f64) -> *mut ObjectHeader { + let obj = crate::object::js_object_alloc(0, 4); + crate::object::js_object_set_field_by_name(obj, atom(name), value); + obj +} +unsafe fn inherited(proto: *mut ObjectHeader) -> *mut ObjectHeader { + let value = crate::object::js_object_create(crate::value::js_nanbox_pointer(proto as i64)); + (value.to_bits() & POINTER_MASK) as *mut ObjectHeader +} +unsafe fn data(answer: Option) -> u64 { + match answer { + Some(Answer::Data(bits)) => bits, + _ => panic!("expected memo data hit"), + } +} +unsafe fn raw_getter(answer: Option) -> usize { + match answer { + Some(Answer::Getter(pair)) => { + crate::object::accessor_pair::pair_of_value(pair) + .unwrap() + .raw_get + } + _ => panic!("expected memo accessor hit"), + } +} + +#[test] +fn two_level_hits_load_live_data_and_fail_on_shadowing() { + let _lock = crate::gc::global_side_table_test_lock(); + let _no_move = crate::gc::GcSuppressScope::new(); + unsafe { + let dir = crate::object::shapes::ordinary_dir_addr(); + let key = atom(b"value"); + let hash = crate::string::property_atom_hash(key); + let top = plain(b"value", 7.0); + let mid = inherited(top); + let recv = inherited(mid); + let mut memo = ReadSiteMemo::EMPTY; + assert_eq!( + data(prime(dir, recv, STRING_TAG | key as u64, hash, &mut memo)), + 7.0f64.to_bits() + ); + assert_eq!( + data(lookup(dir, recv, &memo)), + 7.0f64.to_bits(), + "the warmed path must actually hit" + ); + crate::object::js_object_set_field_by_name(top, key, 8.0); + assert_eq!( + data(lookup(dir, recv, &memo)), + 8.0f64.to_bits(), + "values are not cached" + ); + crate::object::js_object_set_field_by_name(mid, key, 99.0); + assert!( + lookup(dir, recv, &memo).is_none(), + "an intermediate own key invalidates its shape proof" + ); + } +} + +#[test] +fn accessor_hits_reload_the_holder_pair() { + let _lock = crate::gc::global_side_table_test_lock(); + let _no_move = crate::gc::GcSuppressScope::new(); + unsafe extern "C" fn first(_: f64) -> f64 { + 1.0 + } + unsafe extern "C" fn second(_: f64) -> f64 { + 2.0 + } + unsafe { + let dir = crate::object::shapes::ordinary_dir_addr(); + let proto = crate::object::js_object_alloc(0, 4); + let pair = crate::object::accessor_pair::Accessor { + raw_get: first as *const () as usize, + ..Default::default() + }; + crate::object::descriptor_state::set_builtin_accessor_pair_bytes( + proto as usize, + b"value", + pair, + crate::object::PropertyAttrs::new(true, true, true), + ); + let recv = inherited(proto); + let key = atom(b"value"); + let mut memo = ReadSiteMemo::EMPTY; + let found = prime( + dir, + recv, + STRING_TAG | key as u64, + crate::string::property_atom_hash(key), + &mut memo, + ); + assert_eq!(raw_getter(found), first as *const () as usize); + assert_eq!( + raw_getter(lookup(dir, recv, &memo)), + first as *const () as usize + ); + // Mutate the live pair through the holder's slot without touching + // the memo. A memo of a callable would incorrectly retain first. + let record = record_of(dir, (*proto).parent_class_id); + let (pos, _) = own_entry( + record, + STRING_TAG | key as u64, + crate::string::property_atom_hash(key), + ) + .unwrap(); + let keys = (*record).keys as usize as *const crate::array::ArrayHeader; + let slot = crate::object::key_attrs::key_storage_slot(keys, pos); + let bits = *((proto as *const u8) + .add(std::mem::size_of::() + slot as usize * 8) + as *const u64); + let array = (bits & POINTER_MASK) as *mut crate::array::ArrayHeader; + *(crate::array::array_elements_ptr(array) as *mut u64) + .add(crate::object::accessor_pair::PAIR_RAW_GET) = second as *const () as u64; + assert_eq!( + raw_getter(lookup(dir, recv, &memo)), + second as *const () as usize + ); + } +} + +#[test] +fn a_different_receiver_with_the_same_shape_reads_its_own_slot() { + let _lock = crate::gc::global_side_table_test_lock(); + let _no_move = crate::gc::GcSuppressScope::new(); + unsafe { + let dir = crate::object::shapes::ordinary_dir_addr(); + let one = plain(b"value", 1.0); + let two = plain(b"value", 2.0); + assert_eq!((*one).parent_class_id, (*two).parent_class_id); + let key = atom(b"value"); + let mut memo = ReadSiteMemo::EMPTY; + prime( + dir, + one, + STRING_TAG | key as u64, + crate::string::property_atom_hash(key), + &mut memo, + ) + .unwrap(); + assert_eq!(data(lookup(dir, two, &memo)), 2.0f64.to_bits()); + assert_eq!( + std::mem::size_of::(), + 16, + "data hits must not use an Accessor-sized return ABI" + ); + assert_eq!(std::mem::size_of::(), 40); + assert_eq!(std::mem::size_of::(), 192); + } +} + +#[test] +fn rust_runtime_sites_share_the_memo_and_preserve_getter_this() { + let _lock = crate::gc::global_side_table_test_lock(); + let _no_move = crate::gc::GcSuppressScope::new(); + unsafe extern "C-unwind" fn receiver(this: f64) -> f64 { + this + } + unsafe { + let proto = crate::object::js_object_alloc(0, 4); + let pair = crate::object::accessor_pair::Accessor { + raw_get: receiver as *const () as usize, + ..Default::default() + }; + crate::object::descriptor_state::set_builtin_accessor_pair_bytes( + proto as usize, + b"value", + pair, + crate::object::PropertyAttrs::new(true, true, true), + ); + let recv = inherited(proto); + let key = crate::runtime_state_key!(b"value"); + let expected = crate::value::js_nanbox_pointer(recv as i64).to_bits(); + assert_eq!(key.read_object(recv).to_bits(), expected); + let memo = key.memo.unwrap()(); + assert_eq!( + raw_getter(lookup( + crate::object::shapes::ordinary_dir_addr(), + recv, + &*memo + )), + receiver as *const () as usize + ); + assert_eq!( + key.read_object(recv).to_bits(), + expected, + "a warmed Rust site calls with the original receiver" + ); + } +} + +#[test] +fn leaf_answers_never_touch_the_site_memo() { + let _lock = crate::gc::global_side_table_test_lock(); + let _no_move = crate::gc::GcSuppressScope::new(); + unsafe { + let own = plain(b"value", 1.0); + let recv = inherited(plain(b"value", 2.0)); + let key = crate::runtime_state_key!(b"value"); + assert_eq!(key.read_object(own), 1.0); + assert_eq!(key.read_object(recv), 2.0); + let memo = key.memo.unwrap()(); + assert_eq!( + (*memo).pairs, + [0; PAIRS], + "own and inherited data are stateless probe answers" + ); + // The control: a declined read (an inherited getter) does derive. + unsafe extern "C-unwind" fn seven(_: f64) -> f64 { + 7.0 + } + let proto = crate::object::js_object_alloc(0, 4); + crate::object::descriptor_state::set_builtin_accessor_pair_bytes( + proto as usize, + b"value", + crate::object::accessor_pair::Accessor { + raw_get: seven as *const () as usize, + ..Default::default() + }, + crate::object::PropertyAttrs::new(true, true, true), + ); + let getter_recv = inherited(proto); + assert_eq!(key.read_object(getter_recv), 7.0); + assert_ne!( + (*memo).pairs[0], + 0, + "the collecting entrance derives the memo" + ); + } +} + +#[test] +fn an_own_hole_reads_live_presence_before_using_its_tail() { + let _lock = crate::gc::global_side_table_test_lock(); + let _no_move = crate::gc::GcSuppressScope::new(); + unsafe { + let dir = crate::object::shapes::ordinary_dir_addr(); + let key = atom(b"error"); + let word = STRING_TAG | key as u64; + let hash = crate::string::property_atom_hash(key); + let proto = plain(b"error", 7.0); + let recv = crate::object::layout_desc::perry_alloc_layout(std::ptr::addr_of!( + crate::object::runtime_layouts::SPAWN_SYNC_RESULT + )); + crate::object::prototype_chain::object_link_created_prototype( + recv as usize, + crate::value::js_nanbox_pointer(proto as i64).to_bits(), + ); + let shape = (*recv).parent_class_id; + let record = record_of(dir, shape); + let (pos, _) = own_entry(record, word, hash).unwrap(); + let keys = (*record).keys as usize as *const crate::array::ArrayHeader; + assert_ne!( + crate::object::key_attrs::keys_entry(keys, pos) + & crate::object::key_attrs::ENTRY_OPTIONAL, + 0, + "the fixture must permit live presence without a layout change" + ); + let slot = crate::object::key_attrs::key_storage_slot(keys, pos); + assert!(slot < (*record).live_inline_slot_count); + let field = (recv as *mut u8).add(std::mem::size_of::() + slot as usize * 8) + as *mut u64; + assert_eq!(*field, TAG_HOLE); + assert_eq!( + perry_probe_get( + dir, + (recv as usize - perry_abi::RECEIVER_HANDLE_FLOOR) as i64, + word, + hash + ), + 7.0, + "the fixture must have ordinary inherited Get semantics" + ); + let mut memo = ReadSiteMemo::EMPTY; + assert!( + ordinary_word(recv).is_some(), + "the fresh fixture must be memo-admissible" + ); + let primed = prime(dir, recv, word, hash, &mut memo); + assert!( + primed.is_some(), + "the optional slot must retain its prototype tail" + ); + assert_eq!(data(primed), 7.0f64.to_bits()); + assert_eq!(data(lookup(dir, recv, &memo)), 7.0f64.to_bits()); + // Model an emitted optional-slot store: the slot itself owns its + // live presence. Both test words are pointer-free scalars. + // GC_STORE_AUDIT(POINTER_FREE): scalar in a test's optional slot. + *field = 11.0f64.to_bits(); + assert_eq!((*recv).parent_class_id, shape); + assert_eq!( + data(lookup(dir, recv, &memo)), + 11.0f64.to_bits(), + "an own value shadows the remembered tail" + ); + // GC_STORE_AUDIT(POINTER_FREE): hole in a test's optional slot. + *field = TAG_HOLE; + crate::object::js_object_set_field_by_name(proto, key, 8.0); + assert_eq!(data(lookup(dir, recv, &memo)), 8.0f64.to_bits()); + } +} + +#[test] +fn a_decline_is_a_shape_checked_answer_that_never_displaces_a_proof() { + let _lock = crate::gc::global_side_table_test_lock(); + let _no_move = crate::gc::GcSuppressScope::new(); + unsafe { + let dir = crate::object::shapes::ordinary_dir_addr(); + let key = atom(b"value"); + let word = STRING_TAG | key as u64; + let hash = crate::string::property_atom_hash(key); + let a = plain(b"value", 1.0); + let b = plain(b"other", 2.0); + crate::object::js_object_set_field_by_name(b, atom(b"more"), 3.0); + assert_ne!((*a).parent_class_id, (*b).parent_class_id); + let mut memo = ReadSiteMemo::EMPTY; + decline(b, &mut memo); + assert!(declined(b, &memo)); + assert!(!declined(a, &memo), "the decline names b's shape only"); + assert!(lookup(dir, b, &memo).is_none()); + assert!( + collect(dir, b, word, hash, &mut memo).is_none(), + "a declined shape is not derived again" + ); + assert!(declined(b, &memo)); + // A receiver of another shape still derives, replacing the decline. + assert_eq!( + data(collect(dir, a, word, hash, &mut memo)), + 1.0f64.to_bits() + ); + let proof = memo; + decline(b, &mut memo); + assert_eq!(memo.pairs, proof.pairs, "a decline never displaces a proof"); + assert_eq!(data(lookup(dir, a, &memo)), 1.0f64.to_bits()); + } +} diff --git a/crates/perry-runtime/src/object/node_http_prototypes.rs b/crates/perry-runtime/src/object/node_http_prototypes.rs index f2e764be492..9f4ee46f804 100644 --- a/crates/perry-runtime/src/object/node_http_prototypes.rs +++ b/crates/perry-runtime/src/object/node_http_prototypes.rs @@ -503,6 +503,7 @@ extern "C" fn http_getter_thunk( crate::runtime_state_key::NamedStateKey { bytes: key, hash: crate::runtime_state_key::const_key_hash(key), + memo: None, } .read_value(receiver) }; @@ -604,6 +605,7 @@ pub unsafe extern "C" fn js_node_http_incoming_message_init(this: f64, socket: f crate::runtime_state_key::NamedStateKey { bytes, hash: crate::runtime_state_key::const_key_hash(bytes), + memo: None, } .read_value(value) }; @@ -816,6 +818,7 @@ unsafe fn message_init(this: f64, req: Option) { crate::runtime_state_key::NamedStateKey { bytes: key, hash: crate::runtime_state_key::const_key_hash(key), + memo: None, } .read_value(target) } else { @@ -1003,6 +1006,7 @@ mod incoming_init_tests { crate::runtime_state_key::NamedStateKey { bytes, hash: crate::runtime_state_key::const_key_hash(bytes), + memo: None, } .read_value(object) }; diff --git a/crates/perry-runtime/src/runtime_state_key.rs b/crates/perry-runtime/src/runtime_state_key.rs index 4a1b7312283..08677accead 100644 --- a/crates/perry-runtime/src/runtime_state_key.rs +++ b/crates/perry-runtime/src/runtime_state_key.rs @@ -1,6 +1,7 @@ //! Rust call sites read and write fixed public names with ordinary Get/Set -//! semantics, through the same probe as an emitted read. Nothing is kept per -//! site: no key registry, no memo, no object-address cache. +//! semantics through the same shape/slot memo as an emitted read. Fixed-key +//! macro expansions own agent-local integer storage; no heap address or +//! getter value is retained. use crate::object::ObjectHeader; /// A fixed key of a Rust call site. Static bytes are not GC pointers. @@ -9,6 +10,7 @@ pub(crate) struct NamedStateKey { pub(crate) bytes: &'static [u8], /// `key_bytes_hash(bytes)`, computed when the site is compiled. pub(crate) hash: u64, + pub(crate) memo: Option *mut crate::object::layout_probe::site_memo::ReadSiteMemo>, } /// `object::key_bytes_hash` (FNV-1a) as a `const fn`, for a fixed key. @@ -28,9 +30,19 @@ macro_rules! runtime_state_key { ($key:expr) => {{ const KEY: &[u8] = $key; const HASH: u64 = $crate::runtime_state_key::const_key_hash(KEY); + fn site_memo() -> *mut $crate::object::layout_probe::site_memo::ReadSiteMemo { + #[thread_local] + static MEMO: std::cell::UnsafeCell< + $crate::object::layout_probe::site_memo::ReadSiteMemo, + > = std::cell::UnsafeCell::new( + $crate::object::layout_probe::site_memo::ReadSiteMemo::EMPTY, + ); + MEMO.get() + } $crate::runtime_state_key::NamedStateKey { bytes: KEY, hash: HASH, + memo: Some(site_memo), } }}; } @@ -41,7 +53,14 @@ impl NamedStateKey { /// # Safety /// `args` is readable for `argc` live JavaScript values. pub(crate) unsafe fn call_value(self, target: f64, args: *const f64, argc: usize) -> f64 { - js_runtime_state_key_call(target, self.bytes.as_ptr(), self.bytes.len(), args, argc) + runtime_state_key_call_with_memo( + target, + self.bytes.as_ptr(), + self.bytes.len(), + args, + argc, + self.memo.map_or(std::ptr::null_mut(), |memo| memo()), + ) } /// Overwrite a proven own writable data slot found in the receiver's @@ -208,7 +227,14 @@ impl NamedStateKey { /// Receiver is already checked by the module's existing receiver gate. pub(crate) unsafe fn read_object(self, obj: *mut ObjectHeader) -> f64 { - crate::object::field_get_set::runtime_read_site::get(obj, self.bytes) + match self.memo { + Some(memo) => crate::object::field_get_set::runtime_read_site::get_with_memo( + obj, + self.bytes, + memo(), + ), + None => crate::object::field_get_set::runtime_read_site::get(obj, self.bytes), + } } /// Runtime-private state keeps own-only semantics, through the same word. @@ -308,6 +334,7 @@ pub unsafe extern "C-unwind" fn js_runtime_state_key_set( let named = NamedStateKey { bytes, hash: const_key_hash(bytes), + memo: None, }; let Some(obj) = crate::object::field_get_set::runtime_read_site::object_receiver(value) else { named.write_value(value, v); @@ -341,6 +368,17 @@ pub unsafe extern "C-unwind" fn js_runtime_state_key_call( len: usize, args: *const f64, argc: usize, +) -> f64 { + runtime_state_key_call_with_memo(value, key, len, args, argc, std::ptr::null_mut()) +} + +unsafe fn runtime_state_key_call_with_memo( + value: f64, + key: *const u8, + len: usize, + args: *const f64, + argc: usize, + memo: *mut crate::object::layout_probe::site_memo::ReadSiteMemo, ) -> f64 { let scope = crate::gc::RuntimeHandleScope::new(); let value = scope.root_nanbox_f64(value); @@ -353,9 +391,10 @@ pub unsafe extern "C-unwind" fn js_runtime_state_key_call( if let Some(obj) = crate::object::field_get_set::runtime_read_site::object_receiver(value.get_nanbox_f64()) { - let method = crate::object::field_get_set::runtime_read_site::get( + let method = crate::object::field_get_set::runtime_read_site::get_with_memo( obj, std::slice::from_raw_parts(key, len), + memo, ); let live = crate::gc::RuntimeHandleScope::refreshed_nanbox_f64_slice(&args); return crate::closure::native_call_value_this( diff --git a/test-files/test_gap_tsp1_prototype_getter_site_memo.ts b/test-files/test_gap_tsp1_prototype_getter_site_memo.ts new file mode 100644 index 00000000000..976074e1641 --- /dev/null +++ b/test-files/test_gap_tsp1_prototype_getter_site_memo.ts @@ -0,0 +1,67 @@ +function opaque(value: any) { + const box = new Map(); + box.set('value', value); + return box.get('value'); +} +let calls = 0; +class Base { + value = 2; + get width() { calls++; return this.value; } +} +class Child extends Base { marker = 1; } +function read(o: any) { return o.width; } +const child: any = opaque(new Child()); +let sum = 0; +for (let i = 0; i < 100; i++) sum += read(child); +console.log('warm', sum, calls); +Object.defineProperty(Base.prototype, 'width', { + get() { calls++; return this.value + 10; }, configurable: true, +}); +console.log('redefined', read(child), calls); +const other = { get width() { calls++; return this.value + 20; } }; +Object.setPrototypeOf(child, other); +console.log('swapped', read(child), calls); +Object.defineProperty(child, 'width', { value: 99, configurable: true }); +console.log('own', read(child), calls); +delete (child as any).width; +console.log('deleted-own', read(child), calls); +const frozen = Object.freeze({ get width() { calls++; return this.value + 30; } }); +const cold: any = opaque(Object.create(frozen)); +cold.value = 4; +for (let i = 0; i < 100; i++) read(cold); +console.log('frozen', read(cold), calls); +const top = { get width() { calls++; return this.value + 40; } }; +const middle = Object.create(top); +const deep: any = opaque(Object.create(middle)); +deep.value = 5; +for (let i = 0; i < 100; i++) read(deep); +console.log('deep', read(deep), calls); +Object.defineProperty(middle, 'width', { value: 77, configurable: true }); +console.log('middle-shadow', read(deep), calls); +delete middle.width; +let traps = 0; +const proxy = new Proxy(top, { + get(target, key, receiver) { + traps++; + return Reflect.get(target, key, receiver); + }, +}); +Object.setPrototypeOf(middle, proxy); +console.log('proxy', read(deep), traps, calls); +const signal: any = opaque({ get width() { calls++; return calls; } }); +console.log('effects', read(signal), read(signal), read(signal)); + +function keyed(o: any, key: string) { return o[key]; } +const keyedProto = { get a() { return 2; }, b: 4 }; +const keyedObject: any = opaque(Object.create(keyedProto)); +const names = ['a', 'b']; +function keyedSum(o: any) { + let total = 0; + for (let i = 0; i < 100; i++) total += keyed(o, names[i & 1]); + return total; +} +console.log('keyed-warm', keyedSum(keyedObject)); +Object.defineProperty(keyedProto, 'a', { get() { return 10; }, configurable: true }); +console.log('keyed-redefined', keyedSum(keyedObject)); +Object.defineProperty(keyedObject, 'a', { value: 20, configurable: true }); +console.log('keyed-own', keyedSum(keyedObject)); From 35f6e3725b4c9632c6ab1341c952d2c416ad7150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 23 Oct 2026 10:53:52 +0000 Subject: [PATCH 02/12] perf(runtime): share shape-slot memo storage and trim property-read dispatch --- .../12986-prototype-getter-site-memo.md | 4 + .../perry-codegen/src/expr/ic_fast_split.rs | 52 +++-- .../src/expr/index_get/symbol_ic.rs | 19 +- .../expr/property_get/front_contract_tests.rs | 40 ++-- .../src/expr/property_get/generic_dispatch.rs | 18 +- .../src/expr/property_get/tests.rs | 12 +- .../src/gc_effects_test_world.tsv | 2 + .../src/native_root_coverage/ic_fast_split.rs | 4 +- .../src/runtime_decls/objects.rs | 2 + .../perry-codegen/src/wasm32/runtime_abi.tsv | 5 + .../src/object/field_get_set/get_slow.rs | 98 +++++--- .../object/field_get_set/runtime_read_site.rs | 7 +- .../perry-runtime/src/object/layout_probe.rs | 86 +++++-- .../src/object/layout_probe_site_memo.rs | 214 +++++++++++------- .../object/layout_probe_site_memo_tests.rs | 197 +++++++++++++++- crates/perry-runtime/src/runtime_state_key.rs | 12 +- 16 files changed, 573 insertions(+), 199 deletions(-) diff --git a/changelog.d/12986-prototype-getter-site-memo.md b/changelog.d/12986-prototype-getter-site-memo.md index 915202e0abd..a3656a81a80 100644 --- a/changelog.d/12986-prototype-getter-site-memo.md +++ b/changelog.d/12986-prototype-getter-site-memo.md @@ -1 +1,5 @@ Repeated property reads can reuse a per-site shape/slot answer for own and inherited getters or data properties, including fixed-name Rust runtime call sites and short computed string keys. Each use checks the live receiver and prototype shapes and loads the current holder slot; getter redefinition, shadowing, prototype swaps and Proxy chains retain ordinary property semantics without epochs or cached getter values. + +Site storage uses shared atomic integer words rather than TLS. Process-global, never-reused ShapeIds let each word validate its own slot proof while readers follow agent-local live links; competing writers can cause misses without introducing a validity flag. The four computed-key ways bind their content keys once, so concurrent publication cannot mix different keys' proofs. This removes eager per-thread clearing of unused memo arrays and TLS address construction at read sites. + +Named leaves retain stateless data handling with four arguments: the memo replaces the hash argument, and hash-using paths read the atom's existing creation hash. A short own-data hit loads neither hash nor memo. Computed reads share receiver and own-property handling with the existing slow path; proven Symbol keys allocate no short-string memo. diff --git a/crates/perry-codegen/src/expr/ic_fast_split.rs b/crates/perry-codegen/src/expr/ic_fast_split.rs index 619699cdd8f..278a7a856cc 100644 --- a/crates/perry-codegen/src/expr/ic_fast_split.rs +++ b/crates/perry-codegen/src/expr/ic_fast_split.rs @@ -156,7 +156,7 @@ pub(crate) fn read_may_reach_getter(ctx: &FnCtx<'_>, property: &str) -> bool { .map_or(true, |names| names.may_read_through_getter(property)) } -/// Agent-local storage for a site's shape/slot answers. No heap pointer or +/// Shared atomic storage for a site's shape/slot answers. No heap pointer or /// validity state is stored, so these zero-filled integer words need no root. pub(crate) fn emit_read_site_memo(ctx: &mut FnCtx<'_>, words: usize) -> String { let site = ctx.ic_site_counter; @@ -165,19 +165,45 @@ pub(crate) fn emit_read_site_memo(ctx: &mut FnCtx<'_>, words: usize) -> String { "@{}_read_shape_slots", crate::expr::inline_cache_global_name(ctx, site) ); - let tls = if ctx.program_has_worker || ctx.program_has_thread_agents { - "thread_local " - } else { - "" - }; ctx.typed_parse_rodata.push(format!( - "{name} = private {tls}global [{words} x i64] zeroinitializer, align 8" + "{name} = private global [{words} x i64] zeroinitializer, align 8" )); name } #[cfg(test)] mod tests { + + #[test] + fn read_site_symbol_key_owns_no_short_string_memo() { + use perry_hir::{Expr, Module, Stmt}; + let mut module = Module::new("read_site_symbol.ts"); + module.init.push(Stmt::Expr(Expr::IndexGet { + object: Box::new(Expr::GlobalGet(100)), + index: Box::new(Expr::SymbolFor(Box::new(Expr::String("key".into())))), + })); + let ir = String::from_utf8( + crate::compile_module( + &module, + crate::CompileOptions { + emit_ir_only: true, + ..Default::default() + }, + ) + .unwrap(), + ) + .unwrap(); + assert!(!ir.contains("_read_shape_slots = private"), "{ir}"); + assert!(ir.contains("call double @perry_probe_get_keyed("), "{ir}"); + assert!( + ir.contains("call double @js_object_get_keyed_slow("), + "{ir}" + ); + assert!( + !ir.contains("call double @js_object_get_keyed_site_slow("), + "{ir}" + ); + } #[test] fn only_a_declared_getter_name_owns_read_site_storage() { use perry_hir::{Expr, Module, Stmt}; @@ -206,7 +232,7 @@ mod tests { .unwrap(); assert_eq!(ir.contains("_read_shape_slots = private"), memo, "{ir}"); assert_eq!( - ir.contains("call double @perry_probe_get_site("), + ir.contains("call double @perry_probe_get_memo("), memo, "{ir}" ); @@ -222,7 +248,7 @@ mod tests { } #[test] - fn read_site_storage_respects_agent_multiplicity() { + fn read_site_storage_never_needs_tls_initialization() { use perry_hir::{Expr, Module, Stmt}; for (worker, thread_agent) in [(false, false), (true, false), (false, true), (true, true)] { let mut module = Module::new("read_site_agents.ts"); @@ -244,16 +270,12 @@ mod tests { .unwrap(), ) .unwrap(); - assert!(ir.contains("call double @perry_probe_get_site("), "{ir}"); + assert!(ir.contains("call double @perry_probe_get_memo("), "{ir}"); let declaration = ir .lines() .find(|line| line.contains("_read_shape_slots = private")) .expect("a generic read must own its shape/slot storage"); - assert_eq!( - declaration.contains("thread_local"), - worker || thread_agent, - "{declaration}" - ); + assert!(!declaration.contains("thread_local"), "{declaration}"); } } } diff --git a/crates/perry-codegen/src/expr/index_get/symbol_ic.rs b/crates/perry-codegen/src/expr/index_get/symbol_ic.rs index 2ded85ec050..fa0afea49ce 100644 --- a/crates/perry-codegen/src/expr/index_get/symbol_ic.rs +++ b/crates/perry-codegen/src/expr/index_get/symbol_ic.rs @@ -16,7 +16,7 @@ pub(crate) fn lower_symbol_property_get_ic( // Object model step 5 (§3.7 case 3): the receiver's own layout answers // the symbol key (`perry_probe_get_keyed`); a decline takes the complete // keyed read. - emit_keyed_probe(ctx, obj_box, sym_box, |ctx, memo| { + emit_keyed_probe_impl(ctx, obj_box, sym_box, false, |ctx, memo| { keyed_site_slow(ctx, obj_box, sym_box, "0", false, memo) }) } @@ -65,6 +65,16 @@ pub(crate) fn emit_keyed_probe( obj: &str, key: &str, slow: impl FnOnce(&mut FnCtx<'_>, &str) -> String, +) -> String { + emit_keyed_probe_impl(ctx, obj, key, true, slow) +} + +fn emit_keyed_probe_impl( + ctx: &mut FnCtx<'_>, + obj: &str, + key: &str, + can_memoize: bool, + slow: impl FnOnce(&mut FnCtx<'_>, &str) -> String, ) -> String { if crate::target_layout::target_is_ilp32(ctx.target_triple) { return slow(ctx, "null"); @@ -77,7 +87,12 @@ pub(crate) fn emit_keyed_probe( let obj_bits = ctx.block().bitcast_double_to_i64(obj); let key_bits = ctx.block().bitcast_double_to_i64(key); // The leaf stays stateless; only its decline consults the site memo. - let memo = crate::expr::ic_fast_split::emit_read_site_memo(ctx, 24); + // Symbol keys cannot use the short-string memo. + let memo = if can_memoize { + crate::expr::ic_fast_split::emit_read_site_memo(ctx, 24) + } else { + "null".into() + }; let probed = ctx.block().call( DOUBLE, "perry_probe_get_keyed", diff --git a/crates/perry-codegen/src/expr/property_get/front_contract_tests.rs b/crates/perry-codegen/src/expr/property_get/front_contract_tests.rs index 45685bdb191..05ebd1a127c 100644 --- a/crates/perry-codegen/src/expr/property_get/front_contract_tests.rs +++ b/crates/perry-codegen/src/expr/property_get/front_contract_tests.rs @@ -1,4 +1,4 @@ -//! The generic read's shape probe (`perry_probe_get_site`) takes the agent's shape +//! The generic read's shape probe (`perry_probe_get_memo`) takes the agent's shape //! directory, whose lookup is target-specific (extra blocks on Apple aarch64, //! a TEB-derived chain on Windows). Follow the CFG and the call operand //! rather than assuming a single block or ELF. @@ -28,7 +28,7 @@ fn verify_front_flow(blocks: &Blocks) -> Result { .enumerate() .filter(|(_, (_, body))| { body.iter() - .any(|line| line.contains("call double @perry_probe_get_site(")) + .any(|line| line.contains("call double @perry_probe_get_memo(")) }) .collect(); let [(index, (call_label, call_body))] = calls.as_slice() else { @@ -99,7 +99,7 @@ fn verify_front_flow(blocks: &Blocks) -> Result { let (cond, served, declined) = tower_cond_br(call_body); let call = call_body .iter() - .find(|line| line.contains("call double @perry_probe_get_site(")) + .find(|line| line.contains("call double @perry_probe_get_memo(")) .unwrap(); let answer = call.split_once(" = ").unwrap().0; let bits = call_body @@ -140,34 +140,38 @@ fn verify_front_flow(blocks: &Blocks) -> Result { Ok(*index) } -/// Both halves of a fixed read must use this site's integer-only storage. +/// The four-argument leaf and collecting exit share the site's integer storage. pub(super) fn verify_front_site_memo(blocks: &Blocks) -> Result { let index = verify_front_flow(blocks)?; let call = blocks[index] .1 .iter() - .find(|line| line.contains("call double @perry_probe_get_site(")) + .find(|line| line.contains("call double @perry_probe_get_memo(")) .unwrap(); - let memo = call - .rsplit_once(", ptr ") - .and_then(|(_, tail)| tail.split_once(')')) - .map(|(memo, _)| memo) - .ok_or("the probe must receive its site storage")?; - if !memo.starts_with("@perry_ic_") || !memo.ends_with("_read_shape_slots") { - return Err(format!( - "the memo must be this site's shape/slot storage: {memo}" - )); + if !call.contains("_read_shape_slots") || call.matches(", ptr ").count() != 1 { + return Err("the four-argument front must receive one memo and no hash".into()); } let slow: Vec<_> = blocks .iter() .flat_map(|(_, body)| body) .filter(|line| line.contains("@js_object_get_field_value_site(")) .collect(); - if slow.len() != 1 || !slow[0].contains(&format!(", ptr {memo})")) { + if slow.len() != 1 { + return Err(format!("one collecting memo exit required: {slow:?}")); + } + let memo = slow[0] + .rsplit_once(", ptr ") + .and_then(|(_, tail)| tail.split_once(')')) + .map(|(memo, _)| memo) + .ok_or("the collecting exit must receive its site storage")?; + if !memo.starts_with("@perry_ic_") || !memo.ends_with("_read_shape_slots") { return Err(format!( - "the leaf and collecting exit must share one memo: {slow:?}" + "the memo must be this site's shape/slot storage: {memo}" )); } + if !call.contains(&format!(", ptr {memo})")) { + return Err("the front and collecting exit must share their memo".into()); + } Ok(memo.to_string()) } @@ -195,7 +199,7 @@ pub(super) fn verify_front_directory(blocks: &Blocks) -> Result<(), String> { let call = blocks[index] .1 .iter() - .find(|line| line.contains("call double @perry_probe_get_site(")) + .find(|line| line.contains("call double @perry_probe_get_memo(")) .unwrap(); let dir = call .split_once("(ptr ") @@ -385,7 +389,7 @@ fn front_contract_rejects_lookup_bypasses_wrong_slots_and_wrong_declines() { let call = wrong[index] .1 .iter_mut() - .find(|l| l.contains("call double @perry_probe_get_site(")) + .find(|l| l.contains("call double @perry_probe_get_memo(")) .unwrap(); let operand = call .split_once("(ptr ") diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 819d0489d1f..7da6ef7d8b0 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -635,15 +635,15 @@ fn lower_property_site_tail( && property != "__proto__" && crate::expr::ic_fast_split::read_may_reach_getter(ctx, property)) .then(|| crate::expr::ic_fast_split::emit_read_site_memo(ctx, 5)); - let mut fast_args = vec![ + let fast_args = vec![ (PTR, dir.as_str()), (I64, obj_bits.as_str()), (I64, key_bits.as_str()), - (I64, hash.as_str()), + memo.as_ref() + .map_or((I64, hash.as_str()), |memo| (PTR, memo.as_str())), ]; let mut slow_args = entry_args.to_vec(); if let Some(memo) = memo.as_ref() { - fast_args.push((PTR, memo.as_str())); slow_args.push((PTR, memo.as_str())); } crate::expr::note_site("probe-outlined"); @@ -651,7 +651,7 @@ fn lower_property_site_tail( ctx, "pget.outline", if memo.is_some() { - "perry_probe_get_value_site" + "perry_probe_get_value_memo" } else { "perry_probe_get_value" }, @@ -875,15 +875,13 @@ fn lower_property_site_tail( && property != "__proto__" && crate::expr::ic_fast_split::read_may_reach_getter(ctx, property)) .then(|| crate::expr::ic_fast_split::emit_read_site_memo(ctx, 5)); - let mut probe_args = vec![ + let probe_args = vec![ (PTR, dir.as_str()), (I64, probe_recv.as_str()), (I64, key_bits.as_str()), - (I64, hash.as_str()), + memo.as_ref() + .map_or((I64, hash.as_str()), |memo| (PTR, memo.as_str())), ]; - if let Some(memo) = memo.as_ref() { - probe_args.push((PTR, memo.as_str())); - } crate::expr::note_site("probe"); let answered = ctx.block().call( DOUBLE, @@ -893,7 +891,7 @@ fn lower_property_site_tail( } else if inline_string_length { "perry_probe_get_length" } else if memo.is_some() { - "perry_probe_get_site" + "perry_probe_get_memo" } else { "perry_probe_get" }, diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index 19271fe97fc..d31b02fead8 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -224,7 +224,7 @@ fn no_call_location_without_debug_symbols() { } /// The generic property read's hit is ONE leaf call of the receiver layout's -/// probe (`perry_probe_get_site`) over the agent's shape directory, the key, +/// probe (`perry_probe_get_memo`) over the agent's shape directory, the key, /// the receiver's handle and the site's five integer shape/slot pairs. There /// is no inline ShapeId compare, no way token and no epoch; the receiver's /// own shape is the authority. @@ -235,7 +235,7 @@ fn generic_property_get_hit_path_is_shape_id_only() { let (_, ok) = tower_block(&blocks, "pget.recv_ok"); let probes: Vec<&String> = ok .iter() - .filter(|l| l.contains("call double @perry_probe_get_site(")) + .filter(|l| l.contains("call double @perry_probe_get_memo(")) .collect(); assert_eq!(probes.len(), 1, "the hit is one probe call: {ok:?}"); assert!( @@ -455,7 +455,7 @@ fn generic_property_get_tries_ways_before_calling_the_miss_handler() { assert!( probe_block .iter() - .any(|l| l.contains("call double @perry_probe_get_site(")), + .any(|l| l.contains("call double @perry_probe_get_memo(")), "{probe_block:?}" ); assert!( @@ -544,7 +544,7 @@ fn a_spill_entry_is_served_by_the_leaf_front_before_the_slow_call() { let (probe_label, probe_block) = front_call_block(&blocks); let call = probe_block .iter() - .find(|l| l.contains("@perry_probe_get_site(")) + .find(|l| l.contains("@perry_probe_get_memo(")) .unwrap_or_else(|| panic!("the probe call: {probe_block:?}")); let answer = call.split_once(" = ").map(|(r, _)| r).unwrap(); let (_, merge) = tower_block(&blocks, "pget.recv_merge"); @@ -801,13 +801,13 @@ fn the_generic_tower_is_one_leaf_call_one_exit_and_a_bounded_number_of_blocks() sorted.sort(); assert_eq!( sorted, - vec!["js_object_get_field_value_site", "perry_probe_get_site"], + vec!["js_object_get_field_value_site", "perry_probe_get_memo"], "the read must expand the probe and exactly one collecting exit:\n{func}" ); // The probe is nounwind and GC-leaf: a plain call, never an invoke. let probes: Vec<&str> = func .lines() - .filter(|l| l.contains("@perry_probe_get_site(")) + .filter(|l| l.contains("@perry_probe_get_memo(")) .collect(); assert_eq!(probes.len(), 1, "one probe call per read:\n{func}"); assert!( diff --git a/crates/perry-codegen/src/gc_effects_test_world.tsv b/crates/perry-codegen/src/gc_effects_test_world.tsv index da6c73ffe56..f696e652062 100644 --- a/crates/perry-codegen/src/gc_effects_test_world.tsv +++ b/crates/perry-codegen/src/gc_effects_test_world.tsv @@ -73,3 +73,5 @@ perry_probe_get_length Leaf no perry_probe_get_value Leaf no perry_probe_get_value_site Leaf no perry_probe_set Leaf no +perry_probe_get_memo Leaf no +perry_probe_get_value_memo Leaf no diff --git a/crates/perry-codegen/src/native_root_coverage/ic_fast_split.rs b/crates/perry-codegen/src/native_root_coverage/ic_fast_split.rs index c7ce08937dc..19fc4bcb26a 100644 --- a/crates/perry-codegen/src/native_root_coverage/ic_fast_split.rs +++ b/crates/perry-codegen/src/native_root_coverage/ic_fast_split.rs @@ -299,11 +299,11 @@ fn read_site_memos_preserve_leaf_and_collecting_root_contracts() { // leaf stays stateless. Both collecting exits take the memo. let fixed = if outlined { ( - "perry_probe_get_value_site", + "perry_probe_get_value_memo", "js_object_get_field_value_site", ) } else { - ("perry_probe_get_site", "js_object_get_field_value_site") + ("perry_probe_get_memo", "js_object_get_field_value_site") }; for (fast, slow) in [ fixed, diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index d114a95ac8c..abebf8046b7 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -407,6 +407,8 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // A GC leaf; `TAG_HOLE` = declined. module.declare_function("perry_probe_family_method", I64, &[PTR, I64, I64, I64, I32]); module.declare_function("perry_probe_get", DOUBLE, &[PTR, I64, I64, I64]); + module.declare_function("perry_probe_get_memo", DOUBLE, &[PTR, I64, I64, PTR]); + module.declare_function("perry_probe_get_value_memo", DOUBLE, &[PTR, I64, I64, PTR]); module.declare_function("perry_probe_get_site", DOUBLE, &[PTR, I64, I64, I64, PTR]); module.declare_function("js_object_get_field_site_slow", DOUBLE, &[I64, I64, PTR]); module.declare_function( diff --git a/crates/perry-codegen/src/wasm32/runtime_abi.tsv b/crates/perry-codegen/src/wasm32/runtime_abi.tsv index c49d253842a..0b06a9ed07e 100644 --- a/crates/perry-codegen/src/wasm32/runtime_abi.tsv +++ b/crates/perry-codegen/src/wasm32/runtime_abi.tsv @@ -233,7 +233,10 @@ js_arraylike_slice f64 f64,f64,i32s,f64,i32s js_arraylike_some f64 f64,f64,f64 js_arraylike_sort f64 f64,f64 js_arraylike_splice f64 f64,ptr,i32s +js_arraylike_toReversed f64 f64 +js_arraylike_toSorted f64 f64,f64 js_arraylike_unshift f64 f64,ptr,i32s +js_arraylike_with f64 f64,f64,f64 js_assert_assert_ctor f64 f64 js_assert_assertion_error_ctor f64 f64 js_assert_deep_equal f64 f64,f64,f64 @@ -5054,8 +5057,10 @@ perry_probe_get f64 ptr,i64,i64,i64 perry_probe_get_binding f64 ptr,i64,i64,i64 perry_probe_get_keyed f64 ptr,i64,i64 perry_probe_get_length f64 ptr,i64,i64,i64 +perry_probe_get_memo f64 ptr,i64,i64,ptr perry_probe_get_site f64 ptr,i64,i64,i64,ptr perry_probe_get_value f64 ptr,i64,i64,i64 +perry_probe_get_value_memo f64 ptr,i64,i64,ptr perry_probe_get_value_site f64 ptr,i64,i64,i64,ptr perry_probe_own_method f64 ptr,f64,i64,i64 perry_probe_set i32s ptr,i64,i64,i64,f64 diff --git a/crates/perry-runtime/src/object/field_get_set/get_slow.rs b/crates/perry-runtime/src/object/field_get_set/get_slow.rs index de3c5239f28..df26926a069 100644 --- a/crates/perry-runtime/src/object/field_get_set/get_slow.rs +++ b/crates/perry-runtime/src/object/field_get_set/get_slow.rs @@ -187,6 +187,17 @@ pub unsafe extern "C-unwind" fn js_object_get_keyed_slow( key: f64, site_id: u64, dynamic: u32, +) -> f64 { + keyed_slow_impl(obj_box, key, site_id, dynamic, None) +} + +#[inline(always)] +unsafe fn keyed_slow_impl( + obj_box: f64, + key: f64, + site_id: u64, + dynamic: u32, + memo: Option<&crate::object::layout_probe::site_memo::KeyedSiteMemo>, ) -> f64 { crate::object::layout_probe::verify::check( (obj_box.to_bits() & crate::value::POINTER_MASK) as usize, @@ -197,6 +208,33 @@ pub unsafe extern "C-unwind" fn js_object_get_keyed_slow( if let Some(value) = crate::object::overflow::own_get_key(recv, key.to_bits()) { return f64::from_bits(value.bits()); } + // Share the complete read's receiver classification and own-data + // handling. A memo attempt cannot precede or repeat that work. + if let Some(memo) = memo { + use crate::object::layout_probe::{ + self, + site_memo::{self, Answer}, + }; + match site_memo::keyed_answer( + crate::object::shapes::ordinary_dir_addr(), + obj_box.to_bits(), + key.to_bits(), + memo, + ) { + Some(Answer::Data(bits)) => return f64::from_bits(bits), + Some(Answer::Getter(pair)) => { + if let Some(acc) = crate::object::accessor_pair::pair_of_value(pair) { + if layout_probe::verify::on() { + layout_probe::verify::accessor(recv as usize, key.to_bits(), &acc); + } + if let Some(value) = call_getter(recv, acc) { + return value; + } + } + } + None => {} + } + } } match recv { Some(recv) if dynamic == 0 => { @@ -215,7 +253,7 @@ pub unsafe extern "C-unwind" fn js_object_get_keyed_slow( /// Collecting half of the same fixed-name site. Revalidate shapes, then /// load and call the holder's CURRENT accessor; no callable is remembered. /// # Safety -/// memo is the agent-local five-pair storage belonging to this fixed key. +/// memo is the shared atomic five-pair storage belonging to this fixed key. #[no_mangle] pub unsafe extern "C-unwind" fn js_object_get_field_site_slow( obj_handle: i64, @@ -239,7 +277,7 @@ pub unsafe extern "C-unwind" fn js_object_get_field_site_slow( obj, key_bits, crate::string::property_atom_hash(key), - &mut *memo, + &*memo, ); match answer { Some(Answer::Data(bits)) => return f64::from_bits(bits), @@ -262,7 +300,7 @@ pub unsafe extern "C-unwind" fn js_object_get_field_site_slow( /// Boxed collecting entrance for the same fixed-name site. Keep the /// original representation migration and non-pointer dispatch unchanged. /// # Safety -/// memo belongs to this fixed key and this agent, as for the raw entrance. +/// memo belongs to this fixed key (shared across agents), as for the raw entrance. #[no_mangle] pub unsafe extern "C-unwind" fn js_object_get_field_value_site( obj_bits: i64, @@ -289,7 +327,7 @@ pub unsafe extern "C-unwind" fn js_object_get_field_value_site( /// Collecting computed-key half. An accessor is read from its holder's /// live slot after the same shape proof used by the fixed-name entrance. /// # Safety -/// memo belongs to this computed read site and this agent. +/// memo belongs to this computed read site (shared across agents). #[no_mangle] pub unsafe extern "C-unwind" fn js_object_get_keyed_site_slow( obj_box: f64, @@ -298,31 +336,33 @@ pub unsafe extern "C-unwind" fn js_object_get_keyed_site_slow( dynamic: u32, memo: *mut crate::object::layout_probe::site_memo::KeyedSiteMemo, ) -> f64 { - use crate::object::layout_probe::{ - self, - site_memo::{self, Answer}, + let bits = key.to_bits(); + let tag = bits & crate::value::TAG_MASK; + let may_fit = if tag == crate::value::SHORT_STRING_TAG { + true + } else if tag == crate::value::STRING_TAG { + let string = (bits & crate::value::POINTER_MASK) as *const crate::StringHeader; + crate::string::is_valid_string_ptr(string) + && (*string).byte_len as usize <= crate::value::SHORT_STRING_MAX_LEN + } else { + false }; - let addr = (obj_box.to_bits() & crate::value::POINTER_MASK) as usize; - layout_probe::verify::check(addr, key.to_bits()); - let answer = site_memo::keyed_answer( - crate::object::shapes::ordinary_dir_addr(), - obj_box.to_bits(), - key.to_bits(), - &mut *memo, - ); - match answer { - Some(Answer::Data(bits)) => return f64::from_bits(bits), - Some(Answer::Getter(pair)) => { - if let Some(acc) = crate::object::accessor_pair::pair_of_value(pair) { - if layout_probe::verify::on() { - layout_probe::verify::accessor(addr, key.to_bits(), &acc); - } - if let Some(v) = call_getter(addr as *const ObjectHeader, acc) { - return v; - } - } - } - None => {} + if may_fit { + keyed_memo_slow(obj_box, key, site_id, dynamic, memo) + } else { + js_object_get_keyed_slow(obj_box, key, site_id, dynamic) } - js_object_get_keyed_slow(obj_box, key, site_id, dynamic) +} + +/// Keep the refusal dispatcher frameless: unsupported keys tail-call the +/// complete read without reserving the memo path's registers or stack. +#[inline(never)] +unsafe fn keyed_memo_slow( + obj_box: f64, + key: f64, + site_id: u64, + dynamic: u32, + memo: *const crate::object::layout_probe::site_memo::KeyedSiteMemo, +) -> f64 { + keyed_slow_impl(obj_box, key, site_id, dynamic, Some(&*memo)) } diff --git a/crates/perry-runtime/src/object/field_get_set/runtime_read_site.rs b/crates/perry-runtime/src/object/field_get_set/runtime_read_site.rs index c1638e663f4..3d391d24acf 100644 --- a/crates/perry-runtime/src/object/field_get_set/runtime_read_site.rs +++ b/crates/perry-runtime/src/object/field_get_set/runtime_read_site.rs @@ -8,7 +8,7 @@ //! directory, and, when the probe declines, the cacheless complete read //! (`js_object_get_field_slow`). Fixed-name Rust sites use `get_with_memo`: //! the same leaf and collecting entrance as an emitted fixed-name read that -//! owns a shape/slot memo (`perry_probe_get_site`, which asks the memo only +//! owns a shape/slot memo (`perry_probe_get_memo`, which asks the memo only //! after an own miss, then `js_object_get_field_site_slow`). The memo retains //! no heap addresses or getter values. //! @@ -120,7 +120,7 @@ pub(crate) unsafe fn get(obj: *mut ObjectHeader, key: &[u8]) -> f64 { /// asks it; only the collecting entrance derives it. /// /// # Safety -/// As [`get`]; `memo` is null or this site's agent-local storage. +/// As [`get`]; `memo` is null or this site's shared atomic storage. pub(crate) unsafe fn get_with_memo( obj: *mut ObjectHeader, key: &[u8], @@ -145,11 +145,10 @@ pub(crate) unsafe fn get_with_memo( let v = if memo.is_null() { probe_word(obj, key_bits, hash) } else { - crate::object::layout_probe::site_memo::perry_probe_get_site( + crate::object::layout_probe::site_memo::perry_probe_get_memo( crate::object::shapes::ordinary_dir_addr(), (obj as usize).wrapping_sub(perry_abi::RECEIVER_HANDLE_FLOOR) as i64, key_bits, - hash, memo, ) }; diff --git a/crates/perry-runtime/src/object/layout_probe.rs b/crates/perry-runtime/src/object/layout_probe.rs index 26053d6fad9..8c6d45e942b 100644 --- a/crates/perry-runtime/src/object/layout_probe.rs +++ b/crates/perry-runtime/src/object/layout_probe.rs @@ -922,14 +922,25 @@ unsafe fn probe_get( key_bits: u64, hash: u64, ) -> f64 { - probe_get_at::(dir, obj_biased, key_bits, hash, std::ptr::null()) + probe_get_at::(dir, obj_biased, key_bits, hash, std::ptr::null()) +} + +/// Memo fronts use the fourth register for the memo. Their key is a +/// property atom; only hash-using paths load its existing creation hash. +#[inline(always)] +unsafe fn site_key_hash(key_bits: u64, hash: u64) -> u64 { + if LAZY_HASH { + crate::string::property_atom_hash((key_bits & POINTER_MASK) as *const crate::StringHeader) + } else { + hash + } } /// The own-data front. With `SITE` (a compile-time choice) every own miss /// first asks the site's shape/slot memo (`site_memo`), then continues /// exactly as the stateless probe; an own hit never reads the memo. #[inline(always)] -unsafe fn probe_get_at( +unsafe fn probe_get_at( dir: *const u8, obj_biased: i64, key_bits: u64, @@ -943,21 +954,38 @@ unsafe fn probe_get_at( let word = (*obj).parent_class_id; let r = record_of(dir, word); if r.is_null() { - return array_named_get::(dir, obj, key_bits, hash); + return array_named_get::( + dir, + obj, + key_bits, + site_key_hash::(key_bits, hash), + ); } let probe = (*r).probe; if probe & (PROBE_READY | PROBE_DECLINE | PROBE_CHECKED) != PROBE_READY { if SITE { - return site_memo::miss_underived::(dir, obj, key_bits, hash, memo); - } - return probe_get_underived::(dir, obj, word, r, key_bits, hash); + return site_memo::miss_underived::(dir, obj, key_bits, hash, memo); + } + return probe_get_underived::( + dir, + obj, + word, + r, + key_bits, + site_key_hash::(key_bits, hash), + ); } let count = (*r).logical_key_count; if count == 0 { return if SITE { - site_memo::miss_absent::(dir, obj, key_bits, hash, memo) + site_memo::miss_absent::(dir, obj, key_bits, hash, memo) } else { - absent_from::(dir, obj, key_bits, hash) + absent_from::( + dir, + obj, + key_bits, + site_key_hash::(key_bits, hash), + ) }; } let table = (probe & !PROBE_FLAGS) as *const KeyIndex; @@ -979,9 +1007,14 @@ unsafe fn probe_get_at( i += 1; if i == count { return if SITE { - site_memo::miss_absent::(dir, obj, key_bits, hash, memo) + site_memo::miss_absent::(dir, obj, key_bits, hash, memo) } else { - absent_from::(dir, obj, key_bits, hash) + absent_from::( + dir, + obj, + key_bits, + site_key_hash::(key_bits, hash), + ) }; } } @@ -989,26 +1022,38 @@ unsafe fn probe_get_at( let list = crate::array::array_elements_ptr((*r).keys as usize as *const _) as *const u64; let mask = (*table).mask as usize; let slots = KeyIndex::buckets(table); - let mut b = hash as usize & mask; + let mut b = site_key_hash::(key_bits, hash) as usize & mask; loop { let pos = *slots.add(b); if pos == u32::MAX { return if SITE { - site_memo::miss_absent::(dir, obj, key_bits, hash, memo) + site_memo::miss_absent::(dir, obj, key_bits, hash, memo) } else { - absent_from::(dir, obj, key_bits, hash) + absent_from::( + dir, + obj, + key_bits, + site_key_hash::(key_bits, hash), + ) }; } let i = pos & POS_BITS; if pos & TEXT_ENTRY != 0 { // Only a key of the text's hash can be its text. - if KeyIndex::word(table, i) != hash { + if KeyIndex::word(table, i) != site_key_hash::(key_bits, hash) { b = (b + 1) & mask; continue; } // A text compare is a call: the cold path makes it, so the // hit path keeps no frame. - return probe_get_underived::(dir, obj, word, r, key_bits, hash); + return probe_get_underived::( + dir, + obj, + word, + r, + key_bits, + site_key_hash::(key_bits, hash), + ); } let w = if pos & ALIAS_ENTRY != 0 { KeyIndex::word(table, i) @@ -1028,9 +1073,16 @@ unsafe fn probe_get_at( return v; } if SITE { - return site_memo::miss_hole::(dir, obj, pos, key_bits, hash, memo); + return site_memo::miss_hole::(dir, obj, pos, key_bits, hash, memo); } - own_hole::(dir, obj, r, pos, key_bits, hash) + own_hole::( + dir, + obj, + r, + pos, + key_bits, + site_key_hash::(key_bits, hash), + ) } /// Is position `pos` of record `r` a plain data key (no accessor, no private diff --git a/crates/perry-runtime/src/object/layout_probe_site_memo.rs b/crates/perry-runtime/src/object/layout_probe_site_memo.rs index fa296e40871..38e98e74214 100644 --- a/crates/perry-runtime/src/object/layout_probe_site_memo.rs +++ b/crates/perry-runtime/src/object/layout_probe_site_memo.rs @@ -2,16 +2,12 @@ //! The entries contain no heap addresses: each use follows the live shape //! links and reads the current holder slot, including its current getter pair. //! -//! Only a collecting entrance writes a memo: a read the stateless probe -//! declined (a getter on the chain, a computed key a prototype holds) proves -//! the memo's shapes, and otherwise derives it from the live chain before the -//! complete read. A fixed-name site's leaf (`perry_probe_get_site`) is the -//! stateless own-data front; only after an own miss does it compare the -//! memo's first ShapeId with the receiver's and, on a match, answer from the -//! memo (a getter still declines to the collecting entrance). An own hit -//! never reads the memo, and no leaf ever writes it. Computed-key leaves stay -//! stateless (`perry_probe_get_keyed`). +//! Only a collecting entrance writes a memo. Named leaves retain stateless +//! data handling and ask the memo after an own miss. Their four-argument ABI +//! passes the memo instead of a hash: short own lists need neither; other +//! paths load the atom's existing creation hash. Keyed leaves stay stateless. use super::*; +use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; const PAIRS: usize = 5; const NEXT: u32 = u32::MAX; @@ -25,14 +21,29 @@ const SPILL: u32 = 1 << 30; const SLOT_MASK: u32 = SPILL - 1; /// Exactly five (ShapeId, slot) pairs. Zero shape means an empty answer. -/// Kept in the site's own TLS storage, never indexed through a registry. -#[derive(Clone, Copy)] +/// Shared integer storage, never indexed through a registry. ShapeIds are +/// process-global and never reused. Each atomic word is a complete proof for +/// this fixed key; a racing writer can cause a shape mismatch, never a torn +/// (shape, slot). Every continuation revalidates the live link and next shape. +/// There is no whole-memo validity bit, epoch, or cached heap address. #[repr(C)] pub struct ReadSiteMemo { - pairs: [u64; PAIRS], + pairs: [AtomicU64; PAIRS], } impl ReadSiteMemo { - pub(crate) const EMPTY: Self = Self { pairs: [0; PAIRS] }; + pub(crate) const EMPTY: Self = Self { + pairs: [const { AtomicU64::new(0) }; PAIRS], + }; + + fn publish(&self, pairs: &[u64; PAIRS]) { + for (dst, &src) in self.pairs.iter().zip(pairs).rev() { + dst.store(src, Relaxed); + } + } + + fn snapshot(&self) -> [u64; PAIRS] { + std::array::from_fn(|i| self.pairs[i].load(Relaxed)) + } } #[derive(Clone, Copy)] @@ -97,7 +108,8 @@ pub(crate) unsafe fn lookup( mut obj: *const ObjectHeader, memo: &ReadSiteMemo, ) -> Option { - for &pair in &memo.pairs { + for entry in &memo.pairs { + let pair = entry.load(Relaxed); let shape = (pair >> 32) as u32; if shape == 0 || (*obj).parent_class_id != shape || ordinary_word(obj)? != shape { return None; @@ -137,20 +149,26 @@ pub(crate) unsafe fn lookup( /// Has this memo found that a receiver of `obj`'s shape has no memo answer? #[inline(always)] unsafe fn declined(obj: *const ObjectHeader, memo: &ReadSiteMemo) -> bool { - memo.pairs[0] == (u64::from((*obj).parent_class_id) << 32) | u64::from(DECLINE) + memo.pairs[0].load(Relaxed) == (u64::from((*obj).parent_class_id) << 32) | u64::from(DECLINE) } /// Remember that a receiver of `obj`'s shape has no memo answer, unless the /// memo already holds a positive answer for another shape. #[inline] -unsafe fn decline(obj: *const ObjectHeader, memo: &mut ReadSiteMemo) { +unsafe fn decline(obj: *const ObjectHeader, memo: &ReadSiteMemo) { let word = (*obj).parent_class_id; - let held = memo.pairs[0]; + let held = memo.pairs[0].load(Relaxed); if crate::object::shapes::is_site_matchable_shape_id(word) && (held == 0 || held as u32 == DECLINE) { - *memo = ReadSiteMemo::EMPTY; - memo.pairs[0] = (u64::from(word) << 32) | u64::from(DECLINE); + // A decline uses only word zero. Do not clear a racing positive + // proof's tail; the CAS also preserves a concurrently published proof. + let _ = memo.pairs[0].compare_exchange( + held, + (u64::from(word) << 32) | u64::from(DECLINE), + Relaxed, + Relaxed, + ); } } @@ -163,7 +181,7 @@ pub(crate) unsafe fn collect( obj: *const ObjectHeader, key_bits: u64, hash: u64, - memo: &mut ReadSiteMemo, + memo: &ReadSiteMemo, ) -> Option { if let Some(answer) = lookup(dir, obj, memo) { return Some(answer); @@ -187,9 +205,9 @@ pub(crate) unsafe fn prime( mut obj: *const ObjectHeader, key_bits: u64, hash: u64, - memo: &mut ReadSiteMemo, + memo: &ReadSiteMemo, ) -> Option { - let mut learned = ReadSiteMemo::EMPTY; + let mut learned = [0u64; PAIRS]; for i in 0..PAIRS { let shape = ordinary_word(obj)?; let r = record_of(dir, shape); @@ -227,9 +245,9 @@ pub(crate) unsafe fn prime( } else { 0 }; - learned.pairs[i] = prefix | u64::from(slot); + learned[i] = prefix | u64::from(slot); if let Some(answer) = slot_answer(obj, slot) { - *memo = learned; + memo.publish(&learned); return Some(answer); } if slot & ACCESSOR != 0 { @@ -243,15 +261,15 @@ pub(crate) unsafe fn prime( } match link_of(dir, obj, r) { Next::End => { - if learned.pairs[i] == 0 { - learned.pairs[i] = prefix | u64::from(ABSENT); + if learned[i] == 0 { + learned[i] = prefix | u64::from(ABSENT); } - *memo = learned; + memo.publish(&learned); return Some(Answer::Data(crate::value::TAG_UNDEFINED)); } Next::Object(next) => { - if learned.pairs[i] == 0 { - learned.pairs[i] = prefix | u64::from(NEXT); + if learned[i] == 0 { + learned[i] = prefix | u64::from(NEXT); } obj = next; } @@ -270,7 +288,7 @@ unsafe fn leaf_answer( key_bits: u64, memo: *const ReadSiteMemo, ) -> Option { - if ((*memo).pairs[0] >> 32) as u32 != (*obj).parent_class_id { + if ((*memo).pairs[0].load(Relaxed) >> 32) as u32 != (*obj).parent_class_id { return None; } Some(match lookup(dir, obj, &*memo)? { @@ -282,7 +300,7 @@ unsafe fn leaf_answer( /// The own list proved the key absent. #[inline(never)] -pub(super) unsafe fn miss_absent( +pub(super) unsafe fn miss_absent( dir: *const u8, obj: *const ObjectHeader, key_bits: u64, @@ -292,12 +310,13 @@ pub(super) unsafe fn miss_absent( if let Some(v) = leaf_answer(dir, obj, key_bits, memo) { return v; } + let hash = site_key_hash::(key_bits, hash); absent_from::(dir, obj, key_bits, hash) } /// The receiver's record is not derived, or is checked (accessor/private). #[inline(never)] -pub(super) unsafe fn miss_underived( +pub(super) unsafe fn miss_underived( dir: *const u8, obj: *const ObjectHeader, key_bits: u64, @@ -307,13 +326,14 @@ pub(super) unsafe fn miss_underived( if let Some(v) = leaf_answer(dir, obj, key_bits, memo) { return v; } + let hash = site_key_hash::(key_bits, hash); let word = (*obj).parent_class_id; probe_get_underived::(dir, obj, word, record_of(dir, word), key_bits, hash) } /// The own slot at position `pos` read a hole. #[inline(never)] -pub(super) unsafe fn miss_hole( +pub(super) unsafe fn miss_hole( dir: *const u8, obj: *const ObjectHeader, pos: u32, @@ -324,6 +344,7 @@ pub(super) unsafe fn miss_hole( if let Some(v) = leaf_answer(dir, obj, key_bits, memo) { return v; } + let hash = site_key_hash::(key_bits, hash); own_hole::( dir, obj, @@ -334,10 +355,49 @@ pub(super) unsafe fn miss_hole( ) } +/// Four-argument memo front. Own hits do not read the memo, and short own +/// lists need no hash. Other data reads retain the stateless probe behavior. +/// # Safety +/// As perry_probe_get_site, but key_bits must be a property STRING atom. +#[no_mangle] +pub unsafe extern "C" fn perry_probe_get_memo( + dir: *const u8, + obj_biased: i64, + key_bits: u64, + memo: *const ReadSiteMemo, +) -> f64 { + probe_get_at::(dir, obj_biased, key_bits, 0, memo) +} + +/// Boxed four-argument fixed-name memo front. +/// # Safety +/// As perry_probe_get_value_site, but the key must be a property STRING atom. +#[no_mangle] +pub unsafe extern "C" fn perry_probe_get_value_memo( + dir: *const u8, + obj_bits: u64, + key_bits: u64, + memo: *const ReadSiteMemo, +) -> f64 { + if obj_bits & TAG_MASK != POINTER_TAG { + return hole(); + } + let addr = (obj_bits & POINTER_MASK) as usize; + if addr < perry_abi::RECEIVER_HANDLE_FLOOR { + return hole(); + } + perry_probe_get_memo( + dir, + (addr - perry_abi::RECEIVER_HANDLE_FLOOR) as i64, + key_bits, + memo, + ) +} + /// The fixed-name leaf of a site that owns a memo: [`perry_probe_get`] with /// the memo asked after an own miss. A GC leaf; it never writes the memo. /// # Safety -/// As [`perry_probe_get`]; `memo` is this site's agent-local storage. +/// As [`perry_probe_get`]; `memo` is this site's shared atomic storage. #[no_mangle] pub unsafe extern "C" fn perry_probe_get_site( dir: *const u8, @@ -346,13 +406,13 @@ pub unsafe extern "C" fn perry_probe_get_site( hash: u64, memo: *const ReadSiteMemo, ) -> f64 { - probe_get_at::(dir, obj_biased, key_bits, hash, memo) + probe_get_at::(dir, obj_biased, key_bits, hash, memo) } /// [`perry_probe_get_value`] for an outlined or generator site that owns a /// memo. /// # Safety -/// As [`perry_probe_get_value`]; `memo` is this site's agent-local storage. +/// As [`perry_probe_get_value`]; `memo` is this site's shared atomic storage. #[no_mangle] pub unsafe extern "C" fn perry_probe_get_value_site( dir: *const u8, @@ -377,14 +437,15 @@ pub unsafe extern "C" fn perry_probe_get_value_site( ) } -#[derive(Clone, Copy)] #[repr(C)] struct KeyedWay { - key: u64, + // Set once from zero to SSO content, never reassigned. Thus every pair + // in this way proves the same key even while another agent publishes it. + key: AtomicU64, memo: ReadSiteMemo, } -/// Four shape answers for a small rotating set of short string keys. +/// Four shape answers for short string keys, each way bound once on use. /// The key is CONTENT bits, never a GC address or a cached program value. #[repr(C)] pub struct KeyedSiteMemo { @@ -408,7 +469,7 @@ pub(crate) unsafe fn keyed_answer( dir: *const u8, obj_bits: u64, key_bits: u64, - memo: &mut KeyedSiteMemo, + memo: &KeyedSiteMemo, ) -> Option { if obj_bits & TAG_MASK != POINTER_TAG { return None; @@ -419,51 +480,44 @@ pub(crate) unsafe fn keyed_answer( } let key = short_key(key_bits)?; let obj = addr as *const ObjectHeader; - for way in &memo.ways { - if way.key == key { - if let Some(answer) = lookup(dir, obj, &way.memo) { - return Some(answer); - } - if declined(obj, &way.memo) { - return None; - } + if let Some(way) = memo.ways.iter().find(|way| way.key.load(Relaxed) == key) { + if let Some(answer) = lookup(dir, obj, &way.memo) { + return Some(answer); } - } - let mut learned = ReadSiteMemo::EMPTY; - let answer = match computed_key(key_bits) { - Some((canonical, hash)) => prime(dir, obj, canonical, hash, &mut learned), - None => None, - }; - let Some(answer) = answer else { - // This key on this receiver shape has no memo answer: remember the - // decline so its next read takes the complete read at once. - decline(obj, &mut learned); - if learned.pairs[0] != 0 { - remember(memo, KeyedWay { key, memo: learned }); + if declined(obj, &way.memo) { + return None; } + let (canonical, hash) = computed_key(key_bits)?; + return collect(dir, obj, canonical, hash, &way.memo); + } + if memo.ways.iter().all(|way| way.key.load(Relaxed) != 0) { return None; - }; - // A computed key is retained only when it is the content of a real - // holder key at the recorded slot. Absence has no such key to derive - // from the shape. Return this fresh proof without remembering its key. - if learned - .pairs - .iter() - .any(|&pair| pair != 0 && pair as u32 == ABSENT) - { - return Some(answer); } - remember(memo, KeyedWay { key, memo: learned }); - Some(answer) -} - -/// The newest way first; the oldest leaves. -#[inline] -fn remember(memo: &mut KeyedSiteMemo, way: KeyedWay) { - for i in (1..memo.ways.len()).rev() { - memo.ways[i] = memo.ways[i - 1]; + let (canonical, hash) = computed_key(key_bits)?; + let learned = ReadSiteMemo::EMPTY; + let answer = collect(dir, obj, canonical, hash, &learned); + let pairs = learned.snapshot(); + // An absent key has no holder slot to derive its content from. Serve + // that fresh proof without retaining its key, as the stateless path does. + if pairs.iter().any(|&pair| pair != 0 && pair as u32 == ABSENT) { + return answer; + } + if pairs[0] == 0 { + return answer; } - memo.ways[0] = way; + // Claim only an empty way: replacing a key would let another reader + // observe a proof for a different key. A full site uses the complete read. + if let Some(way) = + memo.ways.iter().find( + |way| match way.key.compare_exchange(0, key, Relaxed, Relaxed) { + Ok(_) => true, + Err(found) => found == key, + }, + ) + { + way.memo.publish(&pairs); + } + answer } #[cfg(test)] diff --git a/crates/perry-runtime/src/object/layout_probe_site_memo_tests.rs b/crates/perry-runtime/src/object/layout_probe_site_memo_tests.rs index 9daf5fc6f60..b560178a0aa 100644 --- a/crates/perry-runtime/src/object/layout_probe_site_memo_tests.rs +++ b/crates/perry-runtime/src/object/layout_probe_site_memo_tests.rs @@ -202,22 +202,32 @@ fn rust_runtime_sites_share_the_memo_and_preserve_getter_this() { } #[test] -fn leaf_answers_never_touch_the_site_memo() { +fn data_leaf_answers_never_write_the_site_memo() { let _lock = crate::gc::global_side_table_test_lock(); let _no_move = crate::gc::GcSuppressScope::new(); unsafe { let own = plain(b"value", 1.0); let recv = inherited(plain(b"value", 2.0)); let key = crate::runtime_state_key!(b"value"); + // Derive the own layout independently, then prove its ready fast + // hit does not enter the memo's collecting exit. + let atom = atom(b"value"); + let _ = perry_probe_get( + crate::object::shapes::ordinary_dir_addr(), + (own as usize - perry_abi::RECEIVER_HANDLE_FLOOR) as i64, + STRING_TAG | atom as u64, + crate::string::property_atom_hash(atom), + ); assert_eq!(key.read_object(own), 1.0); - assert_eq!(key.read_object(recv), 2.0); let memo = key.memo.unwrap()(); + assert_eq!((*memo).snapshot(), [0; PAIRS], "own hit is stateless"); + assert_eq!(key.read_object(recv), 2.0); assert_eq!( - (*memo).pairs, + (*memo).snapshot(), [0; PAIRS], - "own and inherited data are stateless probe answers" + "inherited data stays in the leaf" ); - // The control: a declined read (an inherited getter) does derive. + // The collecting exit also reloads and invokes inherited getters. unsafe extern "C-unwind" fn seven(_: f64) -> f64 { 7.0 } @@ -234,7 +244,7 @@ fn leaf_answers_never_touch_the_site_memo() { let getter_recv = inherited(proto); assert_eq!(key.read_object(getter_recv), 7.0); assert_ne!( - (*memo).pairs[0], + (*memo).pairs[0].load(Relaxed), 0, "the collecting entrance derives the memo" ); @@ -340,9 +350,180 @@ fn a_decline_is_a_shape_checked_answer_that_never_displaces_a_proof() { data(collect(dir, a, word, hash, &mut memo)), 1.0f64.to_bits() ); - let proof = memo; + let proof = memo.snapshot(); decline(b, &mut memo); - assert_eq!(memo.pairs, proof.pairs, "a decline never displaces a proof"); + assert_eq!(memo.snapshot(), proof, "a decline never displaces a proof"); assert_eq!(data(lookup(dir, a, &memo)), 1.0f64.to_bits()); } } + +#[test] +fn shared_site_words_remain_proofs_under_concurrent_agent_writes() { + let _lock = crate::gc::global_side_table_test_lock(); + let memo = std::sync::Arc::new(ReadSiteMemo::EMPTY); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let threads: Vec<_> = (1..=2) + .map(|n| { + let memo = memo.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + let _no_move = crate::gc::GcSuppressScope::new(); + unsafe { + let recv = inherited(plain(b"value", n as f64)); + let key = atom(b"value"); + let bits = STRING_TAG | key as u64; + let hash = crate::string::property_atom_hash(key); + let dir = crate::object::shapes::ordinary_dir_addr(); + barrier.wait(); + for _ in 0..20000 { + assert_eq!( + data(collect(dir, recv, bits, hash, &memo)), + (n as f64).to_bits() + ); + if let Some(answer) = lookup(dir, recv, &memo) { + assert_eq!(data(Some(answer)), (n as f64).to_bits()); + } + } + } + }) + }) + .collect(); + for thread in threads { + thread.join().unwrap(); + } +} + +#[test] +fn keyed_ways_never_rebind_a_published_proofs_key() { + let _lock = crate::gc::global_side_table_test_lock(); + let _no_move = crate::gc::GcSuppressScope::new(); + unsafe { + let memo: KeyedSiteMemo = std::mem::zeroed(); + let obj = crate::object::js_object_alloc(0, 8); + for (i, key) in [b"a", b"b", b"c", b"d", b"e"].iter().enumerate() { + crate::object::js_object_set_field_by_name(obj, atom(*key), i as f64); + } + let dir = crate::object::shapes::ordinary_dir_addr(); + let bits = crate::value::js_nanbox_pointer(obj as i64).to_bits(); + for (i, key) in [b"a", b"b", b"c", b"d"].iter().enumerate() { + assert_eq!( + data(keyed_answer( + dir, + bits, + STRING_TAG | atom(*key) as u64, + &memo + )), + (i as f64).to_bits() + ); + } + assert!(keyed_answer(dir, bits, STRING_TAG | atom(b"e") as u64, &memo).is_none()); + for (i, key) in [b"a", b"b", b"c", b"d"].iter().enumerate() { + assert_eq!( + data(keyed_answer( + dir, + bits, + STRING_TAG | atom(*key) as u64, + &memo + )), + (i as f64).to_bits() + ); + } + } +} + +#[test] +fn shared_keyed_ways_do_not_mix_different_keys_proofs() { + let _lock = crate::gc::global_side_table_test_lock(); + let memo = std::sync::Arc::new(unsafe { std::mem::zeroed::() }); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let threads: Vec<_> = (0..2) + .map(|agent| { + let memo = memo.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + let _no_move = crate::gc::GcSuppressScope::new(); + barrier.wait(); + unsafe { + let obj = crate::object::js_object_alloc(0, 4); + let a = atom(b"a"); + let b = atom(b"b"); + // Opposite insertion order makes an accidental key/proof + // combination name the wrong value even within one shape. + let keys = if agent == 0 { [a, b] } else { [b, a] }; + for (slot, key) in keys.into_iter().enumerate() { + crate::object::js_object_set_field_by_name( + obj, + key, + (100 * agent + slot) as f64, + ); + } + let dir = crate::object::shapes::ordinary_dir_addr(); + let bits = crate::value::js_nanbox_pointer(obj as i64).to_bits(); + for i in 0..20000 { + let slot = i & 1; + assert_eq!( + data(keyed_answer( + dir, + bits, + STRING_TAG | keys[slot] as u64, + &memo + )), + ((100 * agent + slot) as f64).to_bits(), + ); + } + } + }) + }) + .collect(); + for thread in threads { + thread.join().unwrap(); + } +} + +#[test] +fn four_argument_leaf_retains_stateless_data_and_live_shadowing() { + let _lock = crate::gc::global_side_table_test_lock(); + let _no_move = crate::gc::GcSuppressScope::new(); + unsafe { + let recv = inherited(plain(b"value", 7.0)); + let bits = STRING_TAG | atom(b"value") as u64; + let dir = crate::object::shapes::ordinary_dir_addr(); + let boxed = crate::value::js_nanbox_pointer(recv as i64).to_bits(); + let memo = ReadSiteMemo::EMPTY; + assert_eq!(perry_probe_get_value_memo(dir, boxed, bits, &memo), 7.0); + assert_eq!(memo.snapshot(), [0; PAIRS]); + crate::object::js_object_set_field_by_name(recv, atom(b"value"), 9.0); + assert_eq!(perry_probe_get_value_memo(dir, boxed, bits, &memo), 9.0); + assert_eq!(memo.snapshot(), [0; PAIRS]); + } +} + +#[test] +fn four_argument_leaf_uses_the_atoms_hash_for_indexed_keys() { + let _lock = crate::gc::global_side_table_test_lock(); + let _no_move = crate::gc::GcSuppressScope::new(); + unsafe { + let obj = crate::object::js_object_alloc(0, 32); + for i in 0..16 { + let name = format!("field{i}"); + crate::object::js_object_set_field_by_name(obj, atom(name.as_bytes()), i as f64); + } + let dir = crate::object::shapes::ordinary_dir_addr(); + let biased = (obj as usize - perry_abi::RECEIVER_HANDLE_FLOOR) as i64; + let memo = ReadSiteMemo::EMPTY; + for i in 0..17 { + let name = format!("field{i}"); + let key = atom(name.as_bytes()); + let bits = STRING_TAG | key as u64; + let expected = + perry_probe_get(dir, biased, bits, crate::string::property_atom_hash(key)); + let actual = perry_probe_get_memo(dir, biased, bits, &memo); + assert_eq!(actual.to_bits(), expected.to_bits(), "{name}"); + } + assert_ne!( + (*record_of(dir, (*obj).parent_class_id)).probe & !PROBE_FLAGS, + 0, + "the fixture must exercise the indexed hash path" + ); + } +} diff --git a/crates/perry-runtime/src/runtime_state_key.rs b/crates/perry-runtime/src/runtime_state_key.rs index 08677accead..110f5d24995 100644 --- a/crates/perry-runtime/src/runtime_state_key.rs +++ b/crates/perry-runtime/src/runtime_state_key.rs @@ -1,6 +1,6 @@ //! Rust call sites read and write fixed public names with ordinary Get/Set //! semantics through the same shape/slot memo as an emitted read. Fixed-key -//! macro expansions own agent-local integer storage; no heap address or +//! macro expansions own shared atomic integer storage; no heap address or //! getter value is retained. use crate::object::ObjectHeader; @@ -31,13 +31,9 @@ macro_rules! runtime_state_key { const KEY: &[u8] = $key; const HASH: u64 = $crate::runtime_state_key::const_key_hash(KEY); fn site_memo() -> *mut $crate::object::layout_probe::site_memo::ReadSiteMemo { - #[thread_local] - static MEMO: std::cell::UnsafeCell< - $crate::object::layout_probe::site_memo::ReadSiteMemo, - > = std::cell::UnsafeCell::new( - $crate::object::layout_probe::site_memo::ReadSiteMemo::EMPTY, - ); - MEMO.get() + static MEMO: $crate::object::layout_probe::site_memo::ReadSiteMemo = + $crate::object::layout_probe::site_memo::ReadSiteMemo::EMPTY; + &MEMO as *const _ as *mut _ } $crate::runtime_state_key::NamedStateKey { bytes: KEY, From 518b1d69ec7a86d35eaa3cba875b257cc1a03194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 23 Oct 2026 12:38:12 +0000 Subject: [PATCH 03/12] perf(runtime): tail-call unproved slot loads from property probes --- .../12986-prototype-getter-site-memo.md | 2 ++ .../perry-runtime/src/object/layout_probe.rs | 8 +++++ .../src/object/layout_probe_unloaded.rs | 31 +++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 crates/perry-runtime/src/object/layout_probe_unloaded.rs diff --git a/changelog.d/12986-prototype-getter-site-memo.md b/changelog.d/12986-prototype-getter-site-memo.md index a3656a81a80..05759e97404 100644 --- a/changelog.d/12986-prototype-getter-site-memo.md +++ b/changelog.d/12986-prototype-getter-site-memo.md @@ -3,3 +3,5 @@ Repeated property reads can reuse a per-site shape/slot answer for own and inher Site storage uses shared atomic integer words rather than TLS. Process-global, never-reused ShapeIds let each word validate its own slot proof while readers follow agent-local live links; competing writers can cause misses without introducing a validity flag. The four computed-key ways bind their content keys once, so concurrent publication cannot mix different keys' proofs. This removes eager per-thread clearing of unused memo arrays and TLS address construction at read sites. Named leaves retain stateless data handling with four arguments: the memo replaces the hash argument, and hash-using paths read the atom's existing creation hash. A short own-data hit loads neither hash nor memo. Computed reads share receiver and own-property handling with the existing slow path; proven Symbol keys allocate no short-string memo. + +Unproved own-slot loads use a shared tail entrance. Ordinary inline loads and memo probe refusals no longer save registers for that uncommon load call. diff --git a/crates/perry-runtime/src/object/layout_probe.rs b/crates/perry-runtime/src/object/layout_probe.rs index 8c6d45e942b..ddfe53451ab 100644 --- a/crates/perry-runtime/src/object/layout_probe.rs +++ b/crates/perry-runtime/src/object/layout_probe.rs @@ -38,6 +38,8 @@ //! `TAG_HOLE`, and the site takes its complete slow operation. #[path = "layout_probe_site_memo.rs"] pub(crate) mod site_memo; +#[path = "layout_probe_unloaded.rs"] +mod unloaded; use super::shapes_store::{ShapeRecord, FLAT_BASE_OFFSET}; use crate::object::layout_desc::note_run; @@ -1066,6 +1068,12 @@ unsafe fn probe_get_at(dir, obj, key_bits, hash, pos, memo); + } let v = hit_value(obj, r, pos, load); // A hole slot: the key is absent (the key view), and the answer // continues as for an absent key. diff --git a/crates/perry-runtime/src/object/layout_probe_unloaded.rs b/crates/perry-runtime/src/object/layout_probe_unloaded.rs new file mode 100644 index 00000000000..242d4672ad8 --- /dev/null +++ b/crates/perry-runtime/src/object/layout_probe_unloaded.rs @@ -0,0 +1,31 @@ +//! The complete load when a probe entry does not prove an inline slot. +//! Re-derive the record here to keep the tail entrance within six integer +//! arguments. The probe is a GC leaf, so its receiver and shape stay live. +use super::*; + +#[inline(never)] +pub(super) unsafe fn get( + dir: *const u8, + obj: *const ObjectHeader, + key_bits: u64, + hash: u64, + pos: u32, + memo: *const site_memo::ReadSiteMemo, +) -> f64 { + let r = record_of(dir, (*obj).parent_class_id); + let v = value_at(obj, r, pos); + if v.to_bits() != TAG_HOLE { + return v; + } + if SITE { + return site_memo::miss_hole::(dir, obj, pos, key_bits, hash, memo); + } + own_hole::( + dir, + obj, + r, + pos, + key_bits, + site_key_hash::(key_bits, hash), + ) +} From 41c97420cf4e9d16ea8ccbe77649ef7fb23f0a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 23 Oct 2026 13:00:28 +0000 Subject: [PATCH 04/12] perf(runtime): separate indexed probe lookup and tail continuations --- .../perry-runtime/src/object/layout_probe.rs | 115 ++++-------------- .../src/object/layout_probe_indexed.rs | 52 ++++++++ .../src/object/layout_probe_unloaded.rs | 65 ++++++++++ 3 files changed, 139 insertions(+), 93 deletions(-) create mode 100644 crates/perry-runtime/src/object/layout_probe_indexed.rs diff --git a/crates/perry-runtime/src/object/layout_probe.rs b/crates/perry-runtime/src/object/layout_probe.rs index ddfe53451ab..bc63ac59a3d 100644 --- a/crates/perry-runtime/src/object/layout_probe.rs +++ b/crates/perry-runtime/src/object/layout_probe.rs @@ -36,6 +36,8 @@ //! absent. Everything else (an accessor, an exotic or per-object receiver, a //! chain that ends without the key, a builtin prototype that misses) is //! `TAG_HOLE`, and the site takes its complete slow operation. +#[path = "layout_probe_indexed.rs"] +mod indexed; #[path = "layout_probe_site_memo.rs"] pub(crate) mod site_memo; #[path = "layout_probe_unloaded.rs"] @@ -990,107 +992,34 @@ unsafe fn probe_get_at(dir, obj, key_bits, hash, memo) - } else { - absent_from::( - dir, - obj, - key_bits, - site_key_hash::(key_bits, hash), - ) - }; - } + if probe & !PROBE_FLAGS != 0 { + return indexed::get::(dir, obj, r, key_bits, hash, memo); + } + // An unindexed list has no accessor or private entry. + let words = crate::array::array_elements_ptr((*r).keys as usize as *const _) as *const u64; + let count = (*r).logical_key_count; + let bound = (*r).position_bound_raw(); + let mut pos = 0u32; + loop { + if *words.add(pos as usize) == key_bits { + break; } - } else { - let list = crate::array::array_elements_ptr((*r).keys as usize as *const _) as *const u64; - let mask = (*table).mask as usize; - let slots = KeyIndex::buckets(table); - let mut b = site_key_hash::(key_bits, hash) as usize & mask; - loop { - let pos = *slots.add(b); - if pos == u32::MAX { - return if SITE { - site_memo::miss_absent::(dir, obj, key_bits, hash, memo) - } else { - absent_from::( - dir, - obj, - key_bits, - site_key_hash::(key_bits, hash), - ) - }; - } - let i = pos & POS_BITS; - if pos & TEXT_ENTRY != 0 { - // Only a key of the text's hash can be its text. - if KeyIndex::word(table, i) != site_key_hash::(key_bits, hash) { - b = (b + 1) & mask; - continue; - } - // A text compare is a call: the cold path makes it, so the - // hit path keeps no frame. - return probe_get_underived::( + pos += 1; + if pos == count { + return if SITE { + site_memo::miss_absent::(dir, obj, key_bits, hash, memo) + } else { + absent_from::( dir, obj, - word, - r, key_bits, site_key_hash::(key_bits, hash), - ); - } - let w = if pos & ALIAS_ENTRY != 0 { - KeyIndex::word(table, i) - } else { - *list.add(i as usize) + ) }; - if w == key_bits { - break (i, entry_load(table, pos, i)); - } - b = (b + 1) & mask; } - }; - // Only the uncommon unproved load calls value_at. Keep that call and - // its continuation together behind a tail entrance, so direct own - // loads and other probe exits do not save registers for a later call. - if load == NO_LOAD { - return unloaded::get::(dir, obj, key_bits, hash, pos, memo); - } - let v = hit_value(obj, r, pos, load); - // A hole slot: the key is absent (the key view), and the answer - // continues as for an absent key. - if v.to_bits() != TAG_HOLE { - return v; - } - if SITE { - return site_memo::miss_hole::(dir, obj, pos, key_bits, hash, memo); } - own_hole::( - dir, - obj, - r, - pos, - key_bits, - site_key_hash::(key_bits, hash), - ) + let load = if pos < bound { pos } else { NO_LOAD }; + unloaded::finish::(dir, obj, key_bits, hash, pos, load, memo) } /// Is position `pos` of record `r` a plain data key (no accessor, no private diff --git a/crates/perry-runtime/src/object/layout_probe_indexed.rs b/crates/perry-runtime/src/object/layout_probe_indexed.rs new file mode 100644 index 00000000000..ca5a7aacb0e --- /dev/null +++ b/crates/perry-runtime/src/object/layout_probe_indexed.rs @@ -0,0 +1,52 @@ +//! Indexed own-key lookup and its continuations share one tail entrance. +//! The list-only front does not keep the table loop's registers live. +use super::*; + +#[inline(never)] +pub(super) unsafe fn get( + dir: *const u8, + obj: *const ObjectHeader, + r: *const ShapeRecord, + key_bits: u64, + hash: u64, + memo: *const site_memo::ReadSiteMemo, +) -> f64 { + let table = ((*r).probe & !PROBE_FLAGS) as *const KeyIndex; + let list = crate::array::array_elements_ptr((*r).keys as usize as *const _) as *const u64; + let mask = (*table).mask as usize; + let slots = KeyIndex::buckets(table); + let mut b = site_key_hash::(key_bits, hash) as usize & mask; + let (pos, load) = loop { + let pos = *slots.add(b); + if pos == u32::MAX { + return if SITE { + site_memo::miss_absent::(dir, obj, key_bits, hash, memo) + } else { + absent_from::( + dir, + obj, + key_bits, + site_key_hash::(key_bits, hash), + ) + }; + } + let i = pos & POS_BITS; + if pos & TEXT_ENTRY != 0 { + if KeyIndex::word(table, i) != site_key_hash::(key_bits, hash) { + b = (b + 1) & mask; + continue; + } + return unloaded::underived::(dir, obj, key_bits, hash); + } + let w = if pos & ALIAS_ENTRY != 0 { + KeyIndex::word(table, i) + } else { + *list.add(i as usize) + }; + if w == key_bits { + break (i, entry_load(table, pos, i)); + } + b = (b + 1) & mask; + }; + unloaded::finish::(dir, obj, key_bits, hash, pos, load, memo) +} diff --git a/crates/perry-runtime/src/object/layout_probe_unloaded.rs b/crates/perry-runtime/src/object/layout_probe_unloaded.rs index 242d4672ad8..06b919a8a21 100644 --- a/crates/perry-runtime/src/object/layout_probe_unloaded.rs +++ b/crates/perry-runtime/src/object/layout_probe_unloaded.rs @@ -29,3 +29,68 @@ pub(super) unsafe fn get(key_bits, hash), ) } + +/// Finish a proved inline load without retaining a ShapeRecord through the +/// lookup loop. The unproved and hole cases derive their record at the exit. +#[inline(always)] +pub(super) unsafe fn finish( + dir: *const u8, + obj: *const ObjectHeader, + key_bits: u64, + hash: u64, + pos: u32, + load: u32, + memo: *const site_memo::ReadSiteMemo, +) -> f64 { + if load == NO_LOAD { + return get::(dir, obj, key_bits, hash, pos, memo); + } + let v = *((obj as *const u8).add(std::mem::size_of::() + load as usize * 8) + as *const f64); + if v.to_bits() != TAG_HOLE { + return v; + } + hole_at::(dir, obj, key_bits, hash, pos, memo) +} + +#[inline(never)] +unsafe fn hole_at( + dir: *const u8, + obj: *const ObjectHeader, + key_bits: u64, + hash: u64, + pos: u32, + memo: *const site_memo::ReadSiteMemo, +) -> f64 { + if SITE { + return site_memo::miss_hole::(dir, obj, pos, key_bits, hash, memo); + } + let r = record_of(dir, (*obj).parent_class_id); + own_hole::( + dir, + obj, + r, + pos, + key_bits, + site_key_hash::(key_bits, hash), + ) +} + +/// Keep the indexed loop's text exit from retaining the record and word. +#[inline(never)] +pub(super) unsafe fn underived( + dir: *const u8, + obj: *const ObjectHeader, + key_bits: u64, + hash: u64, +) -> f64 { + let word = (*obj).parent_class_id; + probe_get_underived::( + dir, + obj, + word, + record_of(dir, word), + key_bits, + site_key_hash::(key_bits, hash), + ) +} From 7b0f28540786521f75d86735b37339d012d97087 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 23 Oct 2026 14:20:56 +0000 Subject: [PATCH 05/12] perf(runtime): tail-continue indexed collisions and memo matches --- .../12986-prototype-getter-site-memo.md | 2 + .../src/object/layout_probe_indexed.rs | 69 +++++++++++++++++-- .../src/object/layout_probe_site_memo.rs | 66 ++++++++++++++++-- .../object/layout_probe_site_memo_tests.rs | 43 ++++++++++++ 4 files changed, 170 insertions(+), 10 deletions(-) diff --git a/changelog.d/12986-prototype-getter-site-memo.md b/changelog.d/12986-prototype-getter-site-memo.md index 05759e97404..2b377b25687 100644 --- a/changelog.d/12986-prototype-getter-site-memo.md +++ b/changelog.d/12986-prototype-getter-site-memo.md @@ -5,3 +5,5 @@ Site storage uses shared atomic integer words rather than TLS. Process-global, n Named leaves retain stateless data handling with four arguments: the memo replaces the hash argument, and hash-using paths read the atom's existing creation hash. A short own-data hit loads neither hash nor memo. Computed reads share receiver and own-property handling with the existing slow path; proven Symbol keys allocate no short-string memo. Unproved own-slot loads use a shared tail entrance. Ordinary inline loads and memo probe refusals no longer save registers for that uncommon load call. + +Indexed reads tail-continue hash collisions separately from the first bucket, and existing accessor entries decline directly to the collecting entrance. Memo mismatches continue statelessly before saving registers for a matched proof. diff --git a/crates/perry-runtime/src/object/layout_probe_indexed.rs b/crates/perry-runtime/src/object/layout_probe_indexed.rs index ca5a7aacb0e..c79dd12f6f3 100644 --- a/crates/perry-runtime/src/object/layout_probe_indexed.rs +++ b/crates/perry-runtime/src/object/layout_probe_indexed.rs @@ -1,5 +1,5 @@ -//! Indexed own-key lookup and its continuations share one tail entrance. -//! The list-only front does not keep the table loop's registers live. +//! The existing indexed lookup starts with one bucket and tail-continues +//! collisions. A hit need not retain the collision loop's mask and index. use super::*; #[inline(never)] @@ -11,11 +11,68 @@ pub(super) unsafe fn get f64 { + let table = ((*r).probe & !PROBE_FLAGS) as *const KeyIndex; + let b = site_key_hash::(key_bits, hash) as usize & (*table).mask as usize; + let pos = *KeyIndex::buckets(table).add(b); + if pos == u32::MAX { + return if SITE { + site_memo::miss_absent::(dir, obj, key_bits, hash, memo) + } else { + absent_from::( + dir, + obj, + key_bits, + site_key_hash::(key_bits, hash), + ) + }; + } + // The complete text path performs the hash/content comparison and + // continues collisions. No text hash need stay live on an atom hit. + if pos & TEXT_ENTRY != 0 { + return unloaded::underived::(dir, obj, key_bits, hash); + } + let i = pos & POS_BITS; + let word = if pos & ALIAS_ENTRY != 0 { + KeyIndex::word(table, i) + } else { + let list = crate::array::array_elements_ptr((*r).keys as usize as *const _) as *const u64; + *list.add(i as usize) + }; + if word != key_bits { + return continued::( + dir, + obj, + (b + 1) as u32, + key_bits, + hash, + memo, + ); + } + // This existing shape fact proves a public accessor. The leaf must + // decline it; the collecting entrance reloads the live getter pair. + let load = entry_load(table, pos, i); + if load == NO_LOAD && pos & ACCESSOR_ENTRY != 0 { + return hole(); + } + unloaded::finish::(dir, obj, key_bits, hash, i, load, memo) +} + +#[inline(never)] +unsafe fn continued( + dir: *const u8, + obj: *const ObjectHeader, + start: u32, + key_bits: u64, + hash: u64, + memo: *const site_memo::ReadSiteMemo, +) -> f64 { + // No collection or user code occurs between the first bucket and here. + let r = record_of(dir, (*obj).parent_class_id); let table = ((*r).probe & !PROBE_FLAGS) as *const KeyIndex; let list = crate::array::array_elements_ptr((*r).keys as usize as *const _) as *const u64; let mask = (*table).mask as usize; let slots = KeyIndex::buckets(table); - let mut b = site_key_hash::(key_bits, hash) as usize & mask; + let mut b = start as usize & mask; let (pos, load) = loop { let pos = *slots.add(b); if pos == u32::MAX { @@ -44,7 +101,11 @@ pub(super) unsafe fn get Option { - if ((*memo).pairs[0].load(Relaxed) >> 32) as u32 != (*obj).parent_class_id { - return None; - } Some(match lookup(dir, obj, &*memo)? { Answer::Data(bits) => verify::answer(obj as usize, key_bits, f64::from_bits(bits)), // The collecting entrance reloads and calls the live getter. @@ -299,13 +296,29 @@ unsafe fn leaf_answer( } /// The own list proved the key absent. -#[inline(never)] +#[inline(always)] pub(super) unsafe fn miss_absent( dir: *const u8, obj: *const ObjectHeader, key_bits: u64, hash: u64, memo: *const ReadSiteMemo, +) -> f64 { + let pair = (*memo).pairs[0].load(Relaxed); + if (pair >> 32) as u32 == (*obj).parent_class_id { + return miss_absent_matched::(dir, obj, key_bits, hash, memo); + } + let hash = site_key_hash::(key_bits, hash); + absent_from::(dir, obj, key_bits, hash) +} + +#[inline(never)] +unsafe fn miss_absent_matched( + dir: *const u8, + obj: *const ObjectHeader, + key_bits: u64, + hash: u64, + memo: *const ReadSiteMemo, ) -> f64 { if let Some(v) = leaf_answer(dir, obj, key_bits, memo) { return v; @@ -315,13 +328,30 @@ pub(super) unsafe fn miss_absent( } /// The receiver's record is not derived, or is checked (accessor/private). -#[inline(never)] +#[inline(always)] pub(super) unsafe fn miss_underived( dir: *const u8, obj: *const ObjectHeader, key_bits: u64, hash: u64, memo: *const ReadSiteMemo, +) -> f64 { + let pair = (*memo).pairs[0].load(Relaxed); + if (pair >> 32) as u32 == (*obj).parent_class_id { + return miss_underived_matched::(dir, obj, key_bits, hash, memo); + } + let hash = site_key_hash::(key_bits, hash); + let word = (*obj).parent_class_id; + probe_get_underived::(dir, obj, word, record_of(dir, word), key_bits, hash) +} + +#[inline(never)] +unsafe fn miss_underived_matched( + dir: *const u8, + obj: *const ObjectHeader, + key_bits: u64, + hash: u64, + memo: *const ReadSiteMemo, ) -> f64 { if let Some(v) = leaf_answer(dir, obj, key_bits, memo) { return v; @@ -332,7 +362,7 @@ pub(super) unsafe fn miss_underived( dir: *const u8, obj: *const ObjectHeader, @@ -340,6 +370,30 @@ pub(super) unsafe fn miss_hole( key_bits: u64, hash: u64, memo: *const ReadSiteMemo, +) -> f64 { + let pair = (*memo).pairs[0].load(Relaxed); + if (pair >> 32) as u32 == (*obj).parent_class_id { + return miss_hole_matched::(dir, obj, pos, key_bits, hash, memo); + } + let hash = site_key_hash::(key_bits, hash); + own_hole::( + dir, + obj, + record_of(dir, (*obj).parent_class_id), + pos, + key_bits, + hash, + ) +} + +#[inline(never)] +unsafe fn miss_hole_matched( + dir: *const u8, + obj: *const ObjectHeader, + pos: u32, + key_bits: u64, + hash: u64, + memo: *const ReadSiteMemo, ) -> f64 { if let Some(v) = leaf_answer(dir, obj, key_bits, memo) { return v; diff --git a/crates/perry-runtime/src/object/layout_probe_site_memo_tests.rs b/crates/perry-runtime/src/object/layout_probe_site_memo_tests.rs index b560178a0aa..7f8583e5b93 100644 --- a/crates/perry-runtime/src/object/layout_probe_site_memo_tests.rs +++ b/crates/perry-runtime/src/object/layout_probe_site_memo_tests.rs @@ -527,3 +527,46 @@ fn four_argument_leaf_uses_the_atoms_hash_for_indexed_keys() { ); } } + +#[test] +fn indexed_collisions_preserve_each_distinct_slots_value() { + let _lock = crate::gc::global_side_table_test_lock(); + let _no_move = crate::gc::GcSuppressScope::new(); + unsafe { + let obj = crate::object::js_object_alloc(0, 32); + let mut keys = Vec::new(); + for n in 0..100000 { + let name = format!("collision{n}"); + let hash = crate::object::key_bytes_hash(name.as_ptr(), name.len()); + if hash & 127 == 0 { + let key = atom(name.as_bytes()); + crate::object::js_object_set_field_by_name(obj, key, keys.len() as f64); + keys.push(key); + if keys.len() == 16 { + break; + } + } + } + assert_eq!(keys.len(), 16); + let dir = crate::object::shapes::ordinary_dir_addr(); + let r = record_of(dir, (*obj).parent_class_id); + let table = (probe_word(r) & !PROBE_FLAGS) as *const KeyIndex; + assert!(!table.is_null()); + assert!((*table).mask <= 127, "all fixture keys must collide"); + let biased = (obj as usize - perry_abi::RECEIVER_HANDLE_FLOOR) as i64; + let memo = ReadSiteMemo::EMPTY; + for (slot, key) in keys.into_iter().enumerate() { + let bits = STRING_TAG | key as u64; + assert_eq!(perry_probe_get_memo(dir, biased, bits, &memo), slot as f64); + assert_eq!( + perry_probe_get(dir, biased, bits, crate::string::property_atom_hash(key)), + slot as f64 + ); + } + assert_eq!( + memo.snapshot(), + [0; PAIRS], + "own hits do not write the memo" + ); + } +} From 770afdc2cf81897a7a7c5fa14cff723c9b73aff6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 23 Oct 2026 14:35:43 +0000 Subject: [PATCH 06/12] perf(runtime): reconstruct collision cursors only on continuation --- .../src/object/layout_probe_indexed.rs | 15 ++++----------- .../src/object/layout_probe_site_memo.rs | 6 +++--- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/crates/perry-runtime/src/object/layout_probe_indexed.rs b/crates/perry-runtime/src/object/layout_probe_indexed.rs index c79dd12f6f3..5cf8e0a696f 100644 --- a/crates/perry-runtime/src/object/layout_probe_indexed.rs +++ b/crates/perry-runtime/src/object/layout_probe_indexed.rs @@ -39,14 +39,7 @@ pub(super) unsafe fn get( - dir, - obj, - (b + 1) as u32, - key_bits, - hash, - memo, - ); + return continued::(dir, obj, key_bits, hash, memo); } // This existing shape fact proves a public accessor. The leaf must // decline it; the collecting entrance reloads the live getter pair. @@ -61,18 +54,18 @@ pub(super) unsafe fn get( dir: *const u8, obj: *const ObjectHeader, - start: u32, key_bits: u64, hash: u64, memo: *const site_memo::ReadSiteMemo, ) -> f64 { - // No collection or user code occurs between the first bucket and here. + // Reconstruct the cursor only after a collision. No collection or user + // code occurs between the first bucket and here. let r = record_of(dir, (*obj).parent_class_id); let table = ((*r).probe & !PROBE_FLAGS) as *const KeyIndex; let list = crate::array::array_elements_ptr((*r).keys as usize as *const _) as *const u64; let mask = (*table).mask as usize; let slots = KeyIndex::buckets(table); - let mut b = start as usize & mask; + let mut b = (site_key_hash::(key_bits, hash) as usize).wrapping_add(1) & mask; let (pos, load) = loop { let pos = *slots.add(b); if pos == u32::MAX { diff --git a/crates/perry-runtime/src/object/layout_probe_site_memo.rs b/crates/perry-runtime/src/object/layout_probe_site_memo.rs index ef83ed9978a..3da5503b9ee 100644 --- a/crates/perry-runtime/src/object/layout_probe_site_memo.rs +++ b/crates/perry-runtime/src/object/layout_probe_site_memo.rs @@ -296,7 +296,7 @@ unsafe fn leaf_answer( } /// The own list proved the key absent. -#[inline(always)] +#[inline(never)] pub(super) unsafe fn miss_absent( dir: *const u8, obj: *const ObjectHeader, @@ -328,7 +328,7 @@ unsafe fn miss_absent_matched( } /// The receiver's record is not derived, or is checked (accessor/private). -#[inline(always)] +#[inline(never)] pub(super) unsafe fn miss_underived( dir: *const u8, obj: *const ObjectHeader, @@ -362,7 +362,7 @@ unsafe fn miss_underived_matched( } /// The own slot at position `pos` read a hole. -#[inline(always)] +#[inline(never)] pub(super) unsafe fn miss_hole( dir: *const u8, obj: *const ObjectHeader, From 20cbe4cab81860ac1bdcad1d3019b99189dffb4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 23 Oct 2026 14:44:43 +0000 Subject: [PATCH 07/12] perf(runtime): select indexed key storage before retaining its slot --- .../src/object/layout_probe_indexed.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/perry-runtime/src/object/layout_probe_indexed.rs b/crates/perry-runtime/src/object/layout_probe_indexed.rs index 5cf8e0a696f..3907cd2ccea 100644 --- a/crates/perry-runtime/src/object/layout_probe_indexed.rs +++ b/crates/perry-runtime/src/object/layout_probe_indexed.rs @@ -31,14 +31,15 @@ pub(super) unsafe fn get(dir, obj, key_bits, hash); } - let i = pos & POS_BITS; - let word = if pos & ALIAS_ENTRY != 0 { - KeyIndex::word(table, i) + // Select the key storage before retaining its slot index. This keeps + // the list header arithmetic from extending the index's live range. + let words = if pos & ALIAS_ENTRY != 0 { + (*table).words.as_ptr() } else { - let list = crate::array::array_elements_ptr((*r).keys as usize as *const _) as *const u64; - *list.add(i as usize) + crate::array::array_elements_ptr((*r).keys as usize as *const _) as *const u64 }; - if word != key_bits { + let i = pos & POS_BITS; + if *words.add(i as usize) != key_bits { return continued::(dir, obj, key_bits, hash, memo); } // This existing shape fact proves a public accessor. The leaf must From a1a9f818992fc3e63efa83a237eaa9df46ef2b69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 23 Oct 2026 15:28:36 +0000 Subject: [PATCH 08/12] perf(runtime): reuse shape proofs and index state on collisions --- .../12986-prototype-getter-site-memo.md | 2 + crates/perry-runtime/src/array/storage.rs | 10 ++++- .../src/object/layout_probe_indexed.rs | 25 ++++++++---- .../src/object/layout_probe_site_memo.rs | 39 +++++++++++++++++++ .../object/layout_probe_site_memo_tests.rs | 19 ++++++++- 5 files changed, 85 insertions(+), 10 deletions(-) diff --git a/changelog.d/12986-prototype-getter-site-memo.md b/changelog.d/12986-prototype-getter-site-memo.md index 2b377b25687..afdf695f792 100644 --- a/changelog.d/12986-prototype-getter-site-memo.md +++ b/changelog.d/12986-prototype-getter-site-memo.md @@ -7,3 +7,5 @@ Named leaves retain stateless data handling with four arguments: the memo replac Unproved own-slot loads use a shared tail entrance. Ordinary inline loads and memo probe refusals no longer save registers for that uncommon load call. Indexed reads tail-continue hash collisions separately from the first bucket, and existing accessor entries decline directly to the collecting entrance. Memo mismatches continue statelessly before saving registers for a matched proof. + +At an indexed collision, a matching memo can prove the complete read before further key traversal; an unanswered proof resumes the same index. Collision continuations reuse its existing table and key-array pointers. Array element addresses use the same allocation extent and remaining capacity directly in bytes, preserving queue offsets and named-property reserves without new metadata. diff --git a/crates/perry-runtime/src/array/storage.rs b/crates/perry-runtime/src/array/storage.rs index cf047c8e7f7..1f561cc233a 100644 --- a/crates/perry-runtime/src/array/storage.rs +++ b/crates/perry-runtime/src/array/storage.rs @@ -39,7 +39,15 @@ pub(crate) unsafe fn array_front_offset(arr: *const ArrayHeader) -> usize { /// interior pointers, the result must not be retained across a safepoint. #[inline] pub unsafe fn array_elements_ptr(arr: *const ArrayHeader) -> *mut u64 { - (arr.add(1) as *mut u64).add(array_front_offset(arr)) + // Both headers occupy whole element words. Rounding the allocation + // down to a word preserves physical_capacity's division, including any + // tail padding, without converting bytes to slots and back again. + const { assert!(crate::gc::GC_HEADER_SIZE % 8 == 0 && std::mem::size_of::() % 8 == 0) }; + let header = (arr as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; + let end = ((*header).size as usize & !7) - crate::gc::GC_HEADER_SIZE; + (arr as *const u8) + .add(end) + .sub((*arr).capacity as usize * 8) as *mut u64 } /// Lower an array's `length` to `new_length`, writing `TAG_HOLE` into every diff --git a/crates/perry-runtime/src/object/layout_probe_indexed.rs b/crates/perry-runtime/src/object/layout_probe_indexed.rs index 3907cd2ccea..39a1a24b1c5 100644 --- a/crates/perry-runtime/src/object/layout_probe_indexed.rs +++ b/crates/perry-runtime/src/object/layout_probe_indexed.rs @@ -33,14 +33,23 @@ pub(super) unsafe fn get(dir, obj, key_bits, hash, memo); + // A matching memo proves the entire live chain, including any + // own key in a later bucket. An unanswered proof still continues + // this same index; the first bucket is not treated as own absence. + if SITE { + return site_memo::collision::( + dir, obj, key_bits, hash, memo, table, keys, + ); + } + return continued::(dir, obj, key_bits, hash, memo, table, keys); } // This existing shape fact proves a public accessor. The leaf must // decline it; the collecting entrance reloads the live getter pair. @@ -52,18 +61,18 @@ pub(super) unsafe fn get( +pub(super) unsafe fn continued( dir: *const u8, obj: *const ObjectHeader, key_bits: u64, hash: u64, memo: *const site_memo::ReadSiteMemo, + table: *const KeyIndex, + keys: u64, ) -> f64 { - // Reconstruct the cursor only after a collision. No collection or user - // code occurs between the first bucket and here. - let r = record_of(dir, (*obj).parent_class_id); - let table = ((*r).probe & !PROBE_FLAGS) as *const KeyIndex; - let list = crate::array::array_elements_ptr((*r).keys as usize as *const _) as *const u64; + // Continue with the same existing table and key array. No collection + // or user code occurs between the first bucket and here. + let list = crate::array::array_elements_ptr(keys as usize as *const _) as *const u64; let mask = (*table).mask as usize; let slots = KeyIndex::buckets(table); let mut b = (site_key_hash::(key_bits, hash) as usize).wrapping_add(1) & mask; diff --git a/crates/perry-runtime/src/object/layout_probe_site_memo.rs b/crates/perry-runtime/src/object/layout_probe_site_memo.rs index 3da5503b9ee..449022b278a 100644 --- a/crates/perry-runtime/src/object/layout_probe_site_memo.rs +++ b/crates/perry-runtime/src/object/layout_probe_site_memo.rs @@ -295,6 +295,45 @@ unsafe fn leaf_answer( }) } +/// An indexed atom comparison collided. The existing shape/slot proof +/// may already answer the whole read; otherwise resume the index exactly +/// after its first bucket. This does not infer absence from a collision. +#[inline(never)] +pub(super) unsafe fn collision( + dir: *const u8, + obj: *const ObjectHeader, + key_bits: u64, + hash: u64, + memo: *const ReadSiteMemo, + table: *const KeyIndex, + keys: u64, +) -> f64 { + let pair = (*memo).pairs[0].load(Relaxed); + if (pair >> 32) as u32 == (*obj).parent_class_id && pair as u32 != DECLINE { + return collision_matched::(dir, obj, key_bits, hash, memo); + } + indexed::continued::(dir, obj, key_bits, hash, memo, table, keys) +} + +#[inline(never)] +unsafe fn collision_matched( + dir: *const u8, + obj: *const ObjectHeader, + key_bits: u64, + hash: u64, + memo: *const ReadSiteMemo, +) -> f64 { + if let Some(v) = leaf_answer(dir, obj, key_bits, memo) { + return v; + } + // A matched but unanswered proof may have a live hole or an unproved + // link. Only that case reconstructs the table after the proof attempt. + let r = record_of(dir, (*obj).parent_class_id); + let table = ((*r).probe & !PROBE_FLAGS) as *const KeyIndex; + let keys = (*r).keys; + indexed::continued::(dir, obj, key_bits, hash, memo, table, keys) +} + /// The own list proved the key absent. #[inline(never)] pub(super) unsafe fn miss_absent( diff --git a/crates/perry-runtime/src/object/layout_probe_site_memo_tests.rs b/crates/perry-runtime/src/object/layout_probe_site_memo_tests.rs index 7f8583e5b93..4b4924f11b0 100644 --- a/crates/perry-runtime/src/object/layout_probe_site_memo_tests.rs +++ b/crates/perry-runtime/src/object/layout_probe_site_memo_tests.rs @@ -555,7 +555,7 @@ fn indexed_collisions_preserve_each_distinct_slots_value() { assert!((*table).mask <= 127, "all fixture keys must collide"); let biased = (obj as usize - perry_abi::RECEIVER_HANDLE_FLOOR) as i64; let memo = ReadSiteMemo::EMPTY; - for (slot, key) in keys.into_iter().enumerate() { + for (slot, key) in keys.iter().copied().enumerate() { let bits = STRING_TAG | key as u64; assert_eq!(perry_probe_get_memo(dir, biased, bits, &memo), slot as f64); assert_eq!( @@ -568,5 +568,22 @@ fn indexed_collisions_preserve_each_distinct_slots_value() { [0; PAIRS], "own hits do not write the memo" ); + // The later buckets must also remain correct when a memo can + // answer before collision traversal, including a live slot write. + for (slot, key) in keys.iter().copied().enumerate() { + let bits = STRING_TAG | key as u64; + let hash = crate::string::property_atom_hash(key); + let warm = ReadSiteMemo::EMPTY; + assert_eq!( + data(prime(dir, obj, bits, hash, &warm)), + (slot as f64).to_bits() + ); + assert_eq!(perry_probe_get_memo(dir, biased, bits, &warm), slot as f64); + let shape = (*obj).parent_class_id; + let value = (1000 + slot) as f64; + crate::object::js_object_set_field_by_name(obj, key, value); + assert_eq!((*obj).parent_class_id, shape); + assert_eq!(perry_probe_get_memo(dir, biased, bits, &warm), value); + } } } From 04fee5f1924bbe53b6d7128d9de32905c9de2b00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 23 Oct 2026 15:57:14 +0000 Subject: [PATCH 09/12] perf(runtime): keep indexed hits free of collision cursor state --- changelog.d/12986-prototype-getter-site-memo.md | 2 ++ crates/perry-runtime/src/array/storage.rs | 12 +++++------- .../src/object/layout_probe_indexed.rs | 15 ++++++++------- .../src/object/layout_probe_site_memo.rs | 11 +++-------- 4 files changed, 18 insertions(+), 22 deletions(-) diff --git a/changelog.d/12986-prototype-getter-site-memo.md b/changelog.d/12986-prototype-getter-site-memo.md index afdf695f792..14d2cf8803a 100644 --- a/changelog.d/12986-prototype-getter-site-memo.md +++ b/changelog.d/12986-prototype-getter-site-memo.md @@ -9,3 +9,5 @@ Unproved own-slot loads use a shared tail entrance. Ordinary inline loads and me Indexed reads tail-continue hash collisions separately from the first bucket, and existing accessor entries decline directly to the collecting entrance. Memo mismatches continue statelessly before saving registers for a matched proof. At an indexed collision, a matching memo can prove the complete read before further key traversal; an unanswered proof resumes the same index. Collision continuations reuse its existing table and key-array pointers. Array element addresses use the same allocation extent and remaining capacity directly in bytes, preserving queue offsets and named-property reserves without new metadata. + +Indexed hits retain no collision cursor across their atom comparison; only unanswered collisions reconstruct it. Array addressing cancels the header offset in word units before scaling, avoiding the expanded byte-address arithmetic in shared callers. diff --git a/crates/perry-runtime/src/array/storage.rs b/crates/perry-runtime/src/array/storage.rs index 1f561cc233a..5d2eedc67bc 100644 --- a/crates/perry-runtime/src/array/storage.rs +++ b/crates/perry-runtime/src/array/storage.rs @@ -39,15 +39,13 @@ pub(crate) unsafe fn array_front_offset(arr: *const ArrayHeader) -> usize { /// interior pointers, the result must not be retained across a safepoint. #[inline] pub unsafe fn array_elements_ptr(arr: *const ArrayHeader) -> *mut u64 { - // Both headers occupy whole element words. Rounding the allocation - // down to a word preserves physical_capacity's division, including any - // tail padding, without converting bytes to slots and back again. + // Both headers occupy whole element words. Cancel the ArrayHeader + // offset before scaling, retaining the allocation's word-floor padding + // and the live front offset in one word-indexed address. const { assert!(crate::gc::GC_HEADER_SIZE % 8 == 0 && std::mem::size_of::() % 8 == 0) }; let header = (arr as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; - let end = ((*header).size as usize & !7) - crate::gc::GC_HEADER_SIZE; - (arr as *const u8) - .add(end) - .sub((*arr).capacity as usize * 8) as *mut u64 + let end = (*header).size as usize / 8 - crate::gc::GC_HEADER_SIZE / 8; + (arr as *mut u64).add(end - (*arr).capacity as usize) } /// Lower an array's `length` to `new_length`, writing `TAG_HOLE` into every diff --git a/crates/perry-runtime/src/object/layout_probe_indexed.rs b/crates/perry-runtime/src/object/layout_probe_indexed.rs index 39a1a24b1c5..19bf90f0cf4 100644 --- a/crates/perry-runtime/src/object/layout_probe_indexed.rs +++ b/crates/perry-runtime/src/object/layout_probe_indexed.rs @@ -46,10 +46,10 @@ pub(super) unsafe fn get( - dir, obj, key_bits, hash, memo, table, keys, + dir, obj, key_bits, hash, memo, ); } - return continued::(dir, obj, key_bits, hash, memo, table, keys); + return continued::(dir, obj, key_bits, hash, memo); } // This existing shape fact proves a public accessor. The leaf must // decline it; the collecting entrance reloads the live getter pair. @@ -67,12 +67,13 @@ pub(super) unsafe fn continued f64 { - // Continue with the same existing table and key array. No collection - // or user code occurs between the first bucket and here. - let list = crate::array::array_elements_ptr(keys as usize as *const _) as *const u64; + // Reconstruct only on an unanswered collision. Keeping this context + // out of the front avoids saving registers on every indexed own hit. + // No collection or user code occurs between the first bucket and here. + let r = record_of(dir, (*obj).parent_class_id); + let table = ((*r).probe & !PROBE_FLAGS) as *const KeyIndex; + let list = crate::array::array_elements_ptr((*r).keys as usize as *const _) as *const u64; let mask = (*table).mask as usize; let slots = KeyIndex::buckets(table); let mut b = (site_key_hash::(key_bits, hash) as usize).wrapping_add(1) & mask; diff --git a/crates/perry-runtime/src/object/layout_probe_site_memo.rs b/crates/perry-runtime/src/object/layout_probe_site_memo.rs index 449022b278a..ce8367ef4aa 100644 --- a/crates/perry-runtime/src/object/layout_probe_site_memo.rs +++ b/crates/perry-runtime/src/object/layout_probe_site_memo.rs @@ -305,14 +305,12 @@ pub(super) unsafe fn collision( key_bits: u64, hash: u64, memo: *const ReadSiteMemo, - table: *const KeyIndex, - keys: u64, ) -> f64 { let pair = (*memo).pairs[0].load(Relaxed); if (pair >> 32) as u32 == (*obj).parent_class_id && pair as u32 != DECLINE { return collision_matched::(dir, obj, key_bits, hash, memo); } - indexed::continued::(dir, obj, key_bits, hash, memo, table, keys) + indexed::continued::(dir, obj, key_bits, hash, memo) } #[inline(never)] @@ -327,11 +325,8 @@ unsafe fn collision_matched( return v; } // A matched but unanswered proof may have a live hole or an unproved - // link. Only that case reconstructs the table after the proof attempt. - let r = record_of(dir, (*obj).parent_class_id); - let table = ((*r).probe & !PROBE_FLAGS) as *const KeyIndex; - let keys = (*r).keys; - indexed::continued::(dir, obj, key_bits, hash, memo, table, keys) + // link. The continuation reconstructs its cursor after the proof attempt. + indexed::continued::(dir, obj, key_bits, hash, memo) } /// The own list proved the key absent. From c29fc58a25ac4efb45f2aeab3bfb223ff3b64693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 23 Oct 2026 16:22:20 +0000 Subject: [PATCH 10/12] docs(runtime): describe final getter memo startup behavior --- changelog.d/12986-prototype-getter-site-memo.md | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/changelog.d/12986-prototype-getter-site-memo.md b/changelog.d/12986-prototype-getter-site-memo.md index 14d2cf8803a..ce84d7149d3 100644 --- a/changelog.d/12986-prototype-getter-site-memo.md +++ b/changelog.d/12986-prototype-getter-site-memo.md @@ -1,13 +1,7 @@ -Repeated property reads can reuse a per-site shape/slot answer for own and inherited getters or data properties, including fixed-name Rust runtime call sites and short computed string keys. Each use checks the live receiver and prototype shapes and loads the current holder slot; getter redefinition, shadowing, prototype swaps and Proxy chains retain ordinary property semantics without epochs or cached getter values. +Repeated property reads can reuse a per-site shape/slot answer for own and inherited getters or data properties, including fixed-name Rust runtime sites and short computed string keys. Every use checks the live receiver and prototype shapes and loads the current holder slot. Getter redefinition, shadowing, prototype swaps, holes and Proxy chains retain ordinary property semantics. -Site storage uses shared atomic integer words rather than TLS. Process-global, never-reused ShapeIds let each word validate its own slot proof while readers follow agent-local live links; competing writers can cause misses without introducing a validity flag. The four computed-key ways bind their content keys once, so concurrent publication cannot mix different keys' proofs. This removes eager per-thread clearing of unused memo arrays and TLS address construction at read sites. +The proofs contain shared atomic integer words in ordinary BSS. Process-global, never-reused ShapeIds validate each slot while readers follow their live links; racing publication can cause a miss without a validity flag, epoch or cached heap pointer. Four computed-key ways bind their content keys once. Unused sites require no per-thread clearing or eager initialization. -Named leaves retain stateless data handling with four arguments: the memo replaces the hash argument, and hash-using paths read the atom's existing creation hash. A short own-data hit loads neither hash nor memo. Computed reads share receiver and own-property handling with the existing slow path; proven Symbol keys allocate no short-string memo. +Named leaves pass four arguments and load the atom's existing hash only when needed. A short own-data hit reads neither hash nor memo. Computed reads share receiver handling with the complete slow path; proven Symbol keys have no short-string memo. Indexed collisions and unproved slot loads use shared tail continuations. A collision can use a validated memo before further traversal; unanswered collisions reconstruct their cursor without extending the ordinary hit's register lifetime. -Unproved own-slot loads use a shared tail entrance. Ordinary inline loads and memo probe refusals no longer save registers for that uncommon load call. - -Indexed reads tail-continue hash collisions separately from the first bucket, and existing accessor entries decline directly to the collecting entrance. Memo mismatches continue statelessly before saving registers for a matched proof. - -At an indexed collision, a matching memo can prove the complete read before further key traversal; an unanswered proof resumes the same index. Collision continuations reuse its existing table and key-array pointers. Array element addresses use the same allocation extent and remaining capacity directly in bytes, preserving queue offsets and named-property reserves without new metadata. - -Indexed hits retain no collision cursor across their atom comparison; only unanswered collisions reconstruct it. Array addressing cancels the header offset in word units before scaling, avoiding the expanded byte-address arithmetic in shared callers. +Array element addressing cancels the header offset in word units before scaling. It preserves allocation padding, front offsets and named-property reserves while reducing repeated address arithmetic in property reads and shared callers. From 90869f1a49a0311d1c17ac49bcee0352f9529647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 23 Oct 2026 18:46:59 +0000 Subject: [PATCH 11/12] fix(runtime): audit getter memo atom and lifetime contracts --- changelog.d/12986-prototype-getter-site-memo.md | 2 ++ crates/perry-runtime/src/object/layout_probe.rs | 7 ++++++- crates/perry-runtime/src/object/layout_probe_indexed.rs | 4 +--- scripts/thread_exit_address_globals.json | 8 ++++++++ 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/changelog.d/12986-prototype-getter-site-memo.md b/changelog.d/12986-prototype-getter-site-memo.md index ce84d7149d3..dbacc2e9204 100644 --- a/changelog.d/12986-prototype-getter-site-memo.md +++ b/changelog.d/12986-prototype-getter-site-memo.md @@ -5,3 +5,5 @@ The proofs contain shared atomic integer words in ordinary BSS. Process-global, Named leaves pass four arguments and load the atom's existing hash only when needed. A short own-data hit reads neither hash nor memo. Computed reads share receiver handling with the complete slow path; proven Symbol keys have no short-string memo. Indexed collisions and unproved slot loads use shared tail continuations. A collision can use a validated memo before further traversal; unanswered collisions reconstruct their cursor without extending the ordinary hit's register lifetime. Array element addressing cancels the header offset in word units before scaling. It preserves allocation padding, front offsets and named-property reserves while reducing repeated address arithmetic in property reads and shared callers. + +Named memo hash reads use the shared string accessor with an explicit debug assertion of the canonical-atom contract. The thread-exit inventory records the memo as address-free shape/slot storage. diff --git a/crates/perry-runtime/src/object/layout_probe.rs b/crates/perry-runtime/src/object/layout_probe.rs index bc63ac59a3d..1761387ae9a 100644 --- a/crates/perry-runtime/src/object/layout_probe.rs +++ b/crates/perry-runtime/src/object/layout_probe.rs @@ -934,7 +934,12 @@ unsafe fn probe_get( #[inline(always)] unsafe fn site_key_hash(key_bits: u64, hash: u64) -> u64 { if LAZY_HASH { - crate::string::property_atom_hash((key_bits & POINTER_MASK) as *const crate::StringHeader) + // Named memo fronts accept canonical property atoms, never an SSO + // immediate. Keep that unsafe entry contract explicit before using + // the heap-string accessor; materialization would violate this leaf. + let key = crate::JSValue::from_bits(key_bits); + debug_assert!(!key.is_short_string(), "memo keys must be property atoms"); + crate::string::property_atom_hash(key.as_string_ptr()) } else { hash } diff --git a/crates/perry-runtime/src/object/layout_probe_indexed.rs b/crates/perry-runtime/src/object/layout_probe_indexed.rs index 19bf90f0cf4..ac00ce1d874 100644 --- a/crates/perry-runtime/src/object/layout_probe_indexed.rs +++ b/crates/perry-runtime/src/object/layout_probe_indexed.rs @@ -45,9 +45,7 @@ pub(super) unsafe fn get( - dir, obj, key_bits, hash, memo, - ); + return site_memo::collision::(dir, obj, key_bits, hash, memo); } return continued::(dir, obj, key_bits, hash, memo); } diff --git a/scripts/thread_exit_address_globals.json b/scripts/thread_exit_address_globals.json index ea250970a23..0b395b7fb08 100644 --- a/scripts/thread_exit_address_globals.json +++ b/scripts/thread_exit_address_globals.json @@ -3760,6 +3760,14 @@ ], "verdict": "no_heap_address", "why": "Startup code function pointer for the native-owner bag adapter. It stores only an extern C function address, never a payload, property bag or GC address." + }, + { + "file": "crates/perry-runtime/src/runtime_state_key.rs", + "names": [ + "MEMO" + ], + "verdict": "no_heap_address", + "why": "The named-state-key macro stores a ReadSiteMemo: exactly five AtomicU64 words containing process-global, never-reused ShapeIds paired with scalar slot indices or NEXT/ABSENT/DECLINE markers. It stores no receiver, holder, getter, key pointer or arena address. Every read follows and validates the current live shape links before loading a slot; retiring an agent cannot leave a dangling address in these words." } ] } From cfec08c1a8272ce13d468e652f844cbc067942da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 23 Oct 2026 19:15:09 +0000 Subject: [PATCH 12/12] chore(codegen): refresh runtime ABI export manifest --- crates/perry-codegen/src/wasm32/runtime_abi.tsv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/perry-codegen/src/wasm32/runtime_abi.tsv b/crates/perry-codegen/src/wasm32/runtime_abi.tsv index 0b06a9ed07e..983260b6477 100644 --- a/crates/perry-codegen/src/wasm32/runtime_abi.tsv +++ b/crates/perry-codegen/src/wasm32/runtime_abi.tsv @@ -4681,8 +4681,11 @@ js_zlib_gunzip void f64,f64,f64 js_zlib_gunzip_init f64 f64,f64 js_zlib_gunzip_sync f64 i64 js_zlib_gzip void f64,f64,f64 +js_zlib_gzip_base_options f64 f64 +js_zlib_gzip_codec_options f64 f64 js_zlib_gzip_init f64 f64,f64 js_zlib_gzip_sync f64 i64,f64 +js_zlib_gzip_transform_options f64 f64,i32s js_zlib_inflate void f64,f64,f64 js_zlib_inflate_init f64 f64,f64 js_zlib_inflate_raw void f64,f64,f64