diff --git a/changelog.d/snap-startup-census.md b/changelog.d/snap-startup-census.md new file mode 100644 index 00000000000..b49acc99d7b --- /dev/null +++ b/changelog.d/snap-startup-census.md @@ -0,0 +1 @@ +Instrumented builds gain the startup-snapshot census (`PERRY_SNAP_CENSUS=` on a binary compiled against the instr runtime): it reports which module inits a startup image could hold, with env cells, one environment guard, the entry body, key-granular globalThis locations and a check run. It is off by default and absent from production builds, whose code and startup cost are unchanged. diff --git a/crates/perry-codegen/src/codegen/entry.rs b/crates/perry-codegen/src/codegen/entry.rs index c72c83cc2ec..9eef6cfe77c 100644 --- a/crates/perry-codegen/src/codegen/entry.rs +++ b/crates/perry-codegen/src/codegen/entry.rs @@ -192,6 +192,16 @@ pub(super) fn compile_module_entry( llmod.declare_function(&format!("{}__init", prefix), VOID, &[]); llmod.declare_function(&format!("__perry_prepare_literals_{}", prefix), VOID, &[]); } + if std::env::var_os("PERRY_SNAP_CENSUS").is_some_and(|v| !v.is_empty()) { + llmod.declare_function("js_snap_entry_body", VOID, &[]); + llmod.declare_function("js_snap_tls_program", VOID, &[PTR, PTR, PTR, PTR]); + for anchor in crate::stubs::SNAP_TLS_ANCHORS { + llmod.add_external_tls_global(anchor, "i8", "localexec"); + } + if entry_in_init { + llmod.declare_function("js_snap_entry_module", VOID, &[]); + } + } // Issue #753: emit a no-op `__init` stub so the // dispatch site in some other module that does `await // import("./entry.ts")` resolves at link time. The entry @@ -468,6 +478,16 @@ pub(super) fn compile_module_entry( let _ = main.create_block("entry"); { let blk = main.block_mut(0).unwrap(); + // SNAP census: the program's TLS slots lie between the anchors the + // driver links before and after the program objects; the runner + // protects those pages only (never runtime thread-locals). + if std::env::var_os("PERRY_SNAP_CENSUS").is_some_and(|v| !v.is_empty()) { + let a = crate::stubs::SNAP_TLS_ANCHORS.map(|n| format!("@{n}")); + blk.call_void( + "js_snap_tls_program", + &[(PTR, &a[0]), (PTR, &a[1]), (PTR, &a[2]), (PTR, &a[3])], + ); + } if let Some((const_name, byte_len)) = process_entry_path.as_ref() { let path_ptr = format!("@{}", const_name); let len_str = byte_len.to_string(); @@ -713,6 +733,11 @@ pub(super) fn compile_module_entry( } blk.call_void(&format!("{}__init", prefix), &[]); } + // SNAP census (`PERRY_SNAP_CENSUS` set while compiling, with + // `PERRY_GC_INSTRUMENTS=1`): the entry's own body starts here. + if std::env::var_os("PERRY_SNAP_CENSUS").is_some_and(|v| !v.is_empty()) { + blk.call_void("js_snap_entry_body", &[]); + } } // Mark the boundary between init prelude and user code so // hoisted post-init setup (cached `@perry_class_keys_*` loads @@ -1131,6 +1156,10 @@ pub(super) fn compile_module_entry( buffer_alias_scopes, }; if entry_in_init { + // SNAP census: the entry module's evaluation is the entry body. + if std::env::var_os("PERRY_SNAP_CENSUS").is_some_and(|v| !v.is_empty()) { + ctx.block().call_void("js_snap_entry_module", &[]); + } ctx.block() .call_void(&format!("{}__init", module_prefix), &[]); } else { diff --git a/crates/perry-codegen/src/expr/array_methods.rs b/crates/perry-codegen/src/expr/array_methods.rs index 398b3e60a63..f330aacd0ce 100644 --- a/crates/perry-codegen/src/expr/array_methods.rs +++ b/crates/perry-codegen/src/expr/array_methods.rs @@ -436,7 +436,10 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // js_getenv_value returns `undefined` (nullish) for an unset // var, not a STRING_TAG'd null pointer — so `?? default` // applies and typeof/JSON.stringify agree (#1312). - Ok(blk.call(DOUBLE, "js_getenv_value", &[(I64, &key_handle)])) + super::snap_env_site::emit(ctx, expr); + Ok(ctx + .block() + .call(DOUBLE, "js_getenv_value", &[(I64, &key_handle)])) } Expr::EnvGetDynamic(name_expr) => { let key_box = lower_expr(ctx, name_expr)?; @@ -445,8 +448,11 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // `process.env[shortName]`); `js_getenv` dereferences it as // `*StringHeader`. #214 SSO bug class. let key_handle = unbox_str_handle(blk, &key_box); + super::snap_env_site::emit(ctx, expr); // `undefined` for unset vars — see EnvGet above (#1312). - Ok(blk.call(DOUBLE, "js_getenv_value", &[(I64, &key_handle)])) + Ok(ctx + .block() + .call(DOUBLE, "js_getenv_value", &[(I64, &key_handle)])) } Expr::ProcessEnv => { // `process.env` (or `globalThis.process.env`) as a value. diff --git a/crates/perry-codegen/src/expr/dispatch.rs b/crates/perry-codegen/src/expr/dispatch.rs index 18dbef31015..ca4153d9098 100644 --- a/crates/perry-codegen/src/expr/dispatch.rs +++ b/crates/perry-codegen/src/expr/dispatch.rs @@ -50,6 +50,9 @@ fn lower_expr_inner(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // one. Handlers that care receive it as an argument, because they consult // it after lowering their operands — by which point the field is gone. let value_discarded = std::mem::take(&mut ctx.discard_this_expr); + // SNAP census (instrumented compiles only): an env read whose value this + // expression stores is a store site (`expr/snap_env_site.rs`). + super::snap_env_site::note_store_parent(ctx, expr); // #9856: under `--debug-symbols`, record this expression's source line // for the DWARF line table (`crate::debug_info`). No-op otherwise. crate::debug_info::mark_expr_location(ctx, expr); diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index 59d22cc1100..22bdc5d309d 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -105,6 +105,7 @@ pub(crate) mod hot_tls; pub(crate) mod inline_birth; pub(crate) mod literal_chunks; mod literal_descriptor; +pub(crate) mod snap_env_site; /// A worker program's literal-descriptor IR, for the dialect reader's round trip. #[cfg(test)] pub(crate) fn literal_descriptor_worker_ir() -> String { diff --git a/crates/perry-codegen/src/expr/snap_env_site.rs b/crates/perry-codegen/src/expr/snap_env_site.rs new file mode 100644 index 00000000000..e00a3998d92 --- /dev/null +++ b/crates/perry-codegen/src/expr/snap_env_site.rs @@ -0,0 +1,105 @@ +//! SNAP census: the consumer kind of a `process.env` read site +//! (`secret-tests/nightly/CW/SNAP_DESIGN_2026-10-30.md` §2.10.5), emitted in +//! instrumented compiles only (`PERRY_SNAP_CENSUS` set while compiling, the +//! switch `js_snap_entry_body` uses). An ordinary compile emits nothing. +//! +//! A read is a *store* site when its value is directly the stored value of a +//! module-global binding (`let`/`const` lowered to a module global, an +//! assignment to one, `GlobalSet`), a property or index set, or an element of +//! an array or object literal. Every other read is an *observe* site. Codegen +//! calls `js_snap_env_site(kind)` right before the read's runtime call; the +//! runtime keeps the kind for that frame's next `js_getenv`. No table is +//! emitted: the kind is the call's argument. + +use std::cell::RefCell; + +use perry_hir::Expr; + +use crate::types::I32; + +use super::FnCtx; + +/// `js_snap_env_site` kinds (the runtime reads the same numbers). +const OBSERVE: i32 = 0; +const STORE: i32 = 1; + +thread_local! { + /// Env-read expressions (by address) whose parent stores their value, + /// noted when the parent is lowered and taken when the read is. + static STORE_SITES: RefCell> = const { RefCell::new(Vec::new()) }; +} + +pub(crate) fn census_compile() -> bool { + std::env::var_os("PERRY_SNAP_CENSUS").is_some_and(|v| !v.is_empty()) +} + +fn is_env_read(e: &Expr) -> bool { + matches!(e, Expr::EnvGet(_) | Expr::EnvGetDynamic(_)) +} + +fn note<'e>(children: impl Iterator + Clone) { + if children.clone().any(is_env_read) && census_compile() { + STORE_SITES.with(|s| { + let mut s = s.borrow_mut(); + for c in children.filter(|c| is_env_read(c)) { + s.push(c as *const Expr as usize); + } + }); + } +} + +/// `expr` is about to be lowered: when it stores the value of a direct +/// env-read child, that child is a store site. +pub(crate) fn note_store_parent(ctx: &FnCtx<'_>, expr: &Expr) { + match expr { + Expr::GlobalSet(_, v) => note(std::iter::once(&**v)), + Expr::LocalSet(id, v) if ctx.module_globals.contains_key(id) => note(std::iter::once(&**v)), + Expr::PropertySet { value, .. } | Expr::IndexSet { value, .. } => { + note(std::iter::once(&**value)) + } + Expr::Object(fields) => note(fields.iter().map(|(_, v)| v)), + // A plain object literal lowers to `new __AnonShape_*(values...)`: + // the literal shape has no constructor, its values are its fields. + Expr::New { + class_name, args, .. + } if ctx + .classes + .get(class_name) + .is_some_and(|c| c.is_literal_shape()) => + { + note(args.iter()) + } + Expr::Array(elems, _) => note(elems.iter()), + _ => {} + } +} + +/// A `let`/`const` binding `id` with initializer `init` is being lowered. +pub(crate) fn note_let(ctx: &FnCtx<'_>, id: u32, init: Option<&Expr>) { + if let Some(init) = init { + if ctx.module_globals.contains_key(&id) { + note(std::iter::once(init)); + } + } +} + +/// Emit the site hook for the env read `expr`, right before its runtime call. +pub(crate) fn emit(ctx: &mut FnCtx<'_>, expr: &Expr) { + if !census_compile() { + return; + } + let addr = expr as *const Expr as usize; + let store = STORE_SITES.with(|s| { + let mut s = s.borrow_mut(); + match s.iter().position(|&a| a == addr) { + Some(i) => { + s.swap_remove(i); + true + } + None => false, + } + }); + let kind = if store { STORE } else { OBSERVE }; + ctx.block() + .call_void("js_snap_env_site", &[(I32, &kind.to_string())]); +} diff --git a/crates/perry-codegen/src/lower_call/native/native_runtime_branch.rs b/crates/perry-codegen/src/lower_call/native/native_runtime_branch.rs index f9741638486..9e7ee402cbc 100644 --- a/crates/perry-codegen/src/lower_call/native/native_runtime_branch.rs +++ b/crates/perry-codegen/src/lower_call/native/native_runtime_branch.rs @@ -374,7 +374,16 @@ return crate::lower_array_record_length::lower(ctx, &args[0]); let init_fn = format!("{prefix}__init"); ctx.pending_declares .push((init_fn.clone(), crate::types::VOID, vec![])); + // SNAP census: the evaluation this guard starts was reached + // by `require`, not by `import()` (instrumented compiles only). + let census = crate::expr::snap_env_site::census_compile(); + if census { + ctx.block().call_void("js_snap_require_edge", &[(crate::types::I32, "1")]); + } ctx.block().call_void(&init_fn, &[]); + if census { + ctx.block().call_void("js_snap_require_edge", &[(crate::types::I32, "0")]); + } return lower_expr(ctx, &args[0]); } "isBuiltin" => { diff --git a/crates/perry-codegen/src/runtime_decls/strings.rs b/crates/perry-codegen/src/runtime_decls/strings.rs index 8e7e879c769..7da943102f0 100644 --- a/crates/perry-codegen/src/runtime_decls/strings.rs +++ b/crates/perry-codegen/src/runtime_decls/strings.rs @@ -944,6 +944,11 @@ pub fn declare_phase_b_strings(module: &mut LlModule) { ); module.declare_function("js_getenv", I64, &[I64]); module.declare_function("js_getenv_value", DOUBLE, &[I64]); + // SNAP census site hook (`expr/snap_env_site.rs`): instrumented compiles only. + if crate::expr::snap_env_site::census_compile() { + module.declare_function("js_snap_env_site", VOID, &[I32]); + module.declare_function("js_snap_require_edge", VOID, &[I32]); + } // #1344: process.env.X = v / delete process.env.X. module.declare_function("js_setenv", VOID, &[I64, DOUBLE]); module.declare_function("js_removeenv", VOID, &[I64]); diff --git a/crates/perry-codegen/src/stmt/mod.rs b/crates/perry-codegen/src/stmt/mod.rs index 144aa8dbab9..21c4f9ab28e 100644 --- a/crates/perry-codegen/src/stmt/mod.rs +++ b/crates/perry-codegen/src/stmt/mod.rs @@ -632,6 +632,7 @@ fn lower_stmt_inner(ctx: &mut FnCtx<'_>, stmt: &Stmt) -> Result<()> { { ctx.typed_element_locals.insert(*id); } + crate::expr::snap_env_site::note_let(ctx, *id, init.as_ref()); lower_let(ctx, *id, name, init.as_ref(), ty, *mutable)?; // Object model step 1: a family prototype's birth. crate::layout_desc::create_families::lower_binding(ctx, *id)?; diff --git a/crates/perry-codegen/src/stubs.rs b/crates/perry-codegen/src/stubs.rs index d5a10703f6d..5eac5e20fa0 100644 --- a/crates/perry-codegen/src/stubs.rs +++ b/crates/perry-codegen/src/stubs.rs @@ -256,6 +256,38 @@ pub fn generate_feature_installer_object( compile_ll_to_object(&ll, triple.as_deref()) } +/// SNAP census TLS anchors (census compiles only, `PERRY_SNAP_CENSUS`): data +/// begin, data end, bss begin, bss end. The driver links the begin object +/// first and the end object after every other compiled object, so the +/// linker, which lays `.tdata` and `.tbss` input sections out in command-line +/// order, puts the program's TLS slots between them and the runtime archive's +/// thread-locals after them. Each anchor is page-aligned, so the program's +/// slots fill whole pages that hold nothing else. +pub const SNAP_TLS_ANCHORS: [&str; 4] = [ + "__perry_snap_tls_data_begin", + "__perry_snap_tls_data_end", + "__perry_snap_tls_bss_begin", + "__perry_snap_tls_bss_end", +]; + +/// The object holding the begin (`end == false`) or end anchors of +/// [`SNAP_TLS_ANCHORS`]: one page-aligned byte in `.tdata` (initialized) and +/// one in `.tbss` (zero). +pub fn generate_snap_tls_anchor_object(end: bool, target: Option<&str>) -> Result> { + let (data, bss) = if end { + (SNAP_TLS_ANCHORS[1], SNAP_TLS_ANCHORS[3]) + } else { + (SNAP_TLS_ANCHORS[0], SNAP_TLS_ANCHORS[2]) + }; + let ll = format!( + "; Perry SNAP census TLS anchors, generated by perry-codegen::stubs\n\n\ + @{data} = thread_local global i8 1, align 4096\n\ + @{bss} = thread_local global i8 0, align 4096\n" + ); + let triple = target.and_then(crate::resolve_target_triple); + compile_ll_to_object(&ll, triple.as_deref()) +} + /// The LLVM IR of the program's static shape seed unit (design step 4, WRT /// step 2): the program's literal seeds as data. For each seed: its layout /// request, its packed key names, and, on targets whose atoms are image diff --git a/crates/perry-runtime/src/arena/allocators.rs b/crates/perry-runtime/src/arena/allocators.rs index 80c961bc7a8..81eb44a446b 100644 --- a/crates/perry-runtime/src/arena/allocators.rs +++ b/crates/perry-runtime/src/arena/allocators.rs @@ -701,6 +701,7 @@ fn arena_alloc_gc_slow(size: usize, align: usize, obj_type: u8, total: usize) -> }; if let Some(user_ptr) = reused { + crate::snap_census::birth_user(user_ptr as usize, total); ARENA.with(|a| unsafe { (&mut *a.get()).note_allocation(total, total >= 16 * 1024) }); // Reusing a free-list slot: the GcHeader is already in place (before user_ptr) // Just update it diff --git a/crates/perry-runtime/src/arena/block.rs b/crates/perry-runtime/src/arena/block.rs index f2cdc0064d2..32292af99c0 100644 --- a/crates/perry-runtime/src/arena/block.rs +++ b/crates/perry-runtime/src/arena/block.rs @@ -578,6 +578,7 @@ impl ArenaBlock { let ptr = unsafe { self.data.add(aligned_offset) }; let next = bumped.checked_add(pad - 1)? & !(pad - 1); self.offset = next; + crate::snap_census::birth(ptr, size); Some(ptr) } @@ -1306,7 +1307,6 @@ thread_local! { /// `arena_reset_empty_blocks`). pub(crate) static ARENA_TOTAL_BYTES: crate::gc::TriggerInput = const { crate::gc::TriggerInput::new(0) }; } - crate::perry_thread_local! { /// Cached running sum of `block.offset` across the old-gen arena — /// the delta-maintained twin of `ARENA_TOTAL_BYTES` above, same diff --git a/crates/perry-runtime/src/arena/inline.rs b/crates/perry-runtime/src/arena/inline.rs index 109743ce4c0..cd138d4ed8e 100644 --- a/crates/perry-runtime/src/arena/inline.rs +++ b/crates/perry-runtime/src/arena/inline.rs @@ -250,7 +250,7 @@ pub fn arena_start_fresh_general_block() { /// extent and the existing coloring/seeding path. #[inline(always)] pub(crate) fn allocation_limit(offset: usize, block_size: usize) -> usize { - if crate::gc::gc_birth_extra_flags() != 0 { + if crate::gc::gc_birth_extra_flags() != 0 || crate::snap_census::inline_bump_closed() { 0 } else { super::alloc_sample::inline_limit(offset, block_size) diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index fcb414b0019..6d98d5a2918 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -231,3 +231,7 @@ pub(crate) use region::new_test_block_backing; pub(crate) use region::release_test_block_backing; pub(crate) use inline::refresh_inline_allocation_limit; +#[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] +pub(crate) use region::reservation::{ + for_each_mapped as snap_reservation_regions, payload_bounds as snap_reservation_bounds, +}; diff --git a/crates/perry-runtime/src/arena/region/reservation.rs b/crates/perry-runtime/src/arena/region/reservation.rs index e24935b0e66..1883df489ab 100644 --- a/crates/perry-runtime/src/arena/region/reservation.rs +++ b/crates/perry-runtime/src/arena/region/reservation.rs @@ -434,10 +434,37 @@ pub(super) unsafe fn map(kind: Kind, mapped: usize) -> *mut u8 { d.publication .store(serial() | kind_bits(kind) | MAPPED, SeqCst); } + crate::snap_census::region_mapped(base, mapped); return data.cast(); } std::ptr::null_mut() } +/// `[base, base + len)` of the reservation's payload (empty before it +/// exists). The SNAP census sizes its page map with it. +#[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] +pub(crate) fn payload_bounds() -> (usize, usize) { + initialized().map_or((0, 0), |r| (r.base, r.base + r.len)) +} + +/// Every mapped region, `[base, end)`, once each. +#[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] +pub(crate) fn for_each_mapped(mut f: impl FnMut(usize, usize)) { + let Some(r) = existing() else { + return; + }; + let mut slot = 0; + while slot < r.len / ALIGN { + let d = r.descriptor(slot); + if d.publication.load(SeqCst) & MAPPED != 0 { + let (base, end) = (d.base.load(SeqCst), d.end.load(SeqCst)); + f(base, end); + slot = (end - r.base).div_ceil(ALIGN).max(slot + 1); + } else { + slot += 1; + } + } +} + pub(super) unsafe fn unmap(data: *mut u8, mapped: usize) { let r = existing().expect("region reservation missing"); let _guard = r.allocation.lock().unwrap_or_else(|e| e.into_inner()); diff --git a/crates/perry-runtime/src/async_context.rs b/crates/perry-runtime/src/async_context.rs index 072e97c5834..2b6878c036b 100644 --- a/crates/perry-runtime/src/async_context.rs +++ b/crates/perry-runtime/src/async_context.rs @@ -17,6 +17,10 @@ pub use job_domain::JobDomainToken; pub struct AsyncContextSnapshot { entries: Vec, job_domain: Option, + /// SNAP census: the init that enqueued the job this snapshot belongs to + /// (stamped at enqueue; a job runs as that init). + #[cfg(perry_gc_instruments)] + pub(crate) snap_init: u32, } /// A native context vector in an ordinary young, movable GC holder. The @@ -135,9 +139,7 @@ impl AsyncLocalStoragePayload { Arc::as_ptr(&self.token) as i64 } } - fn install_noop(_proto: &mut crate::native_payload::PayloadPrototype) {} - static ASYNC_LOCAL_STORAGE_PAYLOAD_FAMILY: crate::native_payload::NativePayloadFamily = crate::native_payload::NativePayloadFamily { class_id: crate::native_class_ids::ASYNC_LOCAL_STORAGE_LEGACY, @@ -147,7 +149,6 @@ static ASYNC_LOCAL_STORAGE_PAYLOAD_FAMILY: crate::native_payload::NativePayloadF links_owner: false, install_prototype: install_noop, }; - /// Resolve an AsyncLocalStorage ordinary object (including a subclass) to the /// native context token used by runtime-owned diagnostics-channel helpers. pub fn token_from_storage_value(value: f64) -> Option { @@ -160,7 +161,6 @@ pub fn token_from_storage_value(value: f64) -> Option { .map(|payload| payload.token()) } } - // `perry_thread_local!`, not `thread_local!`: a landing pad reaches this // through `unwind_context_guards`, and `js_eh_pad_enter` is a Leaf. std's // `LocalKey::with` reads the value through the key's accessor *pointer*; @@ -169,7 +169,7 @@ pub fn token_from_storage_value(value: f64) -> Option { // as reaching JS. `HotKey::with` is `#[inline(always)]`, and its only indirect // call is the audited cold `resolve`. crate::perry_thread_local! { - static ACTIVE_CONTEXT: RefCell = const { RefCell::new(AsyncContextSnapshot { entries: Vec::new(), job_domain: None }) }; + static ACTIVE_CONTEXT: RefCell = const { RefCell::new(AsyncContextSnapshot { entries: Vec::new(), job_domain: None, #[cfg(perry_gc_instruments)] snap_init: 0 }) }; } unsafe fn token_ref(token: i64) -> &'static AsyncLocalStorageTokenInner { @@ -691,6 +691,7 @@ pub(crate) fn test_snapshot_with_store(store: f64) -> AsyncContextSnapshot { token: payload.token, stores: vec![store], }], + ..Default::default() } } diff --git a/crates/perry-runtime/src/async_context/job_domain.rs b/crates/perry-runtime/src/async_context/job_domain.rs index 3b9d3c1c2fe..da82f96385a 100644 --- a/crates/perry-runtime/src/async_context/job_domain.rs +++ b/crates/perry-runtime/src/async_context/job_domain.rs @@ -158,6 +158,10 @@ impl AsyncContextSnapshot { } pub(crate) fn prepare_job_enqueue(&mut self) -> bool { + #[cfg(perry_gc_instruments)] + { + self.snap_init = crate::snap_census::job_owner(); + } prepare_enqueue(&mut self.job_domain) } diff --git a/crates/perry-runtime/src/builtins/globals.rs b/crates/perry-runtime/src/builtins/globals.rs index c7365b26517..d6c8e974c30 100644 --- a/crates/perry-runtime/src/builtins/globals.rs +++ b/crates/perry-runtime/src/builtins/globals.rs @@ -508,6 +508,7 @@ pub extern "C" fn js_queue_microtask(callback: i64) { #[no_mangle] pub extern "C" fn js_queue_next_tick(callback: i64) { + crate::effect!(host_write, tick); queue_microtask_with_type(callback, "TickObject", Vec::new()); } @@ -521,6 +522,7 @@ pub extern "C" fn js_queue_next_tick(callback: i64) { /// `n_args == 0`. #[no_mangle] pub unsafe extern "C" fn js_queue_next_tick_args(callback: i64, args_ptr: *const f64, n_args: i32) { + crate::effect!(host_write, tick); let args: Vec = if args_ptr.is_null() || n_args <= 0 { Vec::new() } else { @@ -588,6 +590,7 @@ pub fn restore_queued_microtask_contexts() { let mut stack = stack.borrow_mut(); while let Some(previous) = stack.pop() { crate::async_context::restore_context(previous); + crate::snap_census::job_exit(); } }); } @@ -635,6 +638,8 @@ pub(crate) fn drain_queued_microtasks_count() -> i32 { let callback_handle = scope.root_raw_const_ptr(cb as *const crate::closure::ClosureHeader); let arg_handles = scope.root_nanbox_f64_slice(&args); + #[cfg(perry_gc_instruments)] + crate::snap_census::job_enter(context.snap_init); let previous = crate::async_context::enter_context(&context); QUEUED_MICROTASK_PREV_CONTEXTS.with(|stack| { stack.borrow_mut().push(previous); @@ -740,6 +745,7 @@ pub(crate) fn drain_queued_microtasks_count() -> i32 { QUEUED_MICROTASK_PREV_CONTEXTS.with(|stack| { if let Some(previous) = stack.borrow_mut().pop() { crate::async_context::restore_context(previous); + crate::snap_census::job_exit(); } }); ran += 1; diff --git a/crates/perry-runtime/src/calendar.rs b/crates/perry-runtime/src/calendar.rs index d2599716c7e..6951a5616df 100644 --- a/crates/perry-runtime/src/calendar.rs +++ b/crates/perry-runtime/src/calendar.rs @@ -5,6 +5,7 @@ /// UTC timestamp to get the local-time representation (i.e. local - UTC). #[cfg(unix)] pub(crate) fn timestamp_to_local_components(secs: i64) -> (i32, u32, u32, u32, u32, u32, i64) { + crate::effect!(host_read, tz); unsafe { let t: libc::time_t = secs as libc::time_t; let mut tm: libc::tm = std::mem::zeroed(); @@ -28,12 +29,14 @@ pub(crate) fn timestamp_to_local_components(secs: i64) -> (i32, u32, u32, u32, u /// so local time is UTC with a zero offset. #[cfg(target_os = "wasi")] pub(crate) fn timestamp_to_local_components(secs: i64) -> (i32, u32, u32, u32, u32, u32, i64) { + crate::effect!(host_read, tz); let (y, m, d, h, mi, s) = timestamp_to_components(secs); (y, m, d, h, mi, s, 0) } #[cfg(windows)] pub(crate) fn timestamp_to_local_components(secs: i64) -> (i32, u32, u32, u32, u32, u32, i64) { + crate::effect!(host_read, tz); unsafe { let t: libc::time_t = secs as libc::time_t; let mut tm: libc::tm = std::mem::zeroed(); diff --git a/crates/perry-runtime/src/clock.rs b/crates/perry-runtime/src/clock.rs index 2c51fbc2ef6..998c76e828d 100644 --- a/crates/perry-runtime/src/clock.rs +++ b/crates/perry-runtime/src/clock.rs @@ -1,6 +1,7 @@ //! Numeric wall and performance clocks. use std::time::{SystemTime, UNIX_EPOCH}; pub(crate) fn wall_millis() -> f64 { + crate::effect!(host_read, clock); if let Some(now) = crate::timer::js_mock_timers_date_now() { return now; } @@ -14,5 +15,6 @@ pub(crate) fn wall_millis() -> f64 { /// Returns ms since performance.timeOrigin as f64. #[no_mangle] pub extern "C" fn js_performance_now() -> f64 { + crate::effect!(host_read, clock); crate::perf_clock::performance_now_ms() } diff --git a/crates/perry-runtime/src/closure/alloc.rs b/crates/perry-runtime/src/closure/alloc.rs index 0805b2c2cbf..38642a57fbd 100644 --- a/crates/perry-runtime/src/closure/alloc.rs +++ b/crates/perry-runtime/src/closure/alloc.rs @@ -1042,6 +1042,9 @@ pub extern "C" fn js_closure_alloc_singleton( if let Some(cached) = SINGLETON_CLOSURES.with(|s| s.borrow().get(&(info as usize)).copied()) { return cached; } + // The body's one no-capture closure, minted on its first use whichever + // init asks first: the SNAP census counts it as the runtime's. + let _census = crate::snap_census::first_use(); let allocated = js_closure_alloc(info, 0); SINGLETON_CLOSURES.with(|s| { s.borrow_mut().insert(info as usize, allocated); @@ -1150,7 +1153,6 @@ pub(crate) fn test_captured_singleton_closure_cache_entries( .unwrap_or_default() }) } - /// Maximum number of (captures-tuple, ClosureHeader) entries cached /// per-`func_ptr` in `SINGLETON_CAPTURED_CLOSURES`. Sized to absorb the /// parallel-instance async-await pattern (e.g. `Promise.all` of N @@ -1172,7 +1174,6 @@ const _: () = assert!( MAX_CAPTURED_CLOSURE_SLOTS <= u8::MAX as usize, "hint_indices_plus_one stores an entry index plus one in a u8" ); - /// Per-body cache miss-streak counter for the adaptive bypass. /// Closures whose captures change every call (per-call boxes for /// `__step` / `__gen_state`, etc.) miss 100% of the time on the @@ -1184,7 +1185,6 @@ const _: () = assert!( /// threshold reset the counter. const CAPTURED_MISS_STREAK_DISABLE: u32 = 256; const CAPTURED_DISABLED_SENTINEL: u32 = u32::MAX; - crate::perry_thread_local! { static CAPTURED_MISS_STREAK: RefCell> = RefCell::new(crate::fast_hash::new_ptr_hash_map()); diff --git a/crates/perry-runtime/src/dyn_eval/mod.rs b/crates/perry-runtime/src/dyn_eval/mod.rs index d73068c2a94..1fc78eec5de 100644 --- a/crates/perry-runtime/src/dyn_eval/mod.rs +++ b/crates/perry-runtime/src/dyn_eval/mod.rs @@ -456,6 +456,7 @@ pub(crate) fn function_from_strings_in_with_codegen( strings_allowed: bool, wasm_allowed: bool, ) -> f64 { + crate::snap_census::dyn_code(0); // Interpreted code reaches `process`/`console` by name. crate::object::js_install_global_value_surfaces(); let base = roots_len(); @@ -509,6 +510,7 @@ pub(crate) fn eval_script_in_with_codegen( strings_allowed: bool, wasm_allowed: bool, ) -> f64 { + crate::snap_census::dyn_code(1); // Interpreted code reaches `process`/`console` by name. crate::object::js_install_global_value_surfaces(); interp_entry(|| { diff --git a/crates/perry-runtime/src/fs/validate.rs b/crates/perry-runtime/src/fs/validate.rs index 4d3d47b764a..54e8ae3787a 100644 --- a/crates/perry-runtime/src/fs/validate.rs +++ b/crates/perry-runtime/src/fs/validate.rs @@ -205,6 +205,7 @@ static KEEP_JS_VALIDATE_EVENT_LISTENER: unsafe extern "C" fn(i64, *const u8, u32 /// `readdirSync`, `unlinkSync`, …). Throws `ERR_INVALID_ARG_TYPE` on any /// non path-like value (including numbers). No-op when the value is valid. pub(crate) fn validate_path(arg_name: &str, value: f64) { + crate::effect!(host_read, fs); if !validate_path_like_value(arg_name, value) { throw_invalid_path_arg(arg_name, value); } @@ -216,6 +217,7 @@ pub(crate) fn validate_path(arg_name: &str, value: f64) { /// throws `EBADF` (matching `fs.readFileSync(123)`); anything else throws /// `ERR_INVALID_ARG_TYPE`. `syscall` names the operation for the EBADF error. pub(crate) fn validate_path_or_fd(arg_name: &str, value: f64, syscall: &'static str) { + crate::effect!(host_read, fs); if validate_path_like_value(arg_name, value) { return; } diff --git a/crates/perry-runtime/src/gc/census.rs b/crates/perry-runtime/src/gc/census.rs index 4ccc7739117..5fbebeef12a 100644 --- a/crates/perry-runtime/src/gc/census.rs +++ b/crates/perry-runtime/src/gc/census.rs @@ -449,6 +449,7 @@ impl Census { } } self.space_live[space].add(size); + crate::snap_census::census_live(header.cast(), obj_type, size); if t > GC_TYPE_MAX as usize { return; } diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 96c15d347a6..1ec7c92a718 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -1897,3 +1897,6 @@ pub fn initialize_runtime_fixture() { static KEEP_JS_GC_INIT_WITH_LAYOUTS: unsafe extern "C" fn( *const crate::codegen_abi::PerryLayoutImageV1, ) = js_gc_init_with_layouts; + +#[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] +pub(crate) use roots::for_each_global_root_slot; diff --git a/crates/perry-runtime/src/gc/old_free.rs b/crates/perry-runtime/src/gc/old_free.rs index 485b9340422..171083380be 100644 --- a/crates/perry-runtime/src/gc/old_free.rs +++ b/crates/perry-runtime/src/gc/old_free.rs @@ -330,11 +330,19 @@ pub(crate) fn old_free_take_exact( total_size: usize, excluded_pages: Option<&crate::fast_hash::PtrHashSet>, ) -> Option { - old_free_take_exact_class(total_size, excluded_pages, false) + let user = old_free_take_exact_class(total_size, excluded_pages, false); + if let Some(user) = user { + crate::snap_census::birth_user(user, total_size); + } + user } pub(crate) fn old_free_take_exact_collecting_holder(total_size: usize) -> Option { - old_free_take_exact_class(total_size, None, true) + let user = old_free_take_exact_class(total_size, None, true); + if let Some(user) = user { + crate::snap_census::birth_user(user, total_size); + } + user } fn old_free_take_exact_class( diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index e0cf8b8a3a6..25286cbe341 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -1761,6 +1761,12 @@ pub fn gc_suppress() { /// every `gc_suppress` caller accounts for the window's growth itself right /// after this call (see [`GC_FLAG_TRIGGER_OWED`]). pub fn gc_unsuppress() { + // The SNAP census holds collection off until its boundary (an address + // names one object for the whole run); a parse window's unsuppress must + // not end that hold. + if crate::snap_census::holds_collection() { + return; + } GC_FLAGS.with(|f| f.set(f.get() & !(GC_FLAG_SUPPRESSED | GC_FLAG_TRIGGER_OWED))); } @@ -3664,12 +3670,10 @@ pub(super) fn gc_mutator_assist_scaled_work_units() -> usize { .saturating_add(arena_units) .saturating_add(malloc_units) } - pub const JS_GC_STEP_STATUS_IDLE: u32 = 0; pub const JS_GC_STEP_STATUS_ACTIVE: u32 = 1; pub const JS_GC_STEP_STATUS_COMPLETED: u32 = 2; pub const JS_GC_STEP_STATUS_SKIPPED: u32 = 3; - #[repr(C)] #[derive(Clone, Copy, Debug, Default)] pub struct JsGcStepResult { @@ -3683,14 +3687,12 @@ pub struct JsGcStepResult { pub malloc_debt_objects: u64, pub old_reclaim_debt_bytes: u64, } - #[derive(Clone, Copy)] enum BudgetedGcRebaseline { ArenaBytes { pre_in_use: usize }, MallocCount { pre_count: usize, pre_in_use: usize }, OldReclaim, } - struct BudgetedGcCycle { state: GcCycleState, trigger_kind: GcTriggerKind, @@ -3699,7 +3701,6 @@ struct BudgetedGcCycle { /// Next automatic slice. Caller-requested budgets do not advance it. automatic_work_units: usize, } - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum BudgetedGcTrigger { OldReclaim, @@ -3716,7 +3717,6 @@ pub(super) enum BudgetedGcTrigger { YoungScavengeCap, MallocCount, } - crate::perry_thread_local! { static GC_BUDGETED_CYCLE: RefCell> = const { RefCell::new(None) }; static GC_BUDGETED_CYCLE_ACTIVE: TriggerInput = const { TriggerInput::new(false) }; @@ -5221,6 +5221,9 @@ extern "C" fn js_gc_collect_from(pc: usize, sp: usize, fp: usize) { } fn js_gc_collect_body() { + if crate::snap_census::explicit_gc() { + return; + } if manual_gc_blocked_by_unsafe_zone() { return; } @@ -5233,6 +5236,7 @@ fn js_gc_collect_body() { let start = std::time::Instant::now(); manual_gc_collect_now(); super::instruments::note_full_collect_us(start.elapsed().as_micros() as u64); + crate::snap_census::explicit_gc_done(); } /// Run an explicit (`gc()`) full collection, with **precise roots** — the same diff --git a/crates/perry-runtime/src/gc/roots.rs b/crates/perry-runtime/src/gc/roots.rs index f5a1ade6507..471a8374712 100644 --- a/crates/perry-runtime/src/gc/roots.rs +++ b/crates/perry-runtime/src/gc/roots.rs @@ -559,6 +559,7 @@ pub extern "C" fn js_gc_register_global_root(ptr: i64) { /// failure unwinds from here; across the C ABI it aborts the process. pub(super) fn register_global_root(root: *mut u64) { if !root.is_null() { + let _bookkeeping = crate::snap_census::bookkeeping(); unsafe { super::root_words::verify_jsvalue_root_word(*root, "global"); runtime_write_barrier_root_nanbox(*root); @@ -567,6 +568,19 @@ pub(super) fn register_global_root(root: *mut u64) { GLOBAL_ROOTS.with(|roots| { roots.borrow_mut().push(root); }); + crate::snap_census::global_root(root as usize); +} + +/// Every registered module-global root slot (the SNAP census tracks them). +#[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] +pub(crate) fn for_each_global_root_slot(mut f: impl FnMut(usize)) { + GLOBAL_ROOTS.with(|roots| { + for &root in roots.borrow().iter() { + if !root.is_null() { + f(root as usize); + } + } + }); } /// Suppress GC triggers. While suppressed, `gc_check_trigger` is a no-op. diff --git a/crates/perry-runtime/src/i18n.rs b/crates/perry-runtime/src/i18n.rs index 52f058f6933..0cce7d6e717 100644 --- a/crates/perry-runtime/src/i18n.rs +++ b/crates/perry-runtime/src/i18n.rs @@ -175,6 +175,7 @@ pub extern "C" fn perry_i18n_init( /// Detect the system locale from environment or platform APIs. fn detect_system_locale() -> Option { + crate::effect!(host_read, locale); // 1. Explicit POSIX env overrides first: LANGUAGE (GNU extension, // colon-separated list), then LC_ALL > LC_MESSAGES > LANG. A user who // runs `LANG=de_DE.UTF-8 ./app` has stated a preference; platform diff --git a/crates/perry-runtime/src/intl/time_zone.rs b/crates/perry-runtime/src/intl/time_zone.rs index 970187c6ba2..abe2d01c69c 100644 --- a/crates/perry-runtime/src/intl/time_zone.rs +++ b/crates/perry-runtime/src/intl/time_zone.rs @@ -14,6 +14,7 @@ use super::*; /// option given — must ALWAYS yield a valid zone: an unrecognized host zone (a /// broken `TZ`) falls back to `"UTC"`, never throws. pub(crate) fn resolved_date_time_zone(options: f64) -> String { + crate::effect!(host_read, tz); let explicit = get_option_string(options, "timeZone"); let tz = explicit .clone() diff --git a/crates/perry-runtime/src/json/mod.rs b/crates/perry-runtime/src/json/mod.rs index 535cef6c313..e9977bd28ac 100644 --- a/crates/perry-runtime/src/json/mod.rs +++ b/crates/perry-runtime/src/json/mod.rs @@ -557,6 +557,10 @@ pub(crate) fn note_parse_shape_cache_carriers() { unsafe fn allocate_parse_shape_keys_array( keys: &[*const StringHeader], ) -> crate::object::canonical_keys::CanonicalKeys { + // A shape-keys cache fill on first use (the temporary source array): the + // first parser of a key list makes it, whichever init that is, so the SNAP + // census counts it as the runtime's. + let _census = crate::snap_census::first_use(); // Shape-cache entries and live receivers own this ordinary array. The // construction helper publishes its pointer layout and, for large arrays // born in old generation, remembers young key strings before return. diff --git a/crates/perry-runtime/src/lib.rs b/crates/perry-runtime/src/lib.rs index d8d3b356b5b..b474d0d1b36 100644 --- a/crates/perry-runtime/src/lib.rs +++ b/crates/perry-runtime/src/lib.rs @@ -1458,3 +1458,5 @@ mod diagnostic_report_roots; mod dyn_eval_hook_roots; mod interp_proto_guard; + +pub mod snap_census; diff --git a/crates/perry-runtime/src/map.rs b/crates/perry-runtime/src/map.rs index c2c68a50d03..8c5230f96f4 100644 --- a/crates/perry-runtime/src/map.rs +++ b/crates/perry-runtime/src/map.rs @@ -4101,3 +4101,6 @@ mod map_tombstone_tests; pub(crate) fn test_enter_catch_foreach(mark: usize) { map_foreach_enter_at(mark, js_map_alloc(4)); } + +#[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] +pub(crate) use store::snap_native_index_range; diff --git a/crates/perry-runtime/src/map/store.rs b/crates/perry-runtime/src/map/store.rs index 495c9cca8f3..df64737fe69 100644 --- a/crates/perry-runtime/src/map/store.rs +++ b/crates/perry-runtime/src/map/store.rs @@ -81,6 +81,19 @@ impl MapStore { } } +/// The SNAP census digest's view of a store: the payload words that are JS +/// state (receiver word, entries alias, capacity, used; then the inline +/// entries). The native indexes between them are derived from the entries, +/// hold malloc pointers, and an absent index leaves its bytes uninitialized, +/// so the digest leaves `[start, end)` out (payload offsets). +#[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] +pub(crate) const fn snap_native_index_range() -> (usize, usize) { + ( + std::mem::offset_of!(MapStore, numeric), + std::mem::size_of::(), + ) +} + /// Bounded native publication: a creating caller allocates the store before /// anything unrooted is live; a growing caller keeps its receiver and values in /// roots and holds a no-move scope until the new store is linked. diff --git a/crates/perry-runtime/src/math.rs b/crates/perry-runtime/src/math.rs index 069f2512415..f1d696cb187 100644 --- a/crates/perry-runtime/src/math.rs +++ b/crates/perry-runtime/src/math.rs @@ -285,6 +285,7 @@ pub extern "C" fn js_math_hypot(a: f64, b: f64) -> f64 { /// Math.random() -> number (0 <= x < 1) #[no_mangle] pub extern "C" fn js_math_random() -> f64 { + crate::effect!(host_read, random); let mut rng = rand::rng(); rng.random::() } diff --git a/crates/perry-runtime/src/module_require.rs b/crates/perry-runtime/src/module_require.rs index bd8655d7a83..c84c1658721 100644 --- a/crates/perry-runtime/src/module_require.rs +++ b/crates/perry-runtime/src/module_require.rs @@ -1244,6 +1244,10 @@ pub extern "C" fn js_register_path_module(path_value: f64, exports: f64) { /// `init_addr` is codegen's `ptrtoint` of an `extern "C" fn()` module body. #[no_mangle] pub unsafe extern "C" fn js_run_module_init_catching(init_addr: i64) { + crate::snap_census::init_part(); + if crate::snap_census::skip_body() { + return; + } let init_fn: extern "C" fn() = std::mem::transmute::(init_addr as usize); let boundary = MODULE_PATH_REGISTRY.with(|r| r.begin_module_boundary()); let outcome = crate::exception::catch_thrown(|| { @@ -1277,6 +1281,8 @@ pub unsafe extern "C" fn js_run_module_init_guarded(init_addr: i64, guard_addr: assert!(registry.register_init(key.clone(), init_addr as usize)); registry.require_with(&key, &run_path_initializer) }); + #[cfg(perry_gc_instruments)] + crate::snap_census::init_guard(init_addr as usize, guard_addr as usize); match outcome { Ok(_) => {} Err(error) => { @@ -1323,11 +1329,14 @@ fn run_path_initializer(addr: usize) -> Result<(), crate::exception::Caught> { // SAFETY: `addr` came from codegen's `ptrtoint` of an `extern "C" fn()` // module initializer and was accepted once for this canonical path. let init_fn: extern "C" fn() = unsafe { std::mem::transmute::(addr) }; - crate::exception::catch_thrown(|| { + crate::snap_census::init_enter(addr); + let outcome = crate::exception::catch_thrown(|| { init_fn(); undefined() }) - .map(|_| ()) + .map(|_| ()); + crate::snap_census::init_exit(outcome.is_err()); + outcome } fn require_path_key(key: &str) -> Result, PathModuleRequireError> { @@ -1340,6 +1349,7 @@ fn require_path_key(key: &str) -> Result, PathModuleRequireError> { /// partial exports, while unrelated waiters receive only the final namespace. #[no_mangle] pub extern "C" fn js_require_path_module(path_value: f64) -> f64 { + let _require = crate::snap_census::require_scope(); let path = value_to_string(path_value, "id"); let key = module_lookup_key(&path); match require_path_key(&key) { @@ -1780,3 +1790,29 @@ mod builtin_allowlist_parity_tests { mod init_guard_tests; pub(crate) mod file_url_text; + +/// The SNAP census check run: evaluate the module whose initializer is +/// `addr` (an imaged init the census entered inside a skipped body of D), +/// once, through its emitted guard, or for a path-wrapper frame through the +/// registry, as a `require` of it would. +#[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] +pub(crate) fn snap_evaluate_init(addr: usize, guard: usize) { + if guard != 0 { + // Static dependencies enter the evaluation directly: their path entry + // names the outer wrapper, so an address lookup cannot replay them. + unsafe { + let done = guard as *mut u8; + if *done != 0 { + return; + } + *done = 1; + js_run_module_init_guarded(addr as i64, guard as i64); + } + return; + } + let Some(key) = path_registry::key_of_init_addr(addr) else { + return; + }; + let _require = crate::snap_census::require_scope(); + let _ = require_path_key(&key); +} diff --git a/crates/perry-runtime/src/module_require/path_registry.rs b/crates/perry-runtime/src/module_require/path_registry.rs index 821f5e1d135..f1d2ce219c0 100644 --- a/crates/perry-runtime/src/module_require/path_registry.rs +++ b/crates/perry-runtime/src/module_require/path_registry.rs @@ -1324,3 +1324,17 @@ pub(crate) fn path_registry_census() -> Vec { )] }) } + +/// The canonical key an initializer address was registered under (the SNAP +/// census check run names an init by its address). +#[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] +pub(super) fn key_of_init_addr(addr: usize) -> Option { + register_pending_path_inits(); + PATH_MODULE_INIT_ADDRS.with(|addrs| { + addrs + .borrow() + .iter() + .find(|(_, &a)| a == addr) + .map(|(k, _)| k.clone()) + }) +} diff --git a/crates/perry-runtime/src/navigator.rs b/crates/perry-runtime/src/navigator.rs index 21d9f2b83d0..38fbfb51170 100644 --- a/crates/perry-runtime/src/navigator.rs +++ b/crates/perry-runtime/src/navigator.rs @@ -81,6 +81,7 @@ fn normalize_navigator_locale(locale: &str) -> Option { } fn navigator_language() -> String { + crate::effect!(host_read, locale); std::env::var("LC_ALL") .ok() .filter(|locale| !locale.is_empty() && locale != "C" && locale != "POSIX") diff --git a/crates/perry-runtime/src/object/alloc.rs b/crates/perry-runtime/src/object/alloc.rs index 26d0d1d2d87..52166df3e6a 100644 --- a/crates/perry-runtime/src/object/alloc.rs +++ b/crates/perry-runtime/src/object/alloc.rs @@ -354,6 +354,9 @@ pub(crate) unsafe fn build_keys_source_array( keys: &[&[u8]], ) -> *mut ArrayHeader { let total = prefix_len as usize + keys.len(); + // A class keys cache fill on first use (the temporary list and its key + // atoms): the SNAP census counts it as the runtime's. + let _census = crate::snap_census::first_use(); let scope = crate::gc::RuntimeHandleScope::new(); let prefix_handle = scope.root_raw_mut_ptr(prefix); let (arr, _) = prefix_handle.across_mut::(|| { diff --git a/crates/perry-runtime/src/object/class_registry/state.rs b/crates/perry-runtime/src/object/class_registry/state.rs index f7e6a8ffcef..3fb1e5a4144 100644 --- a/crates/perry-runtime/src/object/class_registry/state.rs +++ b/crates/perry-runtime/src/object/class_registry/state.rs @@ -1127,9 +1127,32 @@ fn decl_prototype_class_id(ptr: usize) -> Option { // SAFETY: the header names an ordinary object at `ptr`; its first word // is the class id. let class_id = unsafe { (*(ptr as *const ObjectHeader)).class_id }; - (class_id != 0 - && crate::object::class_value::class_decl_prototype_link(class_id) as usize == ptr) - .then_some(class_id) + (class_id != 0 && decl_prototype_link_observed(class_id, ptr) == ptr).then_some(class_id) +} + +/// `class_id`'s declared-prototype link, as `decl_prototype_class_id` +/// compares it with `ptr`. The link of a class that is not `ptr`'s observes +/// nothing: the compare fails whatever the link holds. In instrumented +/// builds the SNAP census counts the read as the init's only on a match (it +/// reads the link again outside the bookkeeping scope); a production build +/// reads it once. +#[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] +#[inline(always)] +fn decl_prototype_link_observed(class_id: u32, ptr: usize) -> usize { + let link = { + let _census = crate::snap_census::bookkeeping(); + crate::object::class_value::class_decl_prototype_link(class_id) as usize + }; + if link == ptr { + return crate::object::class_value::class_decl_prototype_link(class_id) as usize; + } + link +} + +#[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] +#[inline(always)] +fn decl_prototype_link_observed(class_id: u32, _ptr: usize) -> usize { + crate::object::class_value::class_decl_prototype_link(class_id) as usize } /// #7757: a monomorphized specialization (`Gen$num`) must present the GENERIC's @@ -1732,6 +1755,9 @@ pub(crate) fn class_decl_prototype_value_selected(class_id: u32) -> f64 { return crate::value::js_nanbox_pointer(existing as i64); } + // Materializing the declared prototype is a first-use build of static + // declaration data: the SNAP census counts it as the runtime's. + let _census = crate::snap_census::runtime_scope(); // The [[Prototype]] it links to, resolved before the object exists (a // parent that cannot be one throws before anything is linked). let scope = crate::gc::RuntimeHandleScope::new(); diff --git a/crates/perry-runtime/src/object/class_value.rs b/crates/perry-runtime/src/object/class_value.rs index b3e4f0fe869..049f9c255dc 100644 --- a/crates/perry-runtime/src/object/class_value.rs +++ b/crates/perry-runtime/src/object/class_value.rs @@ -368,6 +368,9 @@ pub(crate) fn class_value_is_first_evaluation(class_id: u32) -> bool { #[cold] #[inline(never)] fn class_value_mint(class_id: u32) -> *mut ClosureHeader { + // A class function object is a static cell built on first use, the same + // whichever init asks first: the SNAP census counts it as the runtime's. + let _census = crate::snap_census::runtime_scope(); // Any class id may own one: a native class whose prototype carries // registered members (fs.Stats' Date getters) keeps its prototype link // here like a compiled class. Its band, not its magnitude, places it. diff --git a/crates/perry-runtime/src/object/field_set_by_name.rs b/crates/perry-runtime/src/object/field_set_by_name.rs index 8a1fda2e01a..7ba695b4ba5 100644 --- a/crates/perry-runtime/src/object/field_set_by_name.rs +++ b/crates/perry-runtime/src/object/field_set_by_name.rs @@ -48,6 +48,13 @@ pub extern "C" fn js_object_set_field_by_name( key: *const crate::StringHeader, value: f64, ) { + // Census builds: a store on the realm global names its key (§2.10.13). + #[cfg(perry_gc_instruments)] + let _gkey = + crate::snap_census::global_key((obj as u64 & crate::value::POINTER_MASK) as usize, || { + (!key.is_null()).then(|| crate::regex::string_as_bytes(key)) + }); + #[cfg(debug_assertions)] crate::object::field_get_set::field_ops::debug_assert_one_number_encoding(value.to_bits()); #[cfg(perry_contract_checks)] diff --git a/crates/perry-runtime/src/object/global_this.rs b/crates/perry-runtime/src/object/global_this.rs index 7c33ed176a9..59ed7551678 100644 --- a/crates/perry-runtime/src/object/global_this.rs +++ b/crates/perry-runtime/src/object/global_this.rs @@ -113,6 +113,8 @@ pub(crate) use fetch_globals::collect_before_global_this_alloc_for_test; pub(crate) use fetch_globals::init_fetch_backed_receiver; #[cfg(feature = "temporal")] pub(crate) use fetch_globals::temporal_subclass_super; +#[cfg(perry_gc_instruments)] +pub(crate) use fetch_globals::thread_realm_global_addr; pub(crate) use fetch_globals::{ global_this_blob_thunk, global_this_builtin_noop_thunk, global_this_file_thunk, global_this_headers_thunk, global_this_is_materialized, global_this_request_thunk, diff --git a/crates/perry-runtime/src/object/global_this/fetch_globals.rs b/crates/perry-runtime/src/object/global_this/fetch_globals.rs index 9a9ba95e5b4..442bb7740e5 100644 --- a/crates/perry-runtime/src/object/global_this/fetch_globals.rs +++ b/crates/perry-runtime/src/object/global_this/fetch_globals.rs @@ -86,6 +86,13 @@ pub(crate) fn global_this_is_materialized() -> bool { THREAD_GLOBAL_THIS.with(|c| c.get()) != 0 } +/// This thread's realm global's address, 0 before it exists: the census's +/// key-granular container (design §2.10.13). Census builds only. +#[cfg(perry_gc_instruments)] +pub(crate) fn thread_realm_global_addr() -> usize { + (THREAD_GLOBAL_THIS.with(|c| c.get()) & crate::value::POINTER_MASK) as usize +} + /// Whether `obj` is THIS thread's realm global, as opposed to the global of a /// `vm` context or an eval realm that `populate_global_this_builtins` also /// fills (each of those gets its own intrinsics). @@ -117,6 +124,9 @@ pub extern "C" fn js_get_global_this() -> f64 { // The birth joins population's no-move window (`populate_global_this_builtins`): // the static layout's record is one allocation that can cross an arena // block, and no collection may run anywhere in the bootstrap. + // The realm global is the runtime's, whichever init first asks for it: + // its own birth as well as its population (SNAP census, design §2.10.20). + let _realm = crate::snap_census::realm_build(); let bootstrap_window = crate::gc::GcSuppressScope::new(); let new_ptr = unsafe { crate::realm_decl! { birth crate::object::layout_desc::perry_alloc_layout(crate::object::runtime_layouts::GLOBAL_OBJECT::desc()); realm { "%Global%", kind = namespace, never_move = true, members = { globalThis = ref("%Global%"); global = ref("%Global%"), attrs = writable | enumerable | configurable; Array = ref(Array); Object = ref(Object); String = ref(String); Number = ref(Number); Boolean = ref(Boolean); Function = ref(Function); RegExp = ref(RegExp); Date = ref(Date); Error = ref(Error); TypeError = ref(TypeError); RangeError = ref(RangeError); SyntaxError = ref(SyntaxError); ReferenceError = ref(ReferenceError); EvalError = ref(EvalError); URIError = ref(URIError); AggregateError = ref(AggregateError); Symbol = ref(Symbol); Promise = ref(Promise); Map = ref(Map); Set = ref(Set); WeakMap = ref(WeakMap); WeakSet = ref(WeakSet); WeakRef = ref(WeakRef); Proxy = ref(Proxy); BigInt = ref(BigInt); Uint8Array = ref(Uint8Array); Int8Array = ref(Int8Array); Uint16Array = ref(Uint16Array); Int16Array = ref(Int16Array); Uint32Array = ref(Uint32Array); Int32Array = ref(Int32Array); Float16Array = ref(Float16Array); Float32Array = ref(Float32Array); Float64Array = ref(Float64Array); Uint8ClampedArray = ref(Uint8ClampedArray); BigInt64Array = ref(BigInt64Array); BigUint64Array = ref(BigUint64Array); ArrayBuffer = ref(ArrayBuffer); SharedArrayBuffer = ref(SharedArrayBuffer); DataView = ref(DataView); TextEncoder = ref(TextEncoder); TextDecoder = ref(TextDecoder); TextEncoderStream = ref(TextEncoderStream); TextDecoderStream = ref(TextDecoderStream); CompressionStream = ref(CompressionStream); DecompressionStream = ref(DecompressionStream); ReadableStream = ref(ReadableStream); WritableStream = ref(WritableStream); TransformStream = ref(TransformStream); Navigator = ref(Navigator); URL = ref(URL); URLSearchParams = ref(URLSearchParams); URLPattern = ref(URLPattern); AbortController = ref(AbortController); AbortSignal = ref(AbortSignal); EventTarget = ref(EventTarget); Crypto = ref(Crypto); CryptoKey = ref(CryptoKey); SubtleCrypto = ref(SubtleCrypto); Event = ref(Event); CustomEvent = ref(CustomEvent); MessageEvent = ref(MessageEvent); DOMException = ref(DOMException); FormData = ref(FormData); Blob = ref(Blob); File = ref(File); Headers = ref(Headers); Request = ref(Request); Response = ref(Response); MessageChannel = ref(MessageChannel); MessagePort = ref(MessagePort); BroadcastChannel = ref(BroadcastChannel); Storage = ref(Storage); localStorage = accessor(get = "localStorage.", set = "localStorage."), attrs = enumerable | configurable; sessionStorage = accessor(get = "sessionStorage.", set = "sessionStorage."), attrs = enumerable | configurable; WebSocket = ref(WebSocket); FinalizationRegistry = ref(FinalizationRegistry); DisposableStack = ref(DisposableStack); AsyncDisposableStack = ref(AsyncDisposableStack); SuppressedError = ref(SuppressedError); Buffer = ref(Buffer); eval = ref(eval); fetch = ref(fetch), attrs = writable | enumerable | configurable; structuredClone = ref(structuredClone), attrs = writable | enumerable | configurable; atob = ref(atob), attrs = writable | enumerable | configurable; btoa = ref(btoa), attrs = writable | enumerable | configurable; setTimeout = ref(setTimeout), attrs = writable | enumerable | configurable; clearTimeout = ref(clearTimeout), attrs = writable | enumerable | configurable; setInterval = ref(setInterval), attrs = writable | enumerable | configurable; clearInterval = ref(clearInterval), attrs = writable | enumerable | configurable; setImmediate = ref(setImmediate), attrs = writable | enumerable | configurable; clearImmediate = ref(clearImmediate), attrs = writable | enumerable | configurable; queueMicrotask = ref(queueMicrotask), attrs = writable | enumerable | configurable; gc = ref(gc); parseInt = ref(parseInt); parseFloat = ref(parseFloat); isNaN = ref(isNaN); isFinite = ref(isFinite); encodeURI = ref(encodeURI); decodeURI = ref(decodeURI); encodeURIComponent = ref(encodeURIComponent); decodeURIComponent = ref(decodeURIComponent); escape = ref(escape); unescape = ref(unescape); console = ref(console); process = ref(process); Math = ref(Math); JSON = ref(JSON); Reflect = ref(Reflect); Atomics = ref(Atomics); Intl = ref(Intl); WebAssembly = ref(WebAssembly); Temporal = ref(Temporal); performance = ref(performance), attrs = writable | enumerable | configurable; Performance = ref(Performance), attrs = writable | enumerable | configurable; PerformanceEntry = ref(PerformanceEntry), attrs = writable | enumerable | configurable; PerformanceMark = ref(PerformanceMark), attrs = writable | enumerable | configurable; PerformanceMeasure = ref(PerformanceMeasure), attrs = writable | enumerable | configurable; PerformanceObserver = ref(PerformanceObserver), attrs = writable | enumerable | configurable; PerformanceObserverEntryList = ref(PerformanceObserverEntryList), attrs = writable | enumerable | configurable; PerformanceResourceTiming = ref(PerformanceResourceTiming), attrs = writable | enumerable | configurable; crypto = accessor(get = "crypto."), attrs = enumerable | configurable; navigator = accessor(get = "navigator."), attrs = enumerable | configurable; NaN = NaN; Infinity = Infinity; undefined = undefined; }; } } diff --git a/crates/perry-runtime/src/object/global_this/generator.rs b/crates/perry-runtime/src/object/global_this/generator.rs index ba8b0f4de75..e644b2f7565 100644 --- a/crates/perry-runtime/src/object/global_this/generator.rs +++ b/crates/perry-runtime/src/object/global_this/generator.rs @@ -1014,9 +1014,13 @@ pub extern "C" fn js_generator_attach_closure_prototype( /// agent; called during its global bootstrap or by the lazy accessors. (#3664) pub(crate) fn ensure_generator_intrinsics() { if crate::object::ASYNC_FUNCTION_INTRINSIC_PTR.load(Ordering::Acquire) == 0 { + // A realm intrinsic is the runtime's, whichever init first asks (SNAP census). + let _realm = crate::snap_census::realm_build(); build_async_function_tower(); } if crate::object::GENERATOR_FUNCTION_INTRINSIC_PTR.load(Ordering::Acquire) == 0 { + // A realm intrinsic is the runtime's, whichever init first asks (SNAP census). + let _realm = crate::snap_census::realm_build(); build_generator_tower( false, &crate::object::GENERATOR_FUNCTION_INTRINSIC_PTR, @@ -1025,6 +1029,8 @@ pub(crate) fn ensure_generator_intrinsics() { ); } if crate::object::ASYNC_GENERATOR_FUNCTION_INTRINSIC_PTR.load(Ordering::Acquire) == 0 { + // A realm intrinsic is the runtime's, whichever init first asks (SNAP census). + let _realm = crate::snap_census::realm_build(); build_generator_tower( true, &crate::object::ASYNC_GENERATOR_FUNCTION_INTRINSIC_PTR, diff --git a/crates/perry-runtime/src/object/global_this/object_intrinsic.rs b/crates/perry-runtime/src/object/global_this/object_intrinsic.rs index 578d50ca7a3..cbac6f0f0e0 100644 --- a/crates/perry-runtime/src/object/global_this/object_intrinsic.rs +++ b/crates/perry-runtime/src/object/global_this/object_intrinsic.rs @@ -60,6 +60,9 @@ pub(crate) fn ensure_object_intrinsics() -> ObjectPair { crate::object::OBJECT_INTRINSIC_PROTO_PTR.store(0, Ordering::Release); crate::array::forget_object_prototype_intrinsic(); } + // %Object% and %Object.prototype% are the realm's, whichever init first + // reads them (SNAP census, design §2.10.20). + let _realm = crate::snap_census::realm_build(); let built = build_object_intrinsics(); if let Some((ctor, proto)) = built { crate::object::OBJECT_INTRINSIC_PTR.store(ctor as i64, Ordering::Release); @@ -99,6 +102,8 @@ pub(crate) fn ensure_object_prototype_shape() -> *mut ObjectHeader { // No collection may see the raw object before the intrinsic root is // published. Unlike the complete intrinsic this sentinel is not immortal: // replacing the root lets the next collection reclaim it. + // The stand-in is the realm's as the intrinsic is (SNAP census). + let _realm = crate::snap_census::realm_build(); let _no_move = crate::gc::GcSuppressScope::new(); let proto = js_object_alloc(0, 0); if !proto.is_null() { diff --git a/crates/perry-runtime/src/object/global_this/populate.rs b/crates/perry-runtime/src/object/global_this/populate.rs index 9be6d8754bd..f00b84e9719 100644 --- a/crates/perry-runtime/src/object/global_this/populate.rs +++ b/crates/perry-runtime/src/object/global_this/populate.rs @@ -77,6 +77,8 @@ fn populate_global_this_builtins_inner(singleton_at_entry: *mut ObjectHeader) { if singleton_at_entry.is_null() { return; } + // The realm is the runtime's, whichever init first asks for it. + let _realm = crate::snap_census::runtime_scope(); // #6982: this function installs several hundred builtins, and nearly every // step allocates (name strings, constructor closures, prototype objects). // Any of those allocations can trigger a collection, and an *evacuating* @@ -1064,7 +1066,6 @@ fn alias_number_static_to_global_function(singleton_at_entry: *mut ObjectHeader, return; } let ctor_handle = scope.root_raw_mut_ptr(ctor_ptr); - let static_key = super::global_name_key(name); let ctor_ptr = ctor_handle.get_raw_mut_ptr::(); js_object_set_field_by_name( @@ -1078,7 +1079,6 @@ fn alias_number_static_to_global_function(singleton_at_entry: *mut ObjectHeader, super::super::PropertyAttrs::new(true, false, true), ); } - crate::perry_thread_local! { /// Raw address of THIS thread's `Error` constructor closure, captured at /// install. Read by `error::error_prepare_stack_trace_override` so diff --git a/crates/perry-runtime/src/object/global_this/typed_array.rs b/crates/perry-runtime/src/object/global_this/typed_array.rs index 7a8e38ae562..2eecc3b4898 100644 --- a/crates/perry-runtime/src/object/global_this/typed_array.rs +++ b/crates/perry-runtime/src/object/global_this/typed_array.rs @@ -683,6 +683,8 @@ fn install_typed_array_iterator_symbol(proto_obj: *mut ObjectHeader) { // Reachable ahead of `populate_global_this_builtins` from // `typedarray_props.rs:812` on the allocation-point route. See // `gc::tests::lazy_intrinsic_towers` for the gate. + // %TypedArray% is the realm's, whichever init first asks (SNAP census). + let _realm = crate::snap_census::realm_build(); let _no_move = crate::gc::GcSuppressScope::new(); let ctor = crate::closure::js_closure_alloc( crate::fn_info!(#[realm(realm_decl! { BigInt64Array, kind = constructor, provider = builtin_constructor, extends = "%TypedArray%", members = { BYTES_PER_ELEMENT = 8.0; }, spec = {length = 3}, emit = metadata(realm_suppressederror_metadata); BigUint64Array, kind = constructor, provider = builtin_constructor, extends = "%TypedArray%", members = { BYTES_PER_ELEMENT = 8.0; }, spec = {length = 3}, emit = metadata(realm_suppressederror_metadata); Float16Array, kind = constructor, provider = builtin_constructor, extends = "%TypedArray%", members = { BYTES_PER_ELEMENT = 2.0; }, spec = {length = 3}, emit = metadata(realm_suppressederror_metadata); Float32Array, kind = constructor, provider = builtin_constructor, extends = "%TypedArray%", members = { BYTES_PER_ELEMENT = 4.0; }, spec = {length = 3}, emit = metadata(realm_suppressederror_metadata); Float64Array, kind = constructor, provider = builtin_constructor, extends = "%TypedArray%", members = { BYTES_PER_ELEMENT = 8.0; }, spec = {length = 3}, emit = metadata(realm_suppressederror_metadata); Int16Array, kind = constructor, provider = builtin_constructor, extends = "%TypedArray%", members = { BYTES_PER_ELEMENT = 2.0; }, spec = {length = 3}, emit = metadata(realm_suppressederror_metadata); Int32Array, kind = constructor, provider = builtin_constructor, extends = "%TypedArray%", members = { BYTES_PER_ELEMENT = 4.0; }, spec = {length = 3}, emit = metadata(realm_suppressederror_metadata); Int8Array, kind = constructor, provider = builtin_constructor, extends = "%TypedArray%", members = { BYTES_PER_ELEMENT = 1.0; }, spec = {length = 3}, emit = metadata(realm_suppressederror_metadata); Uint16Array, kind = constructor, provider = builtin_constructor, extends = "%TypedArray%", members = { BYTES_PER_ELEMENT = 2.0; }, spec = {length = 3}, emit = metadata(realm_suppressederror_metadata); Uint32Array, kind = constructor, provider = builtin_constructor, extends = "%TypedArray%", members = { BYTES_PER_ELEMENT = 4.0; }, spec = {length = 3}, emit = metadata(realm_suppressederror_metadata); Uint8Array, kind = constructor, provider = builtin_constructor, extends = "%TypedArray%", members = { fromBase64; fromHex; prototype; BYTES_PER_ELEMENT = 1.0; }, spec = {length = 3}, emit = metadata(realm_suppressederror_metadata); Uint8ClampedArray, kind = constructor, provider = builtin_constructor, extends = "%TypedArray%", members = { BYTES_PER_ELEMENT = 1.0; }, spec = {length = 3}, emit = metadata(realm_suppressederror_metadata); "%TypedArray%", kind = constructor, capture = [ref(Uint8Array.prototype)], members = { from; of; [Symbol.species] = accessor(get = "%TypedArray%[\"@@species\"]."); }, spec = {name = "TypedArray"}, emit = metadata(realm_typed_array_public), births = [ArrayBuffer]; })] export crate::object::global_this::ctor_thunks::typed_array_constructor_call_thunk, 1; with_declared(0)), diff --git a/crates/perry-runtime/src/object/iterator_prototypes.rs b/crates/perry-runtime/src/object/iterator_prototypes.rs index 39febf0cead..500c2c957e6 100644 --- a/crates/perry-runtime/src/object/iterator_prototypes.rs +++ b/crates/perry-runtime/src/object/iterator_prototypes.rs @@ -523,6 +523,8 @@ mod override_probe_premise_tests { pub(crate) fn ensure_iterator_prototypes() { if ITERATOR_PROTOTYPE_PTR.load(Ordering::Acquire) == 0 { + // A realm intrinsic is the runtime's, whichever init first asks (SNAP census). + let _realm = crate::snap_census::realm_build(); build_iterator_prototypes(); } } diff --git a/crates/perry-runtime/src/object/layout_probe.rs b/crates/perry-runtime/src/object/layout_probe.rs index 8484db904d4..f57d478a60e 100644 --- a/crates/perry-runtime/src/object/layout_probe.rs +++ b/crates/perry-runtime/src/object/layout_probe.rs @@ -770,6 +770,15 @@ pub unsafe extern "C" fn perry_probe_get( key_bits: u64, hash: u64, ) -> f64 { + // Census builds: a keyed read of a global property names its key + // (§2.10.13), like the binding read below. + #[cfg(perry_gc_instruments)] + let mut scratch = [0; crate::value::SHORT_STRING_MAX_LEN]; + #[cfg(perry_gc_instruments)] + let _gkey = crate::snap_census::global_key( + (obj_biased as usize).wrapping_add(perry_abi::RECEIVER_HANDLE_FLOOR), + || crate::string::js_string_key_bytes(crate::JSValue::from_bits(key_bits), &mut scratch), + ); probe_get::(dir, obj_biased, key_bits, hash) } @@ -786,6 +795,14 @@ pub unsafe extern "C" fn perry_probe_get_binding( key_bits: u64, hash: u64, ) -> f64 { + // Census builds: a global binding's read names its key (§2.10.13). + #[cfg(perry_gc_instruments)] + let mut scratch = [0; crate::value::SHORT_STRING_MAX_LEN]; + #[cfg(perry_gc_instruments)] + let _gkey = crate::snap_census::global_key( + (obj_biased as usize).wrapping_add(perry_abi::RECEIVER_HANDLE_FLOOR), + || crate::string::js_string_key_bytes(crate::JSValue::from_bits(key_bits), &mut scratch), + ); probe_get::(dir, obj_biased, key_bits, hash) } diff --git a/crates/perry-runtime/src/object/meta_accessors.rs b/crates/perry-runtime/src/object/meta_accessors.rs index ab9bca065f0..2bf9d35fb4c 100644 --- a/crates/perry-runtime/src/object/meta_accessors.rs +++ b/crates/perry-runtime/src/object/meta_accessors.rs @@ -161,8 +161,12 @@ pub(crate) unsafe fn object_meta_ensure_for_cell( // barrier, exactly as `object_meta_ensure` does for an `ObjectHeader`. // The reloaded cell is live; consume its current generation proof. // Young arena holders need only the active incremental-mark barrier. + #[cfg(perry_gc_instruments)] + crate::snap_census::meta_install(slot as usize); *slot = meta; crate::gc::runtime_write_barrier_resolved_slot(user_ptr, slot as usize, meta as u64); + #[cfg(perry_gc_instruments)] + crate::snap_census::meta_installed(slot as usize, meta as usize); Some(meta) } @@ -216,12 +220,16 @@ pub(crate) unsafe fn object_meta_ensure(obj: MetaObj, reason: MetaReason) -> *mu // followed by an object-slot barrier, mirroring `set_object_keys_array`. // Reloaded after the allocation: promotion, if any, is reflected in the // live header. No safepoint intervenes before the resolved-slot barrier. + #[cfg(perry_gc_instruments)] + crate::snap_census::meta_install(&(*obj).meta as *const _ as usize); (*obj).meta = meta; crate::gc::runtime_write_barrier_resolved_slot( obj as usize, &(*obj).meta as *const _ as usize, meta as u64, ); + #[cfg(perry_gc_instruments)] + crate::snap_census::meta_installed(&(*obj).meta as *const _ as usize, meta as usize); meta } diff --git a/crates/perry-runtime/src/object/native_get.rs b/crates/perry-runtime/src/object/native_get.rs index f1ed881fe7b..cf0ffba1339 100644 --- a/crates/perry-runtime/src/object/native_get.rs +++ b/crates/perry-runtime/src/object/native_get.rs @@ -138,6 +138,16 @@ unsafe fn try_data_lookup_key( if FORCE_SLOW.with(|value| value.get()) { return None; } + // Census builds: a lookup on the realm global names its key (§2.10.13). + #[cfg(perry_gc_instruments)] + let _gkey = crate::snap_census::global_key( + if receiver.is_pointer() { + receiver.as_pointer::() as usize + } else { + 0 + }, + || Some(key), + ); if !receiver.is_pointer() || key.first() == Some(&b'#') || key == b"constructor" { return None; } diff --git a/crates/perry-runtime/src/object/native_module/callable_exports.rs b/crates/perry-runtime/src/object/native_module/callable_exports.rs index 31adea13f5a..bb1e4e8afa9 100644 --- a/crates/perry-runtime/src/object/native_module/callable_exports.rs +++ b/crates/perry-runtime/src/object/native_module/callable_exports.rs @@ -111,6 +111,9 @@ pub fn bound_native_callable_export_value(module_name: &str, property_name: &str if let Some(bits) = NATIVE_CALLABLE_EXPORTS.with(|c| c.borrow().get(&key).copied()) { return f64::from_bits(bits); } + // The export's one callable, minted on its first read whichever init + // reads it first: the SNAP census counts it as the runtime's. + let _census = crate::snap_census::first_use(); let method_bytes: &'static [u8] = property_name.as_bytes().to_vec().leak(); let scope = crate::gc::RuntimeHandleScope::new(); diff --git a/crates/perry-runtime/src/object/native_module/static_exports.rs b/crates/perry-runtime/src/object/native_module/static_exports.rs index e3cae0c94b1..39a38bbdf2c 100644 --- a/crates/perry-runtime/src/object/native_module/static_exports.rs +++ b/crates/perry-runtime/src/object/native_module/static_exports.rs @@ -73,6 +73,9 @@ fn export_begin( if let Some(bits) = existing { return ExportStep::Existing(f64::from_bits(bits)); } + // The export's one callable, minted on its first read whichever init + // reads it first: the SNAP census counts it as the runtime's. + let _census = crate::snap_census::first_use(); if let Some(value) = intrinsic_export(module, property) { let value = scope.root_nanbox_f64(value); NATIVE_CALLABLE_EXPORTS.with(|v| { @@ -148,6 +151,8 @@ fn export_value( ExportStep::Done(value) => value, ExportStep::Existing(value) => export_finish(module, property, value), ExportStep::Fresh(value) => { + // Its decoration is part of the first read (the runtime's). + let _census = crate::snap_census::first_use(); let scope = crate::gc::RuntimeHandleScope::new(); let value = scope.root_nanbox_f64(value); decorate( diff --git a/crates/perry-runtime/src/object/overflow.rs b/crates/perry-runtime/src/object/overflow.rs index d8b602bd254..a85ff6ca337 100644 --- a/crates/perry-runtime/src/object/overflow.rs +++ b/crates/perry-runtime/src/object/overflow.rs @@ -245,6 +245,7 @@ unsafe fn install(obj: *mut ObjectHeader, t: *mut Table) { ); (*obj).parent_class_id |= SHADOW_BIT; super::proto_validity::note_object_own_keys_changed(obj as usize); + crate::snap_census::attached(obj as usize, &[meta as usize, t as usize]); } /// Establish order authority once, copying keys and recording slot references. diff --git a/crates/perry-runtime/src/object/proto_validity.rs b/crates/perry-runtime/src/object/proto_validity.rs index 7ab0b9e9a46..165ce0a088a 100644 --- a/crates/perry-runtime/src/object/proto_validity.rs +++ b/crates/perry-runtime/src/object/proto_validity.rs @@ -52,6 +52,11 @@ pub(crate) fn any_prototype_marked() -> bool { /// re-reading through a pointer the allocation may have moved. #[inline] pub(crate) unsafe fn mark_object_as_prototype(obj: usize) -> Option { + // SNAP census: the whole prototype mark (the record, its flag, the + // serial, the private lineage, never-move) is representation (design + // §2.10.11); the image re-applies it lazily on first use. + #[cfg(perry_gc_instruments)] + let _census = crate::snap_census::representation(); if let Some(meta) = ensure_meta_for_mark(obj, crate::object::OBJECT_META_FLAG_IS_PROTOTYPE) { ANY_PROTOTYPE_MARKED.store(true, Ordering::Relaxed); // GC_STORE_AUDIT(POINTER_FREE): scalar classification bit in the meta @@ -172,6 +177,11 @@ pub(crate) fn next_prototype_serial() -> u64 { /// As [`mark_object_as_prototype`]: allocates, and may move the owner. pub(crate) unsafe fn mark_object_as_prototype_with_serial(obj: usize, serial: u64) -> Option { debug_assert!(serial >= FIRST_DYNAMIC_PROTOTYPE_SERIAL); + // SNAP census: the whole prototype mark (the record, its flag, the + // serial, the private lineage, never-move) is representation (design + // §2.10.11); the image re-applies it lazily on first use. + #[cfg(perry_gc_instruments)] + let _census = crate::snap_census::representation(); let meta = ensure_meta_for_mark(obj, crate::object::OBJECT_META_FLAG_IS_PROTOTYPE)?; ANY_PROTOTYPE_MARKED.store(true, Ordering::Relaxed); // GC_STORE_AUDIT(POINTER_FREE): scalar classification bit. @@ -298,6 +308,11 @@ unsafe fn ensure_meta_for_mark(obj: usize, flag: u64) -> Option<*mut crate::obje if !meta.is_null() && (*meta).flags & flag != 0 { return Some(meta); } + // SNAP census: the mark (an empty record, its flag, the private shape + // lineage) classifies an object the program already holds; no program + // operation observes it (design §2.10.11). + #[cfg(perry_gc_instruments)] + let _census = crate::snap_census::representation(); let scope = crate::gc::RuntimeHandleScope::new(); let handle = scope.root_raw_mut_ptr(object); let (meta, object) = handle diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index 2270e815a9c..5f22dd46215 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -5191,7 +5191,7 @@ pub(crate) const PROTO_ID_PER_OBJECT: u64 = PROTO_ID_UNIQUE; /// hands it out. pub(crate) const PROTO_ID_RECORD_HELD: u64 = PROTO_ID_NULL - 1; /// Serial bits a MIXED identity can carry beside a 32-bit class id. -const PROTO_ID_MIXED_SERIAL_BITS: u32 = 30; +pub(crate) const PROTO_ID_MIXED_SERIAL_BITS: u32 = 30; static PROTO_ID_UNIQUE_NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); @@ -5248,7 +5248,7 @@ pub(crate) fn class_proto_id(class_id: u32) -> u64 { /// /// # Safety /// `bits` are a recorded `ObjectMeta.prototype` word. -unsafe fn prototype_serial(bits: u64) -> u64 { +pub(crate) unsafe fn prototype_serial(bits: u64) -> u64 { let value = crate::value::JSValue::from_bits(bits); if !value.is_pointer() { return 0; diff --git a/crates/perry-runtime/src/object/spread.rs b/crates/perry-runtime/src/object/spread.rs index 27083d207ab..f86ae617824 100644 --- a/crates/perry-runtime/src/object/spread.rs +++ b/crates/perry-runtime/src/object/spread.rs @@ -591,6 +591,11 @@ unsafe fn birth_slow( } else { field_rep::REP_ANY }; + // The pair's first use, the same whichever init takes it first: the + // scratch key list, its canonical list, the shape and (below) the learned + // edge. The SNAP census counts them as the runtime's; the result object + // stays the caller's. + let census = crate::snap_census::first_use(); let proof = canonical_keys::SharedLayout::of_receiver(source.cast_mut()).unwrap(); let keys = if prefix_count == 0 && record.summary() == 0 { record.keys_view_of(count) @@ -631,6 +636,7 @@ unsafe fn birth_slow( canonical_keys::canonicalize(&proof, list, words.len() as u32).view() }; let shape = intern_result(keys, rep); + drop(census); let object = allocate_fields(keys.count()); shapes::store_kind::premark_plain_ordinary(object); shapes::stamp_object_shape_id_with_carrier_note(object, LayoutWord(shape)); @@ -638,6 +644,7 @@ unsafe fn birth_slow( let source_keys = positional_source(record, keys.arr()); copy_slots(object, prefix, source, record, Some(result), source_keys); if keys.count() != 0 && source_keys.is_null() { + let _census = crate::snap_census::first_use(); learn( prefix_id, source_id, diff --git a/crates/perry-runtime/src/os.rs b/crates/perry-runtime/src/os.rs index 6ae0e95b335..4fc6fd2a165 100644 --- a/crates/perry-runtime/src/os.rs +++ b/crates/perry-runtime/src/os.rs @@ -555,6 +555,7 @@ pub extern "C" fn js_process_uptime() -> f64 { /// Get the current working directory #[no_mangle] pub extern "C" fn js_process_cwd() -> *mut StringHeader { + crate::effect!(host_read, cwd); let cwd = std::env::current_dir() .map(|p| p.to_string_lossy().into_owned()) .unwrap_or_else(|_| String::new()); @@ -579,6 +580,7 @@ pub unsafe extern "C" fn js_set_process_entry_path(ptr: *const u8, len: u32) { #[no_mangle] pub extern "C" fn js_process_argv_value() -> f64 { + crate::effect!(host_read, argv); if crate::process::worker_restrictions::in_node_worker() { if let Some(value) = crate::object::native_namespace_prop_override_get("process", "argv") { return value; @@ -598,6 +600,7 @@ pub(crate) fn allocate_process_argv(capacity: u32) -> *mut ArrayHeader { /// Returns: string[] (array of NaN-boxed string pointers) #[no_mangle] pub extern "C" fn js_process_argv() -> *mut ArrayHeader { + crate::effect!(host_read, argv); use crate::array::js_array_push_f64; use crate::value::js_nanbox_string; @@ -749,6 +752,7 @@ pub extern "C" fn js_process_versions() -> f64 { /// process.hrtime.bigint() -> bigint of nanoseconds #[no_mangle] pub extern "C" fn js_process_hrtime_bigint() -> f64 { + crate::effect!(host_read, clock); use crate::bigint_hooks::js_bigint_from_u64; use crate::value::js_nanbox_bigint; @@ -766,6 +770,7 @@ pub extern "C" fn js_process_hrtime_bigint() -> f64 { /// diff is returned (clamped to non-negative). #[no_mangle] pub extern "C" fn js_process_hrtime(prior: f64) -> f64 { + crate::effect!(host_read, clock); use crate::value::JSValue; // #2013 — Node throws TypeError ERR_INVALID_ARG_TYPE when `prior` // is supplied but isn't an Array (e.g. `process.hrtime('abc')`). @@ -1337,6 +1342,7 @@ fn options_request_buffer(opts_bits: i64) -> bool { static KEEP_OS_USER_INFO_OPTIONS: extern "C" fn(i64) -> *mut ObjectHeader = js_os_user_info_options; fn js_os_user_info_impl(buffer_encoding: bool) -> *mut ObjectHeader { + crate::effect!(host_read, ids); use crate::value::JSValue; crate::object::runtime_layouts::runtime_record!(obj, RECORD_F414_0); diff --git a/crates/perry-runtime/src/os/os_process_emitter.rs b/crates/perry-runtime/src/os/os_process_emitter.rs index ae716b70de3..8cdd952a99a 100644 --- a/crates/perry-runtime/src/os/os_process_emitter.rs +++ b/crates/perry-runtime/src/os/os_process_emitter.rs @@ -394,6 +394,7 @@ pub(crate) fn emit_process_event(event: &str, args: &[f64]) -> bool { /// (#3047), matching Node's EventEmitter argument handling. #[no_mangle] pub extern "C" fn js_process_on(event_bits: i64, listener_bits: i64) -> f64 { + crate::effect!(host_write, listener); register_process_listener(event_bits, listener_bits, false, false) } @@ -406,6 +407,7 @@ pub extern "C" fn js_process_add_listener(event_bits: i64, listener_bits: i64) - /// process.once(event, listener) — one-shot listener (Node parity). #[no_mangle] pub extern "C" fn js_process_once(event_bits: i64, listener_bits: i64) -> f64 { + crate::effect!(host_write, listener); register_process_listener(event_bits, listener_bits, true, false) } @@ -429,14 +431,12 @@ pub extern "C" fn js_process_prepend_once_listener(event_bits: i64, listener_bit pub extern "C" fn js_process_emit_before_exit(code: f64) { let _ = emit_process_event("beforeExit", &[code]); } - // Codegen-only symbol: nothing inside the runtime calls it, so anchor it // against the auto-optimize whole-program dead-strip (#4876), like the other // epilogue hooks. #[cfg(feature = "keepalive-anchors")] #[used(compiler)] static KEEP_PROCESS_EMIT_BEFORE_EXIT_PENDING: extern "C" fn() = js_process_emit_before_exit_pending; - mod process_exit_tls { type Value = std::cell::Cell; type Storage = crate::tls_hot::HotCell() as usize }>; diff --git a/crates/perry-runtime/src/os_process_stream_write.rs b/crates/perry-runtime/src/os_process_stream_write.rs index 6713eea669a..6bafce66f29 100644 --- a/crates/perry-runtime/src/os_process_stream_write.rs +++ b/crates/perry-runtime/src/os_process_stream_write.rs @@ -188,6 +188,7 @@ fn flush_completely(handle: &mut impl std::io::Write, fd: i32) -> bool { /// chunk is never interleaved with a concurrent `console.log`, and drains /// that handle's buffer first so the two funnels stay in program order. pub(super) fn write_stdout(bytes: &[u8]) { + crate::effect!(host_write, stdio); let stdout = std::io::stdout(); let mut handle = stdout.lock(); #[cfg(unix)] @@ -206,6 +207,7 @@ pub(super) fn write_stdout(bytes: &[u8]) { /// Put `bytes` on fd 2. Same contract as [`write_stdout`]. pub(super) fn write_stderr(bytes: &[u8]) { + crate::effect!(host_write, stdio); let stderr = std::io::stderr(); let mut handle = stderr.lock(); #[cfg(unix)] diff --git a/crates/perry-runtime/src/process.rs b/crates/perry-runtime/src/process.rs index 29b8b09c624..a821b50c544 100644 --- a/crates/perry-runtime/src/process.rs +++ b/crates/perry-runtime/src/process.rs @@ -406,6 +406,7 @@ pub(crate) fn module_builtin_modules_array() -> f64 { } fn process_exec_argv_value() -> f64 { + crate::effect!(host_read, argv); static EXEC_ARGV: OnceLock> = OnceLock::new(); let values = crate::once_init::get_or_init(&EXEC_ARGV, || { let raw = std::env::var("PERRY_PROCESS_EXEC_ARGV").unwrap_or_else(|_| "[]".to_string()); diff --git a/crates/perry-runtime/src/process/credentials.rs b/crates/perry-runtime/src/process/credentials.rs index fb272fee16f..4d29e421d4f 100644 --- a/crates/perry-runtime/src/process/credentials.rs +++ b/crates/perry-runtime/src/process/credentials.rs @@ -12,6 +12,7 @@ use crate::value::JSValue; /// POSIX credential accessors (#1408). On non-unix targets each returns 0. #[no_mangle] pub extern "C" fn js_process_getuid() -> f64 { + crate::effect!(host_read, ids); #[cfg(unix)] unsafe { libc::getuid() as f64 @@ -23,6 +24,7 @@ pub extern "C" fn js_process_getuid() -> f64 { } #[no_mangle] pub extern "C" fn js_process_geteuid() -> f64 { + crate::effect!(host_read, ids); #[cfg(unix)] unsafe { libc::geteuid() as f64 @@ -34,6 +36,7 @@ pub extern "C" fn js_process_geteuid() -> f64 { } #[no_mangle] pub extern "C" fn js_process_getgid() -> f64 { + crate::effect!(host_read, ids); #[cfg(unix)] unsafe { libc::getgid() as f64 @@ -45,6 +48,7 @@ pub extern "C" fn js_process_getgid() -> f64 { } #[no_mangle] pub extern "C" fn js_process_getegid() -> f64 { + crate::effect!(host_read, ids); #[cfg(unix)] unsafe { libc::getegid() as f64 diff --git a/crates/perry-runtime/src/process/env_misc.rs b/crates/perry-runtime/src/process/env_misc.rs index ee764dde241..14dc614e85a 100644 --- a/crates/perry-runtime/src/process/env_misc.rs +++ b/crates/perry-runtime/src/process/env_misc.rs @@ -88,6 +88,7 @@ pub extern "C" fn js_process_add_uncaught_exception_capture_callback(callback: f /// during async event loop drain and V8 isolate destruction. #[no_mangle] pub extern "C" fn js_process_exit(code: f64) { + crate::effect!(host_write, exit); // #3041 — match Node's `parseAndValidateExitCode`: // * `undefined` / `null` → exit with the stored `process.exitCode` // (0 when it was never set / was reset to nullish — #6666). @@ -854,6 +855,8 @@ pub extern "C" fn js_process_constrained_memory() -> f64 { /// Returns a string pointer, or null (0) if not found #[no_mangle] pub extern "C" fn js_getenv(name_ptr: *const StringHeader) -> *mut StringHeader { + // Reads the env: the census records a cell or a guard member + // (`snap_census::env_var`, design §2.10), not a host read. unsafe { let name = match env_name_from_header(name_ptr) { Some(name) => name, @@ -864,9 +867,14 @@ pub extern "C" fn js_getenv(name_ptr: *const StringHeader) -> *mut StringHeader Ok(value) => { // Create a JS string from the value let bytes = value.as_bytes(); - js_string_from_bytes(bytes.as_ptr(), bytes.len() as u32) + let s = js_string_from_bytes(bytes.as_ptr(), bytes.len() as u32); + crate::snap_census::env_var(&name, Some(&value), s as usize); + s + } + Err(_) => { + crate::snap_census::env_var(&name, None, 0); + std::ptr::null_mut() // Not found, return null } - Err(_) => std::ptr::null_mut(), // Not found, return null } } } @@ -887,6 +895,7 @@ pub extern "C" fn js_getenv(name_ptr: *const StringHeader) -> *mut StringHeader /// Node, so `??` won't clobber a legitimately empty value. #[no_mangle] pub extern "C" fn js_getenv_value(name_ptr: *const StringHeader) -> f64 { + // The env read is `js_getenv`'s (`snap_census::env_var`). let ptr = js_getenv(name_ptr); let val = if ptr.is_null() { JSValue::undefined() @@ -990,6 +999,7 @@ static KEEP_PROCESS_PENDING_EXIT_CODE: extern "C" fn() -> i32 = js_process_pendi /// `js_removeenv` below, which the delete path can call directly). #[no_mangle] pub extern "C" fn js_setenv(name_ptr: *const StringHeader, value: f64) { + crate::effect!(host_write, env); use crate::value::js_jsvalue_to_string; unsafe { let name = match env_name_from_header(name_ptr) { @@ -1131,6 +1141,7 @@ static KEEP_JS_MODULE_RESOLVE_LOOKUP_PATHS: extern "C" fn(f64, f64) -> f64 = /// Unset an environment variable. Backs `delete process.env.X` (#1344). #[no_mangle] pub extern "C" fn js_removeenv(name_ptr: *const StringHeader) { + crate::effect!(host_write, env); unsafe { let name = match env_name_from_header(name_ptr) { Some(name) => name, @@ -1167,6 +1178,19 @@ pub extern "C" fn js_removeenv(name_ptr: *const StringHeader) { /// it straight to subsequent PropertyGet dispatch. #[no_mangle] pub extern "C" fn js_process_env() -> f64 { + // SNAP census: the env object and the IPC probe are the runtime's own + // population (init 0); the program taking the object is + // `snap_census::env_object`, not a host read (design §2.10.3). + #[cfg(perry_gc_instruments)] + { + let boxed = { + let _census = crate::snap_census::runtime_scope(); + js_process_env_impl() + }; + crate::snap_census::env_object(boxed); + return boxed; + } + #[cfg(not(perry_gc_instruments))] js_process_env_impl() } @@ -1410,6 +1434,10 @@ pub(crate) fn process_env_has_field( return None; } let name = unsafe { env_name_from_header(key) }; + #[cfg(perry_gc_instruments)] + if let Some(name) = &name { + crate::snap_census::env_has(name, std::env::var_os(name).is_some()); + } Some(name.is_some_and(|name| std::env::var_os(name).is_some())) } @@ -1424,6 +1452,8 @@ fn env_name_is_settable(name: &str) -> bool { } fn js_process_env_impl() -> f64 { + // Not a host read for the census: `js_process_env` runs this as the + // runtime's population and records the object (`snap_census::env_object`). super::ipc_hooks::ensure_initialized(); let cached = CACHED_ENV.with(|c| c.get()); if cached != 0.0 { @@ -1434,6 +1464,7 @@ fn js_process_env_impl() -> f64 { // Pad alloc_limit so small env sets still have headroom; large // environments (CI runners) spill to the overflow Vec path. let alloc_limit = std::cmp::max(vars.len() as u32, crate::object::INLINE_SLOT_FLOOR as u32); + let census_mark = crate::snap_census::env_storage_begin(); let obj = crate::realm_decl! { birth crate::object::js_object_alloc(0, alloc_limit); realm { process.env, kind = exotic, members = { }; } }; for (k, v) in &vars { #[cfg(windows)] @@ -1448,6 +1479,7 @@ fn js_process_env_impl() -> f64 { let val_f64 = f64::from_bits(JSValue::string_ptr(val).bits()); crate::object::js_object_set_field_by_name(obj, key, val_f64); } + crate::snap_census::env_storage_end(census_mark, obj as usize); let boxed = f64::from_bits(JSValue::pointer(obj as *const u8).bits()); // The per-OBJECT half of `ANY_PROCESS_ENV_OBJECT`, for the same reason the // arguments registry sets it beside its insert. diff --git a/crates/perry-runtime/src/process/tick_args.rs b/crates/perry-runtime/src/process/tick_args.rs index 43f0b9c42b5..37ed30e4a6e 100644 --- a/crates/perry-runtime/src/process/tick_args.rs +++ b/crates/perry-runtime/src/process/tick_args.rs @@ -25,6 +25,7 @@ pub(crate) fn collect_emit_args(args: *const ArrayHeader) -> Vec { /// `args` must be a valid NaN-boxed args array pointer, or null. #[no_mangle] pub unsafe extern "C" fn js_process_next_tick(callback_bits: i64, args: *const ArrayHeader) { + crate::effect!(host_write, tick); // Mirror codegen's setTimeout/queueMicrotask validation: the timer // validator always reports the `"callback"` argument name, which is the // wording Node uses for `process.nextTick`. diff --git a/crates/perry-runtime/src/promise/mod.rs b/crates/perry-runtime/src/promise/mod.rs index 29f167c61f9..a3e6d38bd5b 100644 --- a/crates/perry-runtime/src/promise/mod.rs +++ b/crates/perry-runtime/src/promise/mod.rs @@ -821,6 +821,8 @@ pub(crate) fn clear_promise_context_for_gc(promise: *mut Promise) { } pub(crate) fn enter_microtask_context(snapshot: AsyncContextSnapshot) { + #[cfg(perry_gc_instruments)] + crate::snap_census::job_enter(snapshot.snap_init); let previous = crate::async_context::enter_owned_context(snapshot); MICROTASK_PREV_CONTEXTS.with(|stack| stack.borrow_mut().push(previous)); } @@ -829,6 +831,7 @@ pub(crate) fn restore_microtask_context() { MICROTASK_PREV_CONTEXTS.with(|stack| { if let Some(previous) = stack.borrow_mut().pop() { restore_context(previous); + crate::snap_census::job_exit(); } }); } @@ -838,6 +841,7 @@ pub(crate) fn restore_all_microtask_contexts() { let mut stack = stack.borrow_mut(); while let Some(previous) = stack.pop() { restore_context(previous); + crate::snap_census::job_exit(); } }); } diff --git a/crates/perry-runtime/src/regex/literal.rs b/crates/perry-runtime/src/regex/literal.rs index 5382d892cdf..384b93f5ed8 100644 --- a/crates/perry-runtime/src/regex/literal.rs +++ b/crates/perry-runtime/src/regex/literal.rs @@ -71,6 +71,9 @@ pub(crate) fn from_parts( #[cold] #[inline(never)] fn literal_miss(pattern: *const StringHeader, flags: *const StringHeader, word: *mut u64) { + // The site's immutable matcher data, built on first use whichever init + // asks first: the SNAP census counts it as the runtime's. + let _census = crate::snap_census::first_use(); let scope = RuntimeHandleScope::new(); let data = super::perex_api::finish(super::perex_construct::new_data(&scope, pattern, flags)); data.with_mut_ptr::(|data| unsafe { diff --git a/crates/perry-runtime/src/regex/perex_owner.rs b/crates/perry-runtime/src/regex/perex_owner.rs index df534784c76..0dec6d9055b 100644 --- a/crates/perry-runtime/src/regex/perex_owner.rs +++ b/crates/perry-runtime/src/regex/perex_owner.rs @@ -41,6 +41,19 @@ struct ProgramCell { /// needs more than this is searched by the evaluator alone. const DFA_MAX_WORDS: usize = 64 * 1024; +/// SNAP census (design §2.10.16 R2): the immutable program words of the cell +/// at `cell`, the content that names it. Instrumented builds only. +/// +/// # Safety +/// `cell` must be the payload address of a live `GC_TYPE_REGEX_PROGRAM`. +#[cfg(perry_gc_instruments)] +pub(crate) unsafe fn snap_program_words<'a>(cell: usize) -> &'a [u32] { + unsafe { + let c = cell as *const ProgramCell; + std::slice::from_raw_parts(c.add(1).cast(), (*c).word_count) + } +} + /// The address of the program cell's automaton-storage edge, for the /// collector's trace and rewrite of the cell at `cell`. /// diff --git a/crates/perry-runtime/src/snap_census.rs b/crates/perry-runtime/src/snap_census.rs new file mode 100644 index 00000000000..f17bf5e7808 --- /dev/null +++ b/crates/perry-runtime/src/snap_census.rs @@ -0,0 +1,683 @@ +//! SNAP phase 1: what a build-time snapshot of module initialization would +//! image and what it would defer (`secret-tests/nightly/CW/SNAP_DESIGN_2026-10-30.md`). +//! +//! Two parts live here: +//! +//! - **Effect classes.** A runtime entry that observes or changes anything +//! outside the JS heap says so at its definition with [`effect!`]: +//! `target_fact` (a constant of the compile target), `host_read` (a value of +//! the machine or the invocation) or `host_write` (changes the process or +//! the world). An entry that reads and writes only the JS heap says +//! `effect!(pure)`. The declaration is read twice: by the census runner at +//! run time (the macro expands to a call of [`host`]), and from the source +//! by the effect-class gate, which checks that every runtime entry codegen +//! can call carries one (an entry without a class counts as host write). +//! In the default archives [`host`] is an empty inline function, so the +//! declaration compiles to nothing. +//! +//! - **The census runner** (`PERRY_SNAP_CENSUS=`, instrumented runtime +//! only): runs the program's module initialization up to the boundary +//! (the entry body's start, or the first explicit `gc()`), with collection +//! suppressed so an address names one object for the whole run, and records +//! per init: its extent, how it was reached, its exclusive instruction +//! count, its births, its host touches, and (with page protection over the +//! GC reservation, the realm image and the module-global root slots) the +//! exact words the deferred inits read and wrote and the first access of +//! any other init that conflicts with them (design §2.3 rules 2-4). See +//! `snap_census/runner.rs`. +//! +//! Every hook below is `#[inline(always)]` and empty without the +//! `perry_gc_instruments` cfg (and off Linux x86_64, where the runner's page +//! protection and single-step do not exist); with it, a hook that finds the +//! runner off costs one relaxed load. + +/// The effect class of a runtime entry (design §2.2). `pure` has no value: +/// it is the absence of an effect. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[repr(u8)] +pub enum Effect { + /// A constant of the compile target (`process.platform`, `os.EOL`): + /// the snapshot answers the target's value, the init stays imageable. + TargetFact = 1, + /// A value of the machine or the invocation (env, argv, cwd, tty, ids, + /// default locale and time zone, clocks, randomness, fs reads): the + /// reading init is deferred. + HostRead = 2, + /// A change of the process or the world (stdio, env writes, listeners, + /// timers and ticks, exit code, fs/net/child process, workers): the + /// writing init is deferred. + HostWrite = 3, +} + +/// Declare the effect class of the runtime entry this statement opens. +/// +/// `effect!(pure)`, `effect!(target_fact, )`, `effect!(host_read, +/// )`, `effect!(host_write, )`; `` names the host state +/// (`env`, `argv`, `tty`, `clock`, ...) for the census report. The gate reads +/// this form from the source, so it is one statement at the top of the entry +/// body, never computed. +#[macro_export] +macro_rules! effect { + (pure) => {}; + (target_fact, $what:ident) => { + $crate::snap_census::host($crate::snap_census::Effect::TargetFact, stringify!($what)) + }; + (host_read, $what:ident) => { + $crate::snap_census::host($crate::snap_census::Effect::HostRead, stringify!($what)) + }; + (host_write, $what:ident) => { + $crate::snap_census::host($crate::snap_census::Effect::HostWrite, stringify!($what)) + }; +} + +#[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] +#[path = "snap_census/runner.rs"] +mod runner; + +/// The entry that declared `class` is running (see [`effect!`]). +#[inline(always)] +pub fn host(class: Effect, what: &'static str) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::host(class, what); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = (class, what); +} + +/// Dynamic code is made: `kind` 0 = a `new Function` body, 1 = an eval +/// script. Counted per init (design §2.6: these cannot be image cells +/// until build-time compilation); `PERRY_SNAP_DYN=defer` makes them defer +/// the init (option (a), host write by kind). +#[inline(always)] +pub(crate) fn dyn_code(kind: u8) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::dyn_code(kind); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = kind; +} + +/// The census runner is on and has not reached its boundary: collection +/// stays suppressed (see `gc::gc_unsuppress`). +#[inline(always)] +pub(crate) fn holds_collection() -> bool { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + return runner::holds_collection(); + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + false +} + +/// `object_meta_ensure` is about to store a fresh, all-zero record into the +/// owner's meta word `slot` (design §2.10.11: an empty install is +/// representation). Instrumented builds only. +#[cfg(perry_gc_instruments)] +#[inline(always)] +pub(crate) fn meta_install(slot: usize) { + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::meta_install(slot); + } + #[cfg(not(all(target_os = "linux", target_arch = "x86_64")))] + let _ = slot; +} + +/// The record `meta` now hangs off the owner's meta word `slot`: its words +/// are the owner's. Instrumented builds only. +#[cfg(perry_gc_instruments)] +#[inline(always)] +pub(crate) fn meta_installed(slot: usize, meta: usize) { + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::meta_installed(slot, meta); + } + #[cfg(not(all(target_os = "linux", target_arch = "x86_64")))] + let _ = (slot, meta); +} + +/// A classification mark (`ensure_meta_for_mark`) runs while this lives: +/// representation, not program state (design §2.10.11). Instrumented +/// builds only. +#[cfg(perry_gc_instruments)] +#[inline(always)] +pub(crate) fn representation() -> Representation { + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + if runner::on() && runner::representation(1) { + return Representation { active: true }; + } + Representation { active: false } +} + +/// The scope [`representation`] opened: its end closes the representation +/// depth with the bookkeeping depth (R1 tells the two apart). +#[cfg(perry_gc_instruments)] +pub(crate) struct Representation { + active: bool, +} + +#[cfg(perry_gc_instruments)] +impl Drop for Representation { + #[inline(always)] + fn drop(&mut self) { + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + if self.active { + runner::representation(-1); + } + #[cfg(not(all(target_os = "linux", target_arch = "x86_64")))] + let _ = self.active; + } +} + +/// A keyed operation on `obj` (a global lookup of key X, the add or store +/// of key K) names `key` for the scope's extent: on the realm global, the +/// census takes the global's shape word as the location of that key +/// (design §2.10.13); any other object is untouched. Census builds only. +#[cfg(perry_gc_instruments)] +#[inline(always)] +pub(crate) fn global_key<'a>(obj: usize, key: impl FnOnce() -> Option<&'a [u8]>) -> GlobalKey { + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + return GlobalKey { + prev: runner::global_key(obj, key), + }; + } + let _ = (obj, key); + GlobalKey { prev: None } +} + +#[cfg(perry_gc_instruments)] +pub(crate) struct GlobalKey { + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + prev: Option<(u64, [u8; runner::GKEY_NAME], u8)>, + #[cfg(not(all(target_os = "linux", target_arch = "x86_64")))] + prev: Option<()>, +} + +#[cfg(perry_gc_instruments)] +impl Drop for GlobalKey { + #[inline(always)] + fn drop(&mut self) { + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + if let Some(prev) = self.prev.take() { + runner::global_key_end(prev); + } + #[cfg(not(all(target_os = "linux", target_arch = "x86_64")))] + let _ = self.prev; + } +} + +/// `cells` (an object's metadata record, its overflow table) now belong to +/// `obj`: the cells of a transparent object are transparent too. +#[inline(always)] +pub(crate) fn attached(obj: usize, cells: &[usize]) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::attached(obj, cells); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = (obj, cells); +} + +/// A module evaluation (`run_path_initializer`) starts: `addr` is its +/// evaluation body (a guarded module's `__init_evaluate`, a path +/// module's initializer). +#[inline(always)] +pub(crate) fn init_enter(addr: usize) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + runner::init_enter(addr); + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = addr; +} + +/// The evaluation [`init_enter`] opened returned; `threw` when it threw. +#[inline(always)] +pub(crate) fn init_exit(threw: bool) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::init_exit(threw); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = threw; +} + +/// `js_run_module_init_catching` runs one part of the current evaluation +/// (its literal preparation, then its body). +#[inline(always)] +pub(crate) fn init_part() { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::init_part(); + } +} + +/// A `require` of a path module is in progress (the next evaluation it +/// starts was reached by `require`, not by an import). +#[inline(always)] +pub(crate) fn require_scope() -> RequireScope { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::require_depth(1); + return RequireScope { active: true }; + } + RequireScope { active: false } +} + +pub(crate) struct RequireScope { + active: bool, +} + +impl Drop for RequireScope { + #[inline(always)] + fn drop(&mut self) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if self.active { + runner::require_depth(-1); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = self.active; + } +} + +/// The init a job enqueued now belongs to (0: none). +#[inline(always)] +#[cfg_attr(not(perry_gc_instruments), allow(dead_code))] +pub(crate) fn job_owner() -> u32 { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + return runner::current_init(); + } + 0 +} + +/// A job owned by `owner` (from [`job_owner`] at its enqueue) starts. +#[inline(always)] +#[cfg_attr(not(perry_gc_instruments), allow(dead_code))] +pub(crate) fn job_enter(owner: u32) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::job_enter(owner); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = owner; +} + +/// The job [`job_enter`] opened is over. +#[inline(always)] +pub(crate) fn job_exit() { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::job_exit(); + } +} + +/// An explicit `gc()`: the boundary marker of a census run. True when the +/// collection must not run (`PERRY_SNAP_RSET` keeps the heap in place). +#[inline(always)] +pub(crate) fn explicit_gc() -> bool { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + { + if runner::on() || runner::start_if_requested() { + runner::boundary("gc"); + } + return runner::skip_collection(); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + false +} + +/// The census of the collection the boundary's `gc()` runs has finished. +#[inline(always)] +pub(crate) fn explicit_gc_done() { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + runner::after_boundary_collection(); +} + +/// A walkable object is live at the boundary collection's census. +#[inline(always)] +pub(crate) fn census_live(header: *const u8, obj_type: u8, size: usize) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + runner::census_live(header, obj_type, size); + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = (header, obj_type, size); +} + +/// An arena bump handed out `[ptr, ptr + size)`. +#[inline(always)] +pub(crate) fn birth(ptr: *mut u8, size: usize) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::birth(ptr as usize, size); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = (ptr, size); +} + +/// A reused slot handed out at `user` (its GcHeader sits just below): +/// [`birth`] of the allocation. The header address is computed only in +/// instrumented builds, so a production caller computes nothing new. +#[inline(always)] +pub(crate) fn birth_user(user: usize, size: usize) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::birth(user - crate::gc::GC_HEADER_SIZE, size); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = (user, size); +} + +/// While the census runs, compiled code must allocate through the runtime +/// (every birth is logged), so the inline bump bound is closed. +#[inline(always)] +pub(crate) fn inline_bump_closed() -> bool { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + return runner::on(); + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + false +} + +/// The reservation mapped `[base, base + len)` (a new arena block). +#[inline(always)] +pub(crate) fn region_mapped(base: usize, len: usize) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::region_mapped(base, len); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = (base, len); +} + +/// A module-global root slot was registered. +#[inline(always)] +pub(crate) fn global_root(slot: usize) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::global_root(slot); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = slot; +} + +/// Runtime bookkeeping that reads JS state without the program observing it +/// (root registration verifies the word it registers): its accesses are not +/// the current init's. +#[inline(always)] +pub(crate) fn bookkeeping() -> Bookkeeping { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::bookkeeping_depth(1); + return Bookkeeping { active: true }; + } + Bookkeeping { active: false } +} + +pub(crate) struct Bookkeeping { + active: bool, +} + +impl Drop for Bookkeeping { + #[inline(always)] + fn drop(&mut self) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if self.active { + runner::bookkeeping_depth(-1); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = self.active; + } +} + +/// Runtime work that an init triggers but does not own (populating the +/// realm on first use): its births, accesses and host touches are the +/// runtime's (init 0), reported apart. +#[inline(always)] +pub(crate) fn runtime_scope() -> RuntimeScope { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::job_enter(0); + return RuntimeScope { active: true }; + } + RuntimeScope { active: false } +} + +/// The fill of a cache of static data on its first use (a literal class's +/// keys, a regex literal site's matcher data): the same whichever init asks +/// first, so the census counts it as the runtime's, like [`runtime_scope`] +/// (design §2.10.20; `PERRY_SNAP_FIRSTUSE=0` leaves it the asking init's). +#[inline(always)] +pub(crate) fn first_use() -> RuntimeScope { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() && runner::first_use_on() { + runner::job_enter(0); + return RuntimeScope { active: true }; + } + RuntimeScope { active: false } +} + +/// The lazy birth and build of a realm intrinsic (the global object, +/// %Object%, a generator or iterator tower, %TypedArray%): the same +/// whichever init asks first, so the census counts it as the runtime's, like +/// [`runtime_scope`] (design §2.10.20; `PERRY_SNAP_REALM=0` leaves it the +/// asking init's). Outside census builds the scope is a unit value with no +/// drop, so the builders compile exactly as without it. +#[inline(always)] +pub(crate) fn realm_build() -> RealmScope { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() && runner::realm_on() { + runner::job_enter(0); + return RealmScope { active: true }; + } + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + return RealmScope { active: false }; + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + RealmScope {} +} + +/// The scope of [`realm_build`]. +pub(crate) struct RealmScope { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + active: bool, +} + +#[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] +impl Drop for RealmScope { + #[inline(always)] + fn drop(&mut self) { + if self.active { + runner::job_exit(); + } + } +} + +pub(crate) struct RuntimeScope { + active: bool, +} + +impl Drop for RuntimeScope { + #[inline(always)] + fn drop(&mut self) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if self.active { + runner::job_exit(); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = self.active; + } +} + +/// Codegen calls this where the entry module's own body starts (after its +/// eager imports), in instrumented compiles only (`PERRY_SNAP_CENSUS` set +/// while compiling). The census boundary of an ESM entry. +/// The program evaluates its entry as a module (it has a worker): the +/// evaluation that starts next is the entry body (design §2.3). Emitted by +/// census compiles only, right before the entry module's `__init`. +#[cfg(perry_gc_instruments)] +#[no_mangle] +pub extern "C" fn js_snap_entry_module() { + crate::effect!(pure); + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() || runner::start_if_requested() { + runner::entry_module(); + } +} + +/// The program's TLS slot ranges: census compiles call this first in `main` +/// with the addresses of the four page-aligned anchors the driver links +/// before and after the program objects (`.tdata` and `.tbss` each). Only +/// these pages are protected; runtime thread-locals stay outside them. +#[cfg(perry_gc_instruments)] +#[no_mangle] +pub extern "C" fn js_snap_tls_program( + data_begin: *const u8, + data_end: *const u8, + bss_begin: *const u8, + bss_end: *const u8, +) { + crate::effect!(pure); + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + runner::tls_program( + data_begin as usize, + data_end as usize, + bss_begin as usize, + bss_end as usize, + ); + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = (data_begin, data_end, bss_begin, bss_end); +} + +#[cfg(perry_gc_instruments)] +#[no_mangle] +pub extern "C" fn js_snap_entry_body() { + crate::effect!(pure); + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() || runner::start_if_requested() { + runner::entry_body(); + } +} + +// --------------------------------------------------------------------------- +// SNAP3: env reads as startup-filled cells and one image guard (design +// §2.10). The three env funnels call these instead of declaring +// `host_read env`; an init that reads the env through them keeps its image +// and the runner records a cell or a guard member. +// --------------------------------------------------------------------------- + +/// The env object (`process.env` as a value) was handed to the program: +/// `boxed` is its NaN-boxed bits. Taking it is not an observation; its words +/// in the image are `env_object` cells. +#[inline(always)] +#[cfg_attr(not(perry_gc_instruments), allow(dead_code))] +pub(crate) fn env_object(boxed: f64) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::env_object(boxed.to_bits()); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = boxed; +} + +/// The env object's storage is about to be populated (the birth-log mark for +/// [`env_storage_end`]). +#[inline(always)] +pub(crate) fn env_storage_begin() -> usize { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + return runner::births_len(); + } + 0 +} + +/// The env object `obj` was populated: what was born since `begin` (its key +/// and value strings, key lists, overflow) and the object's slots are the +/// env block's words. An init that reads or writes one enumerates or writes +/// the env (rule 1). +#[inline(always)] +pub(crate) fn env_storage_end(begin: usize, obj: usize) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::env_storage(begin, obj); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = (begin, obj); +} + +/// One var read by name (`js_getenv`): `value` is its text and `ptr` the +/// string `js_getenv` returned (0 when the var is unset). +#[inline(always)] +pub(crate) fn env_var(name: &str, value: Option<&str>, ptr: usize) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::env_var(name, value, ptr); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = (name, value, ptr); +} + +/// A presence test of one var (`"X" in env`, `hasOwnProperty`, `Reflect.has`). +#[inline(always)] +#[cfg_attr(not(perry_gc_instruments), allow(dead_code))] +pub(crate) fn env_has(name: &str, present: bool) { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::env_has(name, present); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = (name, present); +} + +/// Codegen calls this right before a `process.env.X` / `process.env[k]` +/// read, in instrumented compiles only (`expr/snap_env_site.rs`): `kind` 1 +/// when the read's value is stored straight into a module global, a +/// property, an element or a literal; 0 otherwise. Kept for the frame's +/// next `js_getenv`. +#[cfg(perry_gc_instruments)] +#[no_mangle] +pub extern "C" fn js_snap_env_site(kind: i32) { + crate::effect!(pure); + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::env_site(kind); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = kind; +} + +/// Codegen calls this around a literal CommonJS require edge's init-guard +/// call, in instrumented compiles only (`cjsRequireEdge`): `on` 1 before, 0 +/// after. The evaluation the guard starts was reached by `require` (a static +/// edge), not by `import()`: it is no member of a dynamic subgraph (design +/// §2.10.7). A missing mark costs precision, never soundness. +#[cfg(perry_gc_instruments)] +#[no_mangle] +pub extern "C" fn js_snap_require_edge(on: i32) { + crate::effect!(pure); + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::require_edge(on != 0); + } + #[cfg(not(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64")))] + let _ = on; +} + +/// The check run (`PERRY_SNAP_SKIP=`): true when the evaluation now +/// running is a member of D, whose body parts are skipped. +#[inline(always)] +pub(crate) fn skip_body() -> bool { + #[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + return runner::skip_body(); + } + false +} + +/// The evaluation's emitted once guard, retained on its census init record. +#[cfg(perry_gc_instruments)] +#[inline(always)] +pub(crate) fn init_guard(addr: usize, guard: usize) { + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + if runner::on() { + runner::init_guard(addr, guard); + } + #[cfg(not(all(target_os = "linux", target_arch = "x86_64")))] + let _ = (addr, guard); +} diff --git a/crates/perry-runtime/src/snap_census/runner.rs b/crates/perry-runtime/src/snap_census/runner.rs new file mode 100644 index 00000000000..571e346fca4 --- /dev/null +++ b/crates/perry-runtime/src/snap_census/runner.rs @@ -0,0 +1,4295 @@ +//! The SNAP census runner (`PERRY_SNAP_CENSUS=`): instrumented runtime +//! only. See `snap_census.rs` for the hooks and the design doc §2.3 for the +//! rules this measures. +//! +//! **Run.** On the first module evaluation the runner turns on: it suppresses +//! collection (nothing moves until the boundary, so an address names one +//! object), closes the inline bump bound (every birth goes through +//! `ArenaBlock::alloc`, which logs it), logs the objects that already exist as +//! runtime births, and opens a perf counter of user instructions. Frames are +//! module evaluations (`run_path_initializer`) and promise jobs (a job runs as +//! the init that enqueued it); every access is the top frame's. The run ends +//! at the first explicit `gc()` (the boundary marker), where the report is +//! written, the protections are lifted and the collection runs (with +//! `PERRY_GC_CENSUS` set, its census of the live heap is attributed to births +//! here). +//! +//! **Deferred inits (D).** Rule 1: an init that runs a `host_read` or +//! `host_write` entry, or throws. `PERRY_SNAP_D=` (one evaluation +//! address per line, hex) seeds D with the inits an earlier pass found, so +//! their accesses are recorded from their start: while a member of D is the +//! top frame, every tracked page is `PROT_NONE` and each access is recorded as +//! a read or write of its 8-byte word (`R_D`, `W_D`); while any other init +//! runs, only the pages that hold a `W_D` word (`PROT_NONE`) or an `R_D` word +//! (`PROT_READ`) are protected. A read of a `W_D` word (rule 2) or a write of a +//! `W_D`/`R_D` word (rule 3) makes that init a member from then on; the next +//! pass records it from its start. Passes repeat until D stops growing. +//! Rule 4 (relaxed, `PERRY_SNAP_RULE4=1`): when the first dynamically +//! imported evaluation starts, every word that exists then is watched for +//! writes; an init of the dynamic subgraph that writes one joins D. +//! +//! Faults single-step: the handler opens the page, sets the trap flag, and +//! the `SIGTRAP` after the one instruction closes it again, so the program +//! runs unchanged. Tracked: the GC reservation's mapped regions, the realm +//! image section and the pages of registered module-global root slots. +//! +//! Linux x86_64 only (the reservation, `REG_ERR`, the trap flag). + +use super::Effect; +use std::sync::atomic::{AtomicBool, AtomicPtr, AtomicU8, AtomicUsize, Ordering::Relaxed}; + +const PAGE: usize = 4096; +const MAX_INITS: usize = 1 << 16; + +/// The runner's own words, alone on their page: every tracked page that the +/// handler might protect is a page of JS state, never this one. +#[repr(C, align(4096))] +struct Anchor { + on: AtomicBool, + started: AtomicU8, + state: AtomicPtr, + /// The program's TLS slot ranges (`js_snap_tls_program`): data begin, + /// data end, bss begin, bss end. + tls: [AtomicUsize; 4], + _pad: [u8; PAGE - 48], +} + +static ANCHOR: Anchor = Anchor { + on: AtomicBool::new(false), + started: AtomicU8::new(0), + state: AtomicPtr::new(std::ptr::null_mut()), + tls: [ + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + ], + _pad: [0; PAGE - 48], +}; + +/// Census compiles call this first in `main` with the addresses of the four +/// page-aligned anchors the driver links around the program objects +/// (`__perry_snap_tls_{data,bss}_{begin,end}`): the program's module-global +/// TLS slots lie in `[data_begin, data_end)` and `[bss_begin, bss_end)`, on +/// pages of their own. Runtime and libc thread-locals are outside both. +pub(super) fn tls_program(data_lo: usize, data_hi: usize, bss_lo: usize, bss_hi: usize) { + for (i, v) in [data_lo, data_hi, bss_lo, bss_hi].into_iter().enumerate() { + ANCHOR.tls[i].store(v, Relaxed); + } +} + +#[inline(always)] +pub(super) fn on() -> bool { + ANCHOR.on.load(Relaxed) +} + +/// Collection is held until the boundary (not in the weights run). +pub(super) fn holds_collection() -> bool { + on() && !state().noprotect +} + +fn state() -> &'static mut State { + // SAFETY: set once at start, never freed; the runner is single-threaded + // (module evaluation and jobs run on the main thread; a fault on another + // thread only reads it). + unsafe { &mut *ANCHOR.state.load(Relaxed) } +} + +// --------------------------------------------------------------------------- +// Fixed-capacity tables in their own anonymous mappings (never protected, no +// allocation from the signal handler). +// --------------------------------------------------------------------------- + +unsafe fn map_zeroed(count: usize) -> *mut T { + let bytes = count * std::mem::size_of::(); + let p = libc::mmap( + std::ptr::null_mut(), + bytes, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_NORESERVE, + -1, + 0, + ); + assert!( + p != libc::MAP_FAILED, + "snap census: mmap {bytes} bytes failed" + ); + p.cast() +} + +/// A removed [`WordSet`] slot (no word is all ones). +const TOMBSTONE: u64 = u64::MAX; + +/// Location ids of the realm global's keys (§2.10.13). Real words are user +/// addresses below 2^47; these ids have the top bit set. `GKEY_ANY`: the +/// global's key set (every add writes it, an unkeyed read reads it: key +/// order and enumeration); `GKEY_UNKEYED`: an unkeyed write (it may change +/// any key: every keyed read reads it); `gkey_loc`: key K. +const GKEY_ANY: u64 = 0x8000_0000_0000_0001; +const GKEY_UNKEYED: u64 = 0x8000_0000_0000_0002; +pub(super) const GKEY_NAME: usize = 40; +const GKEY_CONFLICTS_MAX: usize = 1 << 16; + +fn gkey_loc(key: &[u8]) -> u64 { + // FNV-1a over the key's bytes; the top two bits tag a key id. + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for &b in key { + h = (h ^ b as u64).wrapping_mul(0x0100_0000_01b3); + } + let id = (h >> 2) | 0xC000_0000_0000_0000; + if id == TOMBSTONE { + id - 1 + } else { + id + } +} + +/// Open-addressing set of nonzero words. +struct WordSet { + slots: *mut u64, + mask: usize, + len: usize, +} + +impl WordSet { + unsafe fn new(log2: u32) -> Self { + let n = 1usize << log2; + WordSet { + slots: map_zeroed::(n), + mask: n - 1, + len: 0, + } + } + #[inline] + fn hash(w: u64) -> usize { + (w.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 17) as usize + } + /// True when newly inserted. + fn insert(&mut self, w: u64) -> bool { + assert!(self.len * 2 < self.mask, "snap census: word set full"); + let mut i = Self::hash(w) & self.mask; + unsafe { + loop { + let s = *self.slots.add(i); + if s == w { + return false; + } + if s == 0 { + *self.slots.add(i) = w; + self.len += 1; + return true; + } + i = (i + 1) & self.mask; + } + } + } + /// Drop `w` (a tombstone keeps later probes intact). + fn remove(&mut self, w: u64) { + let mut i = Self::hash(w) & self.mask; + unsafe { + loop { + let s = *self.slots.add(i); + if s == w { + *self.slots.add(i) = TOMBSTONE; + return; + } + if s == 0 { + return; + } + i = (i + 1) & self.mask; + } + } + } + fn contains(&self, w: u64) -> bool { + let mut i = Self::hash(w) & self.mask; + unsafe { + loop { + let s = *self.slots.add(i); + if s == w { + return true; + } + if s == 0 { + return false; + } + i = (i + 1) & self.mask; + } + } + } +} + +/// Open-addressing map from a birth's start to its size. +struct SizeMap { + keys: *mut u64, + vals: *mut u32, + mask: usize, + len: usize, +} + +impl SizeMap { + unsafe fn new(log2: u32) -> Self { + let n = 1usize << log2; + SizeMap { + keys: map_zeroed::(n), + vals: map_zeroed::(n), + mask: n - 1, + len: 0, + } + } + fn insert(&mut self, k: u64, v: u32) { + assert!(self.len * 2 < self.mask, "snap census: size map full"); + let mut i = WordSet::hash(k) & self.mask; + unsafe { + loop { + let s = *self.keys.add(i); + if s == k || s == 0 { + if s == 0 { + self.len += 1; + } + *self.keys.add(i) = k; + *self.vals.add(i) = v; + return; + } + i = (i + 1) & self.mask; + } + } + } + fn get(&self, k: u64) -> Option { + let mut i = WordSet::hash(k) & self.mask; + unsafe { + loop { + let s = *self.keys.add(i); + if s == k { + return Some(*self.vals.add(i)); + } + if s == 0 { + return None; + } + i = (i + 1) & self.mask; + } + } + } +} + +/// Open-addressing map from a word to the member of D that recorded it +/// (`init << 32 | pc`, the pc truncated to 32 bits; pc 0 = a binding the +/// member initialized). The cascade attribution: which member's access a +/// later init conflicted with. +struct CauseMap { + keys: *mut u64, + vals: *mut u64, + mask: usize, + len: usize, +} + +impl CauseMap { + unsafe fn new(log2: u32) -> Self { + let n = 1usize << log2; + CauseMap { + keys: map_zeroed::(n), + vals: map_zeroed::(n), + mask: n - 1, + len: 0, + } + } + fn insert(&mut self, k: u64, init: u32, pc: usize) { + assert!(self.len * 2 < self.mask, "snap census: cause map full"); + let v = (init as u64) << 32 | (pc as u64 & 0xffff_ffff); + let mut i = WordSet::hash(k) & self.mask; + unsafe { + loop { + let s = *self.keys.add(i); + if s == k || s == 0 { + if s == 0 { + self.len += 1; + } + *self.keys.add(i) = k; + *self.vals.add(i) = v; + return; + } + i = (i + 1) & self.mask; + } + } + } + fn get(&self, k: u64) -> Option<(u32, usize)> { + let mut i = WordSet::hash(k) & self.mask; + unsafe { + loop { + let s = *self.keys.add(i); + if s == k { + let v = *self.vals.add(i); + return Some(((v >> 32) as u32, (v & 0xffff_ffff) as usize)); + } + if s == 0 { + return None; + } + i = (i + 1) & self.mask; + } + } + } +} + +/// Append-only log of fixed records. +struct Log { + p: *mut T, + cap: usize, + len: usize, +} + +impl Log { + unsafe fn new(cap: usize) -> Self { + Log { + p: map_zeroed::(cap), + cap, + len: 0, + } + } + fn push(&mut self, v: T) { + assert!(self.len < self.cap, "snap census: log full"); + unsafe { *self.p.add(self.len) = v }; + self.len += 1; + } + fn as_slice(&self) -> &[T] { + unsafe { std::slice::from_raw_parts(self.p, self.len) } + } + fn as_mut_slice(&mut self) -> &mut [T] { + unsafe { std::slice::from_raw_parts_mut(self.p, self.len) } + } +} + +// --------------------------------------------------------------------------- +// State +// --------------------------------------------------------------------------- + +/// How an evaluation was first reached. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +enum Reach { + /// The entry's eager import list (before its body). + EntryEager = 0, + /// A static dependency of another evaluation (before that one's body). + StaticDep = 1, + /// From a body (an evaluation's, the entry's, or a job's) by `import()`. + DynImport = 2, + /// From a body by `require`. + Require = 3, +} + +#[derive(Clone, Copy)] +struct Hit { + class: u8, + what: &'static str, + pcs: [usize; 6], +} + +#[derive(Clone, Copy)] +struct Conflict { + /// 2: read a word a member of D wrote; 3: wrote a word a member of D + /// read or wrote; 4: a dynamic-subgraph init wrote a word that existed + /// when the dynamic import started. + rule: u8, + word: usize, + pc: usize, + write: bool, + /// The member of D whose recorded access this one conflicts with (its + /// last write of the word, or its first read), and that access's pc. + cause: u32, + cause_pc: usize, +} + +#[derive(Clone, Copy)] +struct Init { + addr: usize, + /// The emitted once guard of this evaluation; path-wrapper frames have none. + guard: usize, + parent: u32, + reach: Reach, + /// In the dynamic subgraph: reached by `import()`, or below such an init. + dynamic: bool, + /// Reached from the entry's body (as opposed to its eager list). + from_entry_body: bool, + enter_seq: u32, + exit_seq: u32, + open: bool, + parts: u8, + threw: bool, + seeded: bool, + transparent: bool, + ir: u64, + births: u64, + birth_bytes: u64, + host_count: u32, + host: Option, + conflict: Option, + in_d: bool, + faults: u64, + /// `new Function` bodies and eval scripts made while this init ran. + dyn_fns: u32, + evals: u32, + /// The check run skipped this member's body parts. + skipped: bool, + /// The entry module's own evaluation (a program with a worker evaluates + /// its entry as a module): the entry body. The check run runs it. + entry_module: bool, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +enum FrameKind { + Init = 0, + Job = 1, +} + +#[derive(Clone, Copy)] +struct Frame { + init: u32, + kind: FrameKind, +} + +#[derive(Clone, Copy)] +struct Birth { + addr: usize, + size: u32, + init: u32, +} + +/// Page flags (one byte per page). +const PF_W: u8 = 1; +const PF_R: u8 = 2; +const PF_PRE: u8 = 4; +const PF_LISTED: u8 = 8; +const PF_TRACKED: u8 = 16; +/// The page holds watched env words (`env_words`): closed whatever runs. +const PF_ENV: u8 = 32; + +/// The evaluation address of the entry body's pseudo-init (design §2.10.7). +const ENTRY_BODY_ADDR: usize = 1; + +struct PageMap { + lo: usize, + pages: usize, + flags: *mut u8, +} + +impl PageMap { + unsafe fn new(lo: usize, hi: usize) -> Self { + let lo = lo & !(PAGE - 1); + let pages = (hi - lo).div_ceil(PAGE); + PageMap { + lo, + pages, + flags: map_zeroed::(pages.max(1)), + } + } + #[inline] + fn slot(&self, addr: usize) -> Option<*mut u8> { + let i = addr.wrapping_sub(self.lo) / PAGE; + (addr >= self.lo && i < self.pages).then(|| unsafe { self.flags.add(i) }) + } +} + +struct State { + out: String, + inits: Vec, + frames: Vec, + seq: u32, + seeded: Vec, + rule4: bool, + sabotage_env: bool, + /// `PERRY_SNAP_DYN=defer`: dynamic code defers the init (design §2.6 (a)). + dyn_defer: bool, + /// `PERRY_SNAP_TRANSPARENT`: evaluation bodies whose objects and + /// globals are not locations (a what-if: the CJS loader's shared state + /// split per module). Their objects' overflow and meta cells follow. + transparent: Vec, + transp: WordSet, + transp_no: WordSet, + transp_globals: WordSet, + transp_objs: Vec<(usize, usize)>, + transparent_hits: u64, + entry_body_seq: u32, + require_depth: i32, + /// A literal CommonJS require edge is calling its target's init guard + /// from this frame's init (`js_snap_require_edge`): the next evaluation + /// that frame starts was reached by `require`, not by `import()`. + require_edge: Option, + bookkeeping: i32, + w: WordSet, + r: WordSet, + /// Who recorded each `w` / `r` word (the cascade attribution). + wby: CauseMap, + rby: CauseMap, + /// Registered module-global root slots (the handler cannot borrow + /// `GLOBAL_ROOTS`: the interrupted code may hold it). + globals: WordSet, + /// One bit per 8-byte word of the reservation: a birth starts there. + starts: *mut u64, + /// Host touches while the runtime (init 0) works for an init: the realm + /// populating itself on first use. (what, call chain), first per what. + realm_hosts: Vec<(&'static str, [usize; 6])>, + excluded: u64, + /// R1 (design §2.10.16; `PERRY_SNAP_R1=0` turns it off): representation + /// decides ordering, not ownership. Words a member of D wrote inside a + /// representation scope, as a meta install or into an excluded cell, of + /// an object no member of D bore: rolled back, left out of the digest, + /// no rule-2/3 edge. + r1: bool, + repw: WordSet, + repw_writes: u64, + /// (word, value before the first rolled-back write). + repw_old: Log<(u64, u64)>, + /// The value before the store being classified after its step. + store_before: Option, + repw_owned: u64, + repw_reads: u64, + /// Depth of the representation scopes inside `bookkeeping`. + repr_depth: i32, + /// The init whose frame was on top at each birth (by start). + born_by: SizeMap, + /// R2: a shared immutable cell (string, regex program, canonical key + /// list) is named by its content and takes no rank. `PERRY_SNAP_R2=0`. + r2: bool, + /// Key-name rule (design §2.10.18): a key word is named by what it is + /// (a leaked symbol by its description), never by its address. + /// `PERRY_SNAP_KEYNAME=0`. + keyname: bool, + /// R3: a WeakStorage digests its entry set, not its buckets. `PERRY_SNAP_R3=0`. + r3: bool, + /// Serial rule (design §2.10.17): a dynamic prototype serial is drawn in + /// allocation order (`proto_validity::next_prototype_serial`), so it is + /// representation: the digest names it by the birth name of the + /// prototype that carries it. `PERRY_SNAP_SERIAL=0`. + serial: bool, + /// Symbol identity (design §2.10.18): a fresh symbol's id is drawn in + /// creation order (`symbol::next_id`), so the digest names the id word + /// by the symbol's own birth name. `PERRY_SNAP_SYMID=0`. + symid: bool, + /// First-use rule (design §2.10.20): a cache of static data filled on + /// first use (a literal class's keys, a regex literal site's matcher + /// data) is filled by whichever init asks first, D in the census and an + /// imaged init in the check: the fill is the runtime's (init 0). + /// `PERRY_SNAP_FIRSTUSE=0`. + first_use: bool, + /// Realm rule (design §2.10.20): a realm intrinsic (the global object, + /// %Object%, the generator and iterator towers, %TypedArray%) is born + /// lazily inside whichever init first asks for it, D in the census and + /// an imaged init in the check: its birth and build are the runtime's + /// (init 0). `PERRY_SNAP_REALM=0`. + realm_rule: bool, + /// Slack rule (design §2.10.20): an object's inline width is learned + /// allocation state (`shapes_birth_width`, `LEARNED_INLINE_FIELDS`) that + /// D's births and spills move: capacity past the key count is + /// representation. The digest names an object's shape with its live + /// bound clipped to its key count and leaves the slack slots out. + /// `PERRY_SNAP_SLACK=0`. + slack: bool, + /// Tail shape rule: an arguments object's private tail + /// (`ArgumentsTail`: length, birth ShapeId) carries a second copy of a + /// ShapeId, a creation-order number; it is named by the shape it names, + /// as the header word is. `PERRY_SNAP_TAILSHAPE=0`. + tail_shape: bool, + /// Slot owner rule: a module-global root slot is owned by its first + /// registrant in the census (the defining module); in the check run a + /// slot of D's arrives poisoned, and an importer's registration of it is + /// the duplicate registration, never ownership. `PERRY_SNAP_SLOTOWNER=0`. + slot_owner_rule: bool, + /// Backing rule: canonical key lists share one backing per growth chain, + /// and whether a node sits on a backing with room (and a later list + /// extends it in place) depends on who built the chain first. A word + /// into a canonical backing outside a shape word names the chain by its + /// first key; the node's count is checked where it lives (shape words). + /// `PERRY_SNAP_BACKING=0`. + backing: bool, + /// Check order rule: an imaged init the census entered inside the body + /// of a member of D runs in the check where that body is skipped, in + /// entry order (`.dkids`, beside the D file the check reads). + /// `PERRY_SNAP_DKIDS=0`. + dkids: Vec<(usize, usize, usize)>, + /// D-written slot rule: a module-global slot a member of D wrote is + /// left out of the census digest and named in `.wd` (`0slot`, the + /// dig's own name for a slot), so the check leaves it out too, as a + /// birth word D wrote. `PERRY_SNAP_WDSLOT=0`. + wd_slot: bool, + /// Size of every birth and realm image cell, by start. + sizes: SizeMap, + /// One bit per 8-byte word of the realm section: an image cell starts. + realm_starts: *mut u64, + free_space: u64, + reborn: u64, + silent_stores: u64, + noprotect: bool, + /// The meta rule (design §2.10.11; `PERRY_SNAP_META=0` turns it off): + /// installing an empty meta record and marking an object a prototype + /// are representation, and a meta record's words are its owner's. + meta_rule: bool, + /// The owner's meta word whose (empty) record is being stored now. + meta_slot: usize, + /// Meta installs seen; records of pre-existing owners added to rule 4's + /// pre-existing set; classification marks (representation scopes). + meta_installs: u64, + /// Key-granular globalThis (design §2.10.13; `PERRY_SNAP_GKEY=0` turns + /// it off): the realm global's shape word is no location by itself; a + /// keyed operation on the global (a lookup of key X, the add or store of + /// key K) names the key it accesses, and an unkeyed access stands for + /// every key. `gshape`: the global's shape word (0 until it exists). + gkey_rule: bool, + gshape: usize, + /// The key the running keyed operation on the global names (a location + /// id, 0: none) and its name for the report. + gkey: u64, + gkey_name: [u8; GKEY_NAME], + gkey_len: u8, + /// Shape-word accesses: keyed reads, keyed writes, unkeyed reads, + /// unkeyed writes. + gkey_counts: [u64; 4], + /// The key names of keyed conflicts, per init (filled at the conflict). + gkey_conflicts: Vec<(u32, u64, [u8; GKEY_NAME], u8)>, + meta_owned: u64, + marks: u64, + /// `js_snap_entry_module`: the next evaluation is the entry module's own + /// (its body is the entry body, design §2.3). + entry_module_next: bool, + /// Global root slots on no tracked page that are not the runtime's own + /// thread-locals (a program slot the census does not watch): must be 0. + untracked_slots: u64, + /// The first of them (0: none), for the report. + untracked_first: usize, + /// Roots the runtime registers in its own thread-locals (inside the + /// static TLS block, outside the program's anchored pages, e.g. the realm + /// global's cache word): representation, counted on their own report + /// line, never protected. + runtime_tls_roots: u64, + runtime_tls_first: usize, + /// The executable's static TLS block `[fs - PT_TLS size, fs)`. + tls_block: (usize, usize), + /// The program's TLS slots are anchored on whole pages of their own. + tls_whole: bool, + /// `PERRY_SNAP_MODE=top`: every member of D may read and write every + /// location that exists when it starts (a ⊤ summary). Members run + /// untracked; any other init that touches such a location joins D. + top_mode: bool, + /// One bit per reservation word, at birth starts: born before the latest + /// member's start (the ⊤ pre-existing set). + pre_bits: *mut u64, + pre_upto: usize, + globals_log: Vec, + pre_globals: WordSet, + pre_globals_upto: usize, + top_armed: bool, + births: Log, + listed: Log, + regions: Vec<(usize, usize)>, + heap: PageMap, + bin: PageMap, + /// The main thread's program TLS slots (initialized data): module + /// globals live there in a program with a Worker (per-thread init flags + /// and bindings, #10399). Only the anchored range (`tls_program`). + tls: PageMap, + /// The same for the program's zero-initialized TLS slots. + tlsb: PageMap, + tls_ranges: [usize; 4], + /// Pages opened for the current instruction (the handler's own state is + /// never in TLS: a TLS page may be protected). + pending: [usize; 8], + /// Stores the faulting instruction is about to make: (word, value + /// before, pc). Classified once it has run (see `on_trap`). + stores: [(usize, u64, usize); 4], + rset_word: usize, + /// Handler depth: a fault inside the handler is reported, not hidden. + in_handler: u32, + /// The thread that runs module evaluation; the runner tracks only its + /// memory (a worker agent has its own heap regions). + main_tid: i64, + /// Single steps of other threads: (tid, page). Those accesses belong + /// to no init; they are stepped through and counted. + foreign: [(i64, usize); 64], + foreign_faults: u64, + realm: (usize, usize), + d_mode: bool, + perf_fd: i32, + last_ir: u64, + /// Birth-log length when the first dynamic import started (rule 4). + pre_births: usize, + pre_sorted: Vec<(usize, usize)>, + rule4_armed: bool, + host_outside: u64, + faults_total: u64, + faults_ignored: u64, + boundary_seq: u32, + boundary_at: &'static str, + // census at the boundary collection + live_by: std::collections::HashMap<(u32, u8), [u64; 2]>, + live_unattributed: [u64; 2], + boundary_done: bool, + /// Ground truth: every `getenv` by any code while the census runs + /// (init 0 included), with the value it found. + env_reads: Vec<(u32, String, Option)>, + /// SNAP3 (b): env reads are cells or guard members, not rule 1 + /// (`PERRY_SNAP_ENV=0`: rule 1 as before). + env_rule: bool, + /// SNAP3 (a): write containment for the `import()` subgraph + /// (`PERRY_SNAP_Q7=0`: every init of the subgraph defers). + q7: bool, + /// The kind `js_snap_env_site` left for the frame's next `js_getenv`. + env_site: Option<(u32, u8)>, + env_obj: u64, + env_obj_last: u32, + env_obs: Vec, + /// Watched words: the env block (`ENV_BLOCK`), store-site string bodies + /// (their `env_obs` index), poisoned slots (`POISON`). + env_words: SizeMap, + env_storage: Vec<(usize, usize)>, + /// The init that registered each `globals_log` slot. + slot_owner: Vec, + /// The check run (`PERRY_SNAP_SKIP=`): D's evaluation addresses + /// (sorted), whose bodies are skipped. + skip: Vec, + /// The check run's poisoned-binding reads. Pushed in signal context: + /// the capacity is reserved up front (a push that grows would allocate + /// inside the fault handler). + check_fail: Vec<(u32, usize, usize, bool)>, + /// The check run writes each poisoned read to `.checkfail` as it + /// happens (signal-safe), so a run that never reaches the boundary + /// still leaves its report. + checkfail_fd: i32, + /// The check run's digest leaves out the words the census pass's D wrote + /// (`PERRY_SNAP_WD`, by logical name). + wd_names: std::collections::HashSet<(u64, usize)>, + entry_init: u32, + mark: bool, + /// R5 (`PERRY_SNAP_RSET=1`): after the boundary, the imaged cells stay + /// write-protected and their writes are counted until exit. + rset: bool, + rset_active: bool, + /// Imaged births, sorted by address: (start, end). + cand: Vec<(usize, usize)>, + cand_writes: Vec, + cand_ptr_writes: Vec, + rset_faults: u64, + t0: std::time::Instant, +} + +// --------------------------------------------------------------------------- +// Start +// --------------------------------------------------------------------------- + +pub(super) fn start_if_requested() -> bool { + match ANCHOR.started.load(Relaxed) { + 1 => return false, + 2 => return true, + _ => {} + } + let Ok(out) = std::env::var("PERRY_SNAP_CENSUS") else { + ANCHOR.started.store(1, Relaxed); + return false; + }; + if out.is_empty() { + ANCHOR.started.store(1, Relaxed); + return false; + } + ANCHOR.started.store(2, Relaxed); + unsafe { start_census(out) }; + true +} + +// Not `start`: scripts/gc_runtime_root_holders.py walks function names, and a +// census fn named like a scanner callee would read GLOBAL_ROOTS as covered. +unsafe fn start_census(out: String) { + let seeded: Vec = std::env::var("PERRY_SNAP_D") + .ok() + .and_then(|p| std::fs::read_to_string(p).ok()) + .map(|t| { + t.lines() + .filter_map(|l| usize::from_str_radix(l.trim().trim_start_matches("0x"), 16).ok()) + .collect() + }) + .unwrap_or_default(); + let (res_lo, res_hi) = crate::arena::snap_reservation_bounds(); + let realm = crate::value::addr_class::realm::bounds(); + let (bin_lo, bin_hi) = writable_image_range(realm.0); + // The program's TLS slots: only the anchored ranges, whole pages that hold + // program slots and nothing else (runtime thread-locals, libc's TLS, the + // TCB and its rseq area are outside them, so the handler path may touch + // them freely). Without the anchors (not a census compile) no TLS page is + // tracked and every TLS slot counts as untracked. + let t: [usize; 4] = std::array::from_fn(|i| ANCHOR.tls[i].load(Relaxed)); + let tls_whole = t[0] != 0 && t.iter().all(|&a| a % PAGE == 0) && t[0] <= t[1] && t[2] <= t[3]; + let (tls_lo, tls_hi, tlsb_lo, tlsb_hi) = if tls_whole { + (t[0], t[1], t[2], t[3]) + } else { + (0, 0, 0, 0) + }; + // x86-64 variant II: the static TLS block ends at the thread pointer. + let mut fs = 0usize; + libc::syscall( + libc::SYS_arch_prctl, + 0x1003, /* ARCH_GET_FS */ + &mut fs as *mut usize, + ); + let tls_block = (fs.saturating_sub(executable_tls_size()), fs); + let mut st = Box::new(State { + out, + inits: Vec::with_capacity(4096), + frames: Vec::with_capacity(256), + seq: 0, + seeded, + rule4: !std::env::var("PERRY_SNAP_Q7").is_ok_and(|v| v == "0"), + sabotage_env: std::env::var("PERRY_SNAP_SABOTAGE").is_ok_and(|v| v == "env"), + dyn_defer: std::env::var("PERRY_SNAP_DYN").is_ok_and(|v| v == "defer"), + transparent: std::env::var("PERRY_SNAP_TRANSPARENT") + .unwrap_or_default() + .split(',') + .filter_map(|a| usize::from_str_radix(a.trim().trim_start_matches("0x"), 16).ok()) + .collect(), + transp: WordSet::new(18), + transp_no: WordSet::new(20), + transp_globals: WordSet::new(14), + transp_objs: Vec::with_capacity(1 << 15), + transparent_hits: 0, + entry_body_seq: 0, + require_depth: 0, + require_edge: None, + bookkeeping: 0, + w: WordSet::new(28), + r: WordSet::new(28), + wby: CauseMap::new(28), + rby: CauseMap::new(28), + globals: WordSet::new(22), + starts: map_zeroed::(((res_hi - res_lo) / 8).div_ceil(64).max(1)), + realm_hosts: Vec::new(), + excluded: 0, + r1: !std::env::var("PERRY_SNAP_R1").is_ok_and(|v| v == "0"), + repw: WordSet::new(24), + repw_writes: 0, + repw_old: Log::new(1 << 24), + store_before: None, + repw_owned: 0, + repw_reads: 0, + repr_depth: 0, + born_by: SizeMap::new(26), + r2: !std::env::var("PERRY_SNAP_R2").is_ok_and(|v| v == "0"), + keyname: !std::env::var("PERRY_SNAP_KEYNAME").is_ok_and(|v| v == "0"), + r3: !std::env::var("PERRY_SNAP_R3").is_ok_and(|v| v == "0"), + serial: !std::env::var("PERRY_SNAP_SERIAL").is_ok_and(|v| v == "0"), + symid: !std::env::var("PERRY_SNAP_SYMID").is_ok_and(|v| v == "0"), + slack: !std::env::var("PERRY_SNAP_SLACK").is_ok_and(|v| v == "0"), + tail_shape: !std::env::var("PERRY_SNAP_TAILSHAPE").is_ok_and(|v| v == "0"), + first_use: !std::env::var("PERRY_SNAP_FIRSTUSE").is_ok_and(|v| v == "0"), + realm_rule: !std::env::var("PERRY_SNAP_REALM").is_ok_and(|v| v == "0"), + slot_owner_rule: !std::env::var("PERRY_SNAP_SLOTOWNER").is_ok_and(|v| v == "0"), + backing: !std::env::var("PERRY_SNAP_BACKING").is_ok_and(|v| v == "0"), + dkids: if std::env::var("PERRY_SNAP_DKIDS").is_ok_and(|v| v == "0") { + Vec::new() + } else { + std::env::var("PERRY_SNAP_SKIP") + .ok() + .and_then(|p| std::fs::read_to_string(format!("{}kids", p)).ok()) + .map(|t| { + t.lines() + .filter_map(|l| { + let mut row = l.split('\t'); + Some(( + usize::from_str_radix(row.next()?, 16).ok()?, + usize::from_str_radix(row.next()?, 16).ok()?, + usize::from_str_radix(row.next().unwrap_or("0"), 16).ok()?, + )) + }) + .collect() + }) + .unwrap_or_default() + }, + wd_slot: !std::env::var("PERRY_SNAP_WDSLOT").is_ok_and(|v| v == "0"), + sizes: SizeMap::new(26), + realm_starts: map_zeroed::(((realm.1 - realm.0) / 8).div_ceil(64).max(1)), + free_space: 0, + reborn: 0, + silent_stores: 0, + noprotect: std::env::var("PERRY_SNAP_NOPROTECT").is_ok_and(|v| v == "1"), + meta_rule: !std::env::var("PERRY_SNAP_META").is_ok_and(|v| v == "0"), + meta_slot: 0, + meta_installs: 0, + gkey_rule: !std::env::var("PERRY_SNAP_GKEY").is_ok_and(|v| v == "0"), + gshape: 0, + gkey: 0, + gkey_name: [0; GKEY_NAME], + gkey_len: 0, + gkey_counts: [0; 4], + gkey_conflicts: Vec::with_capacity(GKEY_CONFLICTS_MAX), + meta_owned: 0, + marks: 0, + entry_module_next: false, + untracked_slots: 0, + untracked_first: 0, + runtime_tls_roots: 0, + runtime_tls_first: 0, + tls_block, + tls_whole, + top_mode: std::env::var("PERRY_SNAP_MODE").is_ok_and(|v| v == "top"), + pre_bits: map_zeroed::(((res_hi - res_lo) / 8).div_ceil(64).max(1)), + pre_upto: 0, + globals_log: Vec::new(), + pre_globals: WordSet::new(22), + pre_globals_upto: 0, + top_armed: false, + births: Log::new(1 << 28), + listed: Log::new(1 << 26), + regions: Vec::new(), + heap: PageMap::new(res_lo, res_hi), + bin: PageMap::new(bin_lo, bin_hi), + tls: PageMap::new(tls_lo, tls_hi), + tlsb: PageMap::new(tlsb_lo, tlsb_hi), + tls_ranges: t, + pending: [0; 8], + stores: [(0, 0, 0); 4], + rset_word: 0, + in_handler: 0, + main_tid: libc::syscall(libc::SYS_gettid), + foreign: [(0, 0); 64], + foreign_faults: 0, + realm, + d_mode: false, + perf_fd: open_instruction_counter(), + last_ir: 0, + pre_births: usize::MAX, + pre_sorted: Vec::new(), + rule4_armed: false, + host_outside: 0, + faults_total: 0, + faults_ignored: 0, + boundary_seq: 0, + boundary_at: "", + live_by: std::collections::HashMap::new(), + live_unattributed: [0; 2], + boundary_done: false, + env_reads: Vec::new(), + env_rule: !std::env::var("PERRY_SNAP_ENV").is_ok_and(|v| v == "0"), + q7: !std::env::var("PERRY_SNAP_Q7").is_ok_and(|v| v == "0"), + env_site: None, + env_obj: 0, + env_obj_last: 0, + env_obs: Vec::new(), + env_words: SizeMap::new(20), + env_storage: Vec::new(), + slot_owner: Vec::new(), + skip: { + let mut v = std::env::var("PERRY_SNAP_SKIP") + .map(|p| read_hex_lines(&p)) + .unwrap_or_default(); + v.sort_unstable(); + v + }, + check_fail: Vec::with_capacity(CHECK_FAIL_MAX), + checkfail_fd: -1, + wd_names: std::env::var("PERRY_SNAP_WD") + .ok() + .and_then(|p| std::fs::read_to_string(p).ok()) + .map(|t| { + t.lines() + .filter_map(|l| { + let (n, o) = l.split_once('\t')?; + Some((u64::from_str_radix(n, 16).ok()?, o.parse().ok()?)) + }) + .collect() + }) + .unwrap_or_default(), + entry_init: 0, + mark: std::env::var("PERRY_SNAP_MARK").is_ok_and(|v| v == "1"), + rset: std::env::var("PERRY_SNAP_RSET").is_ok_and(|v| v == "1"), + rset_active: false, + cand: Vec::new(), + cand_writes: Vec::new(), + cand_ptr_writes: Vec::new(), + rset_faults: 0, + t0: std::time::Instant::now(), + }); + // Init 0: the runtime's own start and the entry's body. + st.inits.push(blank_init(0, 0, Reach::EntryEager)); + st.last_ir = read_ir(st.perf_fd); + ANCHOR.state.store(&mut *st as *mut State, Relaxed); + std::mem::forget(st); + let st = state(); + // Nothing moves until the boundary. The weights run (no protection) + // collects as production does: its Ir per init includes the collections. + if !st.noprotect { + crate::gc::gc_suppress(); + } + // The objects that exist now are the runtime's (init 0). + crate::arena::arena_walk_objects_with_block_index(|header, _| { + let size = (*(header as *const crate::gc::GcHeader)).size as usize; + set_start(st, header as usize); + st.sizes.insert(header as u64, size as u32); + st.births.push(Birth { + addr: header as usize, + size: size as u32, + init: 0, + }); + }); + crate::arena::snap_reservation_regions(|base, end| st.regions.push((base, end))); + for &(b, e) in &st.regions.clone() { + mark_tracked(b, e); + } + mark_tracked(realm.0, realm.1); + let mut slots = Vec::new(); + crate::gc::for_each_global_root_slot(|slot| slots.push(slot)); + for slot in slots { + count_unwatched_root(&mut *st, slot); + st.globals.insert(slot as u64); + st.globals_log.push(slot); + st.slot_owner.push(0); + mark_tracked(slot, slot + 8); + } + walk_realm_cells(st); + // The check run's report file, opened before any page closes: this + // allocates, and the handlers are not installed yet. + if !st.skip.is_empty() { + let path = format!("{}.checkfail\0", st.out); + st.checkfail_fd = libc::open( + path.as_ptr().cast(), + libc::O_WRONLY | libc::O_CREAT | libc::O_TRUNC | libc::O_CLOEXEC, + 0o644, + ); + libc::atexit(check_exit); + } + // The check run: the global slots the census pass's D registered are + // poisoned (any access is a check failure). + if let Ok(p) = std::env::var("PERRY_SNAP_POISON") { + for slot in read_hex_lines(&p) { + if page_flags(st, slot).is_some() { + mark_tracked(slot, slot + 8); + watch_env_words(st, slot, slot + 8, POISON); + } + } + } + install_handlers(); + ANCHOR.on.store(true, Relaxed); + crate::arena::refresh_inline_allocation_limit(); +} + +/// Record the realm section's image cells (header + size, back to back, +/// zero padding between them skipped). +unsafe fn walk_realm_cells(st: &mut State) { + let (lo, hi) = st.realm; + let mut p = (lo + 7) & !7; + while p + 8 <= hi { + let g = &*(p as *const crate::gc::GcHeader); + let size = g.size as usize; + if size >= 8 && size % 8 == 0 && p + size <= hi && g.obj_type != 0 { + let i = (p - lo) / 8; + *st.realm_starts.add(i / 64) |= 1 << (i % 64); + st.sizes.insert(p as u64, size as u32); + p += size; + } else { + p += 8; + } + } +} + +/// The realm image cell that holds `word`. +fn realm_cell(st: &State, word: usize) -> Option { + let (lo, hi) = st.realm; + if word < lo || word >= hi { + return None; + } + let mut i = (word - lo) / 8; + loop { + let w = unsafe { *st.realm_starts.add(i / 64) } & (u64::MAX >> (63 - (i % 64))); + if w != 0 { + let bit = 63 - w.leading_zeros() as usize; + return Some(lo + ((i / 64) * 64 + bit) * 8); + } + if i < 64 { + return None; + } + i = i - (i % 64) - 1; + } +} + +fn blank_init(addr: usize, parent: u32, reach: Reach) -> Init { + Init { + addr, + guard: 0, + parent, + reach, + dynamic: false, + from_entry_body: false, + enter_seq: 0, + exit_seq: 0, + open: false, + parts: 0, + threw: false, + seeded: false, + transparent: false, + ir: 0, + births: 0, + birth_bytes: 0, + host_count: 0, + host: None, + conflict: None, + dyn_fns: 0, + evals: 0, + in_d: false, + faults: 0, + skipped: false, + entry_module: false, + } +} + +/// The executable's writable mapping(s) around `inside` (its .data/.bss, +/// where module globals and the realm section live), from /proc/self/maps. +fn writable_image_range(inside: usize) -> (usize, usize) { + let maps = std::fs::read_to_string("/proc/self/maps").unwrap_or_default(); + let mut rows: Vec<(usize, usize, bool)> = Vec::new(); + for line in maps.lines() { + let mut it = line.split_whitespace(); + let (Some(range), Some(perms)) = (it.next(), it.next()) else { + continue; + }; + let Some((a, b)) = range.split_once('-') else { + continue; + }; + let (Ok(a), Ok(b)) = (usize::from_str_radix(a, 16), usize::from_str_radix(b, 16)) else { + continue; + }; + rows.push((a, b, perms.starts_with("rw"))); + } + let Some(i) = rows.iter().position(|&(a, b, _)| a <= inside && inside < b) else { + return (inside, inside + PAGE); + }; + let (mut lo, mut hi) = (rows[i].0, rows[i].1); + let mut j = i; + while j > 0 && rows[j - 1].2 && rows[j - 1].1 == lo { + j -= 1; + lo = rows[j].0; + } + let mut k = i; + while k + 1 < rows.len() && rows[k + 1].2 && rows[k + 1].0 == hi { + k += 1; + hi = rows[k].1; + } + (lo, hi) +} + +fn open_instruction_counter() -> i32 { + // perf_event_attr, version 0 layout (64 bytes). + let mut attr = [0u64; 8]; + let type_and_size: u64 = /* PERF_TYPE_HARDWARE */ 0 | (64u64 << 32); + attr[0] = type_and_size; + attr[1] = 1; // PERF_COUNT_HW_INSTRUCTIONS + attr[5] = (1 << 5) | (1 << 6); // exclude_kernel | exclude_hv + let fd = unsafe { + libc::syscall( + libc::SYS_perf_event_open, + attr.as_ptr(), + 0 as libc::pid_t, + -1 as libc::c_int, + -1 as libc::c_int, + 0 as libc::c_ulong, + ) + }; + fd as i32 +} + +fn read_ir(fd: i32) -> u64 { + if fd < 0 { + return 0; + } + let mut v = 0u64; + unsafe { libc::read(fd, (&mut v as *mut u64).cast(), 8) }; + v +} + +// --------------------------------------------------------------------------- +// Frames +// --------------------------------------------------------------------------- + +fn charge(st: &mut State) { + let now = read_ir(st.perf_fd); + let top = st.frames.last().map_or(0, |f| f.init); + st.inits[top as usize].ir += now.saturating_sub(st.last_ir); + st.last_ir = now; +} + +pub(super) fn current_init() -> u32 { + state().frames.last().map_or(0, |f| f.init) +} + +fn set_mode_for_top(st: &mut State) { + let top = st.frames.last().map_or(0, |f| f.init) as usize; + if st.mark { + // A syscall trace (strace) attributes what follows to this init. + unsafe { + libc::syscall( + libc::SYS_lseek, + -1 as libc::c_int, + top as libc::c_long, + 0x5a, + ) + }; + } + let want = top != 0 && st.inits[top].in_d && !st.noprotect; + if want != st.d_mode { + st.d_mode = want; + unsafe { apply_protection(st) }; + } +} + +pub(super) fn init_enter(addr: usize) { + if !on() && !start_if_requested() { + return; + } + let st = state(); + charge(st); + let parent = st.frames.last().copied(); + let parent_id = parent.map_or(0, |f| f.init); + let parent_in_body = match parent { + None => st.entry_body_seq != 0, + Some(f) if f.kind == FrameKind::Job => true, + Some(f) => { + st.inits[f.init as usize].parts >= 2 + || st.inits[f.init as usize].addr == 0 + || st.inits[f.init as usize].addr == ENTRY_BODY_ADDR + } + }; + let edge = st.require_edge.take() == Some(parent_id); + let entry_module = std::mem::take(&mut st.entry_module_next) + && st.entry_init != 0 + && parent_id == st.entry_init; + let reach = if entry_module { + Reach::EntryEager + } else if st.require_depth > 0 || edge { + Reach::Require + } else if !parent_in_body { + if parent.is_none() { + Reach::EntryEager + } else { + Reach::StaticDep + } + } else { + Reach::DynImport + }; + st.seq += 1; + let id = st.inits.len() as u32; + assert!((id as usize) < MAX_INITS, "snap census: too many inits"); + let mut init = blank_init(addr, parent_id, reach); + init.enter_seq = st.seq; + init.open = true; + let parent_dynamic = parent_id != 0 && st.inits[parent_id as usize].dynamic; + init.dynamic = !entry_module && (reach == Reach::DynImport || parent_dynamic); + if entry_module { + // The entry module's evaluation is the entry body (§2.10.7): in D, + // reached at startup in every run, never part of the import() subgraph. + init.entry_module = true; + init.in_d = true; + init.host = Some(Hit { + class: Effect::HostRead as u8, + what: "entry_body", + pcs: [0; 6], + }); + } + if init.dynamic && !st.q7 { + // Without write containment an init of the `import()` subgraph has + // no proof of its own: it runs at the import, after the entry body. + init.in_d = true; + init.host = Some(Hit { + class: Effect::HostWrite as u8, + what: "dyn_import", + pcs: [0; 6], + }); + } + init.from_entry_body = (parent.is_none() && st.entry_body_seq != 0) + || (parent_id != 0 && st.inits[parent_id as usize].from_entry_body); + init.seeded = st.seeded.contains(&addr); + init.transparent = st.transparent.contains(&addr); + init.in_d = init.seeded || init.in_d; + if st.top_mode && init.in_d { + unsafe { extend_top(st) }; + } + if init.dynamic && st.rule4 && !st.rule4_armed { + unsafe { arm_rule4(st) }; + } + st.inits.push(init); + st.frames.push(Frame { + init: id, + kind: FrameKind::Init, + }); + set_mode_for_top(st); +} + +pub(super) fn init_exit(threw: bool) { + let st = state(); + charge(st); + let Some(f) = st.frames.pop() else { + return; + }; + st.seq += 1; + let i = &mut st.inits[f.init as usize]; + i.open = false; + i.exit_seq = st.seq; + if threw { + i.threw = true; + i.in_d = true; + } + set_mode_for_top(st); +} + +/// Attach the evaluation's emitted once guard after its guarded call returns. +/// The existing init record owns this property, including failed evaluations. +pub(super) fn init_guard(addr: usize, guard: usize) { + if let Some(i) = state().inits.iter_mut().rfind(|i| i.addr == addr) { + i.guard = guard; + } +} + +pub(super) fn init_part() { + let st = state(); + if let Some(f) = st.frames.last() { + if f.kind == FrameKind::Init { + let i = &mut st.inits[f.init as usize]; + i.parts = i.parts.saturating_add(1); + } + } +} + +pub(super) fn require_depth(delta: i32) { + state().require_depth += delta; +} + +/// `js_snap_require_edge`: 1 before a literal CommonJS require edge calls its +/// target's init guard, 0 after it (the target was already evaluated, so no +/// evaluation took the mark). +pub(super) fn require_edge(on: bool) { + let st = state(); + st.require_edge = on.then(|| top_init(st)); +} + +/// The first-use rule is on (`PERRY_SNAP_FIRSTUSE` is not `0`). +pub(super) fn first_use_on() -> bool { + state().first_use +} + +/// The realm rule is on (`PERRY_SNAP_REALM` is not `0`). +pub(super) fn realm_on() -> bool { + state().realm_rule +} + +pub(super) fn bookkeeping_depth(delta: i32) { + state().bookkeeping += delta; +} + +pub(super) fn job_enter(owner: u32) { + let st = state(); + charge(st); + st.frames.push(Frame { + init: owner, + kind: FrameKind::Job, + }); + set_mode_for_top(st); +} + +pub(super) fn job_exit() { + let st = state(); + charge(st); + if st.frames.last().is_some_and(|f| f.kind == FrameKind::Job) { + st.frames.pop(); + } + set_mode_for_top(st); +} + +pub(super) fn entry_body() { + let st = state(); + if st.entry_body_seq == 0 { + st.seq += 1; + st.entry_body_seq = st.seq; + // The entry's own body runs at startup in every run: a pseudo-init in + // D (design §2.10.7), so what it writes and reads is D's, and the + // inits it reaches (its `import()`) are checked against it. + charge(st); + let id = st.inits.len() as u32; + let mut init = blank_init(ENTRY_BODY_ADDR, 0, Reach::EntryEager); + init.enter_seq = st.seq; + init.open = true; + init.in_d = true; + init.from_entry_body = true; + init.host = Some(Hit { + class: Effect::HostRead as u8, + what: "entry_body", + pcs: [0; 6], + }); + st.inits.push(init); + st.entry_init = id; + st.frames.push(Frame { + init: id, + kind: FrameKind::Init, + }); + set_mode_for_top(st); + } +} + +/// The entry module's evaluation is next (`js_snap_entry_module`). +pub(super) fn entry_module() { + state().entry_module_next = true; +} + +/// `object_meta_ensure` stores a fresh (all-zero) record into the owner's +/// meta word at `slot`: an empty meta behaves as no meta, so the store is +/// representation, like the string alias hint (design §2.10.11). +pub(super) fn meta_install(slot: usize) { + let st = state(); + if st.meta_rule { + st.meta_slot = slot & !7; + } +} + +/// The record `meta` now hangs off the owner's meta word `slot`. What is +/// written into it later is a write of the owner: under rule 4 the record +/// of a pre-existing owner is pre-existing too. +pub(super) fn meta_installed(slot: usize, meta: usize) { + let st = state(); + st.meta_slot = 0; + st.meta_installs += 1; + if !st.meta_rule || !st.rule4_armed || !pre_existing(st, slot) { + return; + } + let Some(h) = object_start(st, meta) else { + return; + }; + let Some(size) = st.sizes.get(h as u64) else { + return; + }; + let e = h + size as usize; + let i = st.pre_sorted.partition_point(|&(a, _)| a <= h); + st.pre_sorted.insert(i, (h, e)); + st.meta_owned += 1; + let mut p = h & !(PAGE - 1); + while p < e { + flag_page(st, p, PF_PRE); + unsafe { libc::mprotect(p as *mut _, PAGE, policy(st, p)) }; + p += PAGE; + } +} + +/// A classification mark (`ensure_meta_for_mark`: the prototype or exotic +/// read receiver flag, its empty record, the private shape lineage) starts +/// (`delta` 1) or ends (-1): representation, not program state. Returns +/// whether the scope is active (the meta rule is on). +/// A keyed operation on `obj` names `key` (§2.10.13): when `obj` is the +/// realm global, the census takes accesses of its shape word as accesses of +/// `key` until [`global_key_end`]. Returns the previous key to restore. +pub(super) fn global_key<'a>( + obj: usize, + key: impl FnOnce() -> Option<&'a [u8]>, +) -> Option<(u64, [u8; GKEY_NAME], u8)> { + let st = state(); + if !st.gkey_rule { + return None; + } + if st.gshape == 0 { + st.gshape = crate::object::global_this::thread_realm_global_addr(); + } + if obj == 0 || obj != st.gshape { + return None; + } + let prev = (st.gkey, st.gkey_name, st.gkey_len); + // The key's bytes may sit on a tracked page: reading them is census + // bookkeeping, not an access of the running init. + st.bookkeeping += 1; + let named = key().map(|key| { + st.gkey = gkey_loc(key); + let n = key.len().min(GKEY_NAME); + st.gkey_name[..n].copy_from_slice(&key[..n]); + st.gkey_len = n as u8; + }); + st.bookkeeping -= 1; + // A key the operation cannot name leaves the access unkeyed. + named.map(|()| prev) +} + +/// The keyed operation [`global_key`] opened is over. +pub(super) fn global_key_end(prev: (u64, [u8; GKEY_NAME], u8)) { + let st = state(); + (st.gkey, st.gkey_name, st.gkey_len) = prev; +} + +pub(super) fn representation(delta: i32) -> bool { + let st = state(); + if !st.meta_rule { + return false; + } + if delta > 0 { + st.marks += 1; + } + st.bookkeeping += delta; + st.repr_depth += delta; + true +} + +pub(super) fn attached(obj: usize, cells: &[usize]) { + let st = state(); + if st.transparent.is_empty() { + return; + } + let Some(h) = object_start(st, obj) else { + return; + }; + if !st.transp.contains(h as u64) { + return; + } + for &c in cells { + if let Some(ch) = object_start(st, c) { + st.transp.insert(ch as u64); + st.transp_no.remove(ch as u64); + if let Some(s) = st.sizes.get(ch as u64) { + st.transp_objs.push((ch, ch + s as usize)); + } + } + } +} + +pub(super) fn dyn_code(kind: u8) { + let st = state(); + let top = st.frames.last().map_or(0, |f| f.init) as usize; + if top == 0 { + return; + } + if kind == 0 { + st.inits[top].dyn_fns += 1; + } else { + st.inits[top].evals += 1; + } + if st.dyn_defer { + host( + Effect::HostWrite, + if kind == 0 { "new_function" } else { "eval" }, + ); + } +} + +pub(super) fn host(class: Effect, what: &'static str) { + let st = state(); + if st.sabotage_env && what == "env" { + return; + } + let top = st.frames.last().map_or(0, |f| f.init); + if top == 0 { + st.host_outside += 1; + if class != Effect::TargetFact && !st.realm_hosts.iter().any(|(w, _)| *w == what) { + let mut pcs = [0usize; crate::error::MAX_CAPTURED_FRAMES]; + let n = crate::error::capture_ips(&mut pcs); + let mut six = [0usize; 6]; + for (d, s) in six.iter_mut().zip(pcs[..n].iter().skip(1)) { + *d = *s; + } + st.realm_hosts.push((what, six)); + } + return; + } + let i = &mut st.inits[top as usize]; + i.host_count += 1; + if class != Effect::TargetFact && !i.in_d { + i.in_d = true; + if st.top_mode { + unsafe { extend_top(st) }; + } + set_mode_for_top(st); + } + let i = &mut st.inits[top as usize]; + if i.host.is_none() || (class != Effect::TargetFact && i.host.is_some_and(|h| h.class == 1)) { + let mut pcs = [0usize; crate::error::MAX_CAPTURED_FRAMES]; + let n = crate::error::capture_ips(&mut pcs); + let mut six = [0usize; 6]; + for (d, s) in six.iter_mut().zip(pcs[..n].iter().skip(1)) { + *d = *s; + } + i.host = Some(Hit { + class: class as u8, + what, + pcs: six, + }); + } +} + +fn set_start(st: &mut State, addr: usize) { + if let Some(i) = start_bit(st, addr) { + unsafe { *st.starts.add(i / 64) |= 1 << (i % 64) }; + } +} + +fn start_bit(st: &State, addr: usize) -> Option { + let lo = st.heap.lo; + (addr >= lo && addr < lo + st.heap.pages * PAGE).then(|| (addr - lo) / 8) +} + +/// The birth that holds `word` (its header address), from the start bitmap. +fn object_start(st: &State, word: usize) -> Option { + let mut i = start_bit(st, word)?; + let floor = i.saturating_sub(1 << 22); + loop { + let w = unsafe { *st.starts.add(i / 64) } & (u64::MAX >> (63 - (i % 64))); + if w != 0 { + let bit = 63 - w.leading_zeros() as usize; + return Some(st.heap.lo + ((i / 64) * 64 + bit) * 8); + } + if i < 64 || i - (i % 64) <= floor { + return None; + } + i = i - (i % 64) - 1; + } +} + +pub(super) fn birth(addr: usize, size: usize) { + let st = state(); + // Memory handed out again (a free-list slot, a rewound bump) holds new + // locations: an earlier birth there and what members of D did to its + // words are gone. + let earlier = object_start(st, addr + size - 1).is_some_and(|h| { + st.sizes + .get(h as u64) + .is_some_and(|s| addr < h + s as usize) + }); + if earlier { + st.reborn += 1; + for w in (addr..addr + size).step_by(8) { + st.w.remove(w as u64); + st.r.remove(w as u64); + } + if let (Some(a), Some(b)) = (start_bit(st, addr), start_bit(st, addr + size - 1)) { + for i in a..=b { + unsafe { *st.starts.add(i / 64) &= !(1 << (i % 64)) }; + } + } + } + set_start(st, addr); + st.sizes.insert(addr as u64, size as u32); + let top = st.frames.last().map_or(0, |f| f.init); + if st.inits[top as usize].transparent { + st.transp.insert(addr as u64); + st.transp_objs.push((addr, addr + size)); + } + st.born_by.insert(addr as u64, top); + st.births.push(Birth { + addr, + size: size as u32, + init: top, + }); + let i = &mut st.inits[top as usize]; + i.births += 1; + i.birth_bytes += size as u64; +} + +pub(super) fn region_mapped(base: usize, len: usize) { + let st = state(); + if unsafe { libc::syscall(libc::SYS_gettid) } != st.main_tid { + return; + } + st.regions.push((base, base + len)); + mark_tracked(base, base + len); + if st.d_mode && !st.noprotect { + unsafe { libc::mprotect(base as *mut _, len, libc::PROT_NONE) }; + } +} + +pub(super) fn global_root(slot: usize) { + let st = state(); + // A slot registered again is one address linked into several modules + // (codegen registers a class-static global in its defining module and in + // every importer that names it). The registration is GC bookkeeping, not + // an access: the slot keeps its first owner, and the re-registering init + // neither initializes it (rule 2) nor owns it (the check run's poison). + if st.globals.contains(slot as u64) { + return; + } + // The check run: a slot whose first registrant in the census was a member + // of D (the defining module) arrives poisoned (the census's `.dslots`). + // That member is skipped here, so the first registration this run sees + // is an importer's duplicate: bookkeeping as above, and the slot stays + // D's binding, owned by no imaged init. + if st.slot_owner_rule + && !st.skip.is_empty() + && st.env_words.get(slot as u64) == Some(POISON) + { + return; + } + count_unwatched_root(st, slot); + st.globals.insert(slot as u64); + st.globals_log.push(slot); + st.slot_owner.push(st.frames.last().map_or(0, |f| f.init)); + if st.inits[st.frames.last().map_or(0, |f| f.init) as usize].transparent { + st.transp_globals.insert(slot as u64); + } + mark_tracked(slot, slot + 8); + // Rule 2 names every binding a member of D initializes, written or not. + let top = st.frames.last().map_or(0, |f| f.init) as usize; + if top != 0 && st.inits[top].in_d { + st.w.insert(slot as u64); + st.wby.insert(slot as u64, top as u32, 0); + flag_page(st, slot & !(PAGE - 1), PF_W); + } + let page = slot & !(PAGE - 1); + if st.d_mode && page_flags(st, page).is_some() { + unsafe { libc::mprotect(page as *mut _, PAGE, libc::PROT_NONE) }; + } +} + +fn mark_tracked(lo: usize, hi: usize) { + let st = state(); + let mut p = lo & !(PAGE - 1); + while p < hi { + if let Some(f) = page_flags(st, p) { + unsafe { *f |= PF_TRACKED }; + } + p += PAGE; + } +} + +/// A registered root slot on no tracked page: inside the static TLS block it +/// is one of the runtime's own thread-locals (the program's TLS slots all lie +/// on the anchored pages), representation with its own count; anywhere else +/// it is a program slot the census misses. +fn count_unwatched_root(st: &mut State, slot: usize) { + if page_flags(st, slot).is_some() { + return; + } + if slot >= st.tls_block.0 && slot < st.tls_block.1 { + st.runtime_tls_roots += 1; + if st.runtime_tls_first == 0 { + st.runtime_tls_first = slot; + } + } else { + st.untracked_slots += 1; + if st.untracked_first == 0 { + st.untracked_first = slot; + } + } +} + +/// `PT_TLS` memsz of the executable, rounded up to its alignment. +fn executable_tls_size() -> usize { + unsafe extern "C" fn each( + info: *mut libc::dl_phdr_info, + _size: libc::size_t, + data: *mut libc::c_void, + ) -> libc::c_int { + let info = &*info; + for i in 0..info.dlpi_phnum as usize { + let ph = &*info.dlpi_phdr.add(i); + if ph.p_type == libc::PT_TLS { + let align = (ph.p_align as usize).max(1); + *(data as *mut usize) = (ph.p_memsz as usize).div_ceil(align) * align; + } + } + 1 // the executable is the first object; stop + } + let mut size = 0usize; + unsafe { libc::dl_iterate_phdr(Some(each), (&mut size as *mut usize).cast()) }; + size +} + +#[inline] +fn page_flags(st: &State, addr: usize) -> Option<*mut u8> { + st.heap + .slot(addr) + .or_else(|| st.bin.slot(addr)) + .or_else(|| tls_slot(st, addr)) +} + +#[inline] +fn tls_slot(st: &State, addr: usize) -> Option<*mut u8> { + st.tls.slot(addr).or_else(|| st.tlsb.slot(addr)) +} + +// --------------------------------------------------------------------------- +// Protection +// --------------------------------------------------------------------------- + +fn policy(st: &State, page: usize) -> i32 { + // The weights run (`PERRY_SNAP_NOPROTECT=1`) takes no fault: its Ir per + // init is the program's, not the census's. + if st.noprotect { + return libc::PROT_READ | libc::PROT_WRITE; + } + let Some(f) = page_flags(st, page) else { + return libc::PROT_READ | libc::PROT_WRITE; + }; + let f = unsafe { *f }; + if f & PF_TRACKED == 0 { + return libc::PROT_READ | libc::PROT_WRITE; + } + if f & PF_ENV != 0 && !st.noprotect { + return libc::PROT_NONE; + } + if st.top_mode { + return if !st.d_mode && f & PF_PRE != 0 { + libc::PROT_NONE + } else { + libc::PROT_READ | libc::PROT_WRITE + }; + } + if st.d_mode { + return libc::PROT_NONE; + } + if f & PF_W != 0 { + libc::PROT_NONE + } else if f & (PF_R | PF_PRE) != 0 { + libc::PROT_READ + } else { + libc::PROT_READ | libc::PROT_WRITE + } +} + +/// Every tracked range: the reservation's regions, the realm section, the +/// pages of global root slots (the binary-range pages flagged tracked). +unsafe fn for_each_tracked_range(st: &State, mut f: impl FnMut(usize, usize)) { + for &(b, e) in &st.regions { + f(b, e); + } + for m in [&st.bin, &st.tls, &st.tlsb] { + let mut i = 0; + while i < m.pages { + if *m.flags.add(i) & PF_TRACKED != 0 { + let s = i; + while i < m.pages && *m.flags.add(i) & PF_TRACKED != 0 { + i += 1; + } + f(m.lo + s * PAGE, m.lo + i * PAGE); + } else { + i += 1; + } + } + } +} + +unsafe fn apply_protection(st: &mut State) { + if st.top_mode { + for_each_tracked_range(st, |b, e| { + libc::mprotect(b as *mut _, e - b, libc::PROT_READ | libc::PROT_WRITE); + }); + if !st.d_mode { + for &page in st.listed.as_slice() { + let p = policy(st, page); + if p != libc::PROT_READ | libc::PROT_WRITE { + libc::mprotect(page as *mut _, PAGE, p); + } + } + } + return; + } + if st.d_mode { + for_each_tracked_range(st, |b, e| { + libc::mprotect(b as *mut _, e - b, libc::PROT_NONE); + }); + return; + } + for_each_tracked_range(st, |b, e| { + libc::mprotect(b as *mut _, e - b, libc::PROT_READ | libc::PROT_WRITE); + }); + for &page in st.listed.as_slice() { + let p = policy(st, page); + if p != libc::PROT_READ | libc::PROT_WRITE { + libc::mprotect(page as *mut _, PAGE, p); + } + } +} + +fn flag_page(st: &mut State, page: usize, bit: u8) { + if let Some(f) = page_flags(st, page) { + unsafe { + if *f & PF_LISTED == 0 { + *f |= PF_LISTED; + st.listed.push(page); + } + *f |= bit; + } + } +} + +/// Rule 4 (relaxed): the words that exist when the first dynamic import +/// starts are watched for writes by the dynamic subgraph. +unsafe fn arm_rule4(st: &mut State) { + st.rule4_armed = true; + st.pre_births = st.births.len; + let mut v: Vec<(usize, usize)> = st + .births + .as_slice() + .iter() + .map(|b| (b.addr, b.addr + b.size as usize)) + .collect(); + v.sort_unstable(); + st.pre_sorted = v; + let pages: Vec = st + .pre_sorted + .iter() + .flat_map(|&(a, e)| (a & !(PAGE - 1)..e).step_by(PAGE)) + .collect(); + for p in pages { + flag_page(st, p, PF_PRE); + } + let (rb, re) = st.realm; + let mut p = rb & !(PAGE - 1); + while p < re { + flag_page(st, p, PF_PRE); + p += PAGE; + } + let mut globals = Vec::new(); + crate::gc::for_each_global_root_slot(|s| globals.push(s)); + for s in globals { + st.pre_sorted.push((s, s + 8)); + flag_page(st, s & !(PAGE - 1), PF_PRE); + } + st.pre_sorted.sort_unstable(); + apply_protection(st); +} + +/// ⊤ mode: everything born or registered so far is pre-existing for the +/// member that starts now. +unsafe fn extend_top(st: &mut State) { + let from = st.pre_upto; + st.pre_upto = st.births.len; + for k in from..st.pre_upto { + let b = *st.births.p.add(k); + if let Some(i) = start_bit(st, b.addr) { + *st.pre_bits.add(i / 64) |= 1 << (i % 64); + } + let mut p = b.addr & !(PAGE - 1); + while p < b.addr + b.size as usize { + flag_page(st, p, PF_PRE); + p += PAGE; + } + } + for k in st.pre_globals_upto..st.globals_log.len() { + let s = st.globals_log[k]; + st.pre_globals.insert(s as u64); + flag_page(st, s & !(PAGE - 1), PF_PRE); + } + st.pre_globals_upto = st.globals_log.len(); + if !st.top_armed { + st.top_armed = true; + let (rb, re) = st.realm; + let mut p = rb & !(PAGE - 1); + while p < re { + flag_page(st, p, PF_PRE); + p += PAGE; + } + } +} + +fn top_pre_existing(st: &State, word: usize) -> bool { + if !st.top_armed { + return false; + } + if word >= st.realm.0 && word < st.realm.1 { + return true; + } + if st.pre_globals.contains(word as u64) { + return true; + } + match object_start(st, word).and_then(|h| start_bit(st, h)) { + Some(i) => unsafe { *st.pre_bits.add(i / 64) & (1 << (i % 64)) != 0 }, + None => false, + } +} + +fn pre_existing(st: &State, word: usize) -> bool { + let (rb, re) = st.realm; + if word >= rb && word < re { + return true; + } + let i = st.pre_sorted.partition_point(|&(a, _)| a <= word); + i > 0 && word < st.pre_sorted[i - 1].1 +} + +// --------------------------------------------------------------------------- +// Signals +// --------------------------------------------------------------------------- + +/// The dispositions the runner replaced, alone on their page (a page of +/// `.bss` may also hold module globals, which the runner protects). +#[repr(C, align(4096))] +struct Prev(std::mem::MaybeUninit); +static mut PREV_SEGV: Prev = Prev(std::mem::MaybeUninit::uninit()); +static mut PREV_TRAP: Prev = Prev(std::mem::MaybeUninit::uninit()); + +unsafe fn install_handlers() { + // The handlers run on their own stack: a fault whose frame cannot be + // pushed on the current stack (a protected page) still reaches them. + let size = 1 << 20; + let ss = libc::stack_t { + ss_sp: map_zeroed::(size).cast(), + ss_flags: 0, + ss_size: size, + }; + libc::sigaltstack(&ss, std::ptr::null_mut()); + let mut a: libc::sigaction = std::mem::zeroed(); + a.sa_sigaction = on_segv as *const () as usize; + a.sa_flags = libc::SA_SIGINFO | libc::SA_NODEFER | libc::SA_ONSTACK; + libc::sigemptyset(&mut a.sa_mask); + libc::sigaction( + libc::SIGSEGV, + &a, + std::ptr::addr_of_mut!(PREV_SEGV.0).cast(), + ); + let mut t: libc::sigaction = std::mem::zeroed(); + t.sa_sigaction = on_trap as *const () as usize; + t.sa_flags = libc::SA_SIGINFO | libc::SA_ONSTACK; + libc::sigemptyset(&mut t.sa_mask); + libc::sigaction( + libc::SIGTRAP, + &t, + std::ptr::addr_of_mut!(PREV_TRAP.0).cast(), + ); +} + +/// One stderr line of hex words, without allocating (signal context). +unsafe fn diag(words: &[u64]) { + let mut buf = [0u8; 256]; + let mut n = 0; + for s in b"snap census: foreign fault" { + buf[n] = *s; + n += 1; + } + for &w in words { + buf[n] = b' '; + n += 1; + for i in (0..16).rev() { + buf[n] = b"0123456789abcdef"[((w >> (i * 4)) & 15) as usize]; + n += 1; + } + } + buf[n] = b'\n'; + libc::write(2, buf.as_ptr().cast(), n + 1); +} + +unsafe fn chain(prev: *const libc::sigaction, sig: i32) { + // The check run after a poisoned read: the program ran on with a skipped + // member's binding, so a later fault is that read's consequence. The + // `#checkfail` lines are in `.checkfail` already; end with them. + let st_ptr = ANCHOR.state.load(Relaxed); + if sig == libc::SIGSEGV && !st_ptr.is_null() && !(*st_ptr).check_fail.is_empty() { + let m = b"snap census: check run fault after a poisoned read (see .checkfail)\n"; + libc::write(2, m.as_ptr().cast(), m.len()); + libc::_exit(3); + } + // Not ours: restore the previous disposition; the instruction faults + // again and takes it. + libc::sigaction(sig, prev, std::ptr::null_mut()); +} + +const EFLAGS_TF: i64 = 0x100; + +extern "C" fn on_segv(sig: libc::c_int, info: *mut libc::siginfo_t, ctx: *mut libc::c_void) { + unsafe { + let addr = (*info).si_addr() as usize; + let st_ptr = ANCHOR.state.load(Relaxed); + if st_ptr.is_null() { + return chain(std::ptr::addr_of!(PREV_SEGV.0).cast(), sig); + } + let st = &mut *st_ptr; + let tid = libc::syscall(libc::SYS_gettid); + if tid != st.main_tid { + let page = addr & !(PAGE - 1); + let tracked = page_flags(st, page).is_some_and(|f| *f & PF_TRACKED != 0); + let free = st.foreign.iter().position(|e| e.0 == 0); + if !tracked || free.is_none() { + let uc = ctx as *mut libc::ucontext_t; + let pc = (*uc).uc_mcontext.gregs[libc::REG_RIP as usize] as u64; + diag(&[0x7d, tid as u64, addr as u64, pc]); + return chain(std::ptr::addr_of!(PREV_SEGV.0).cast(), sig); + } + st.foreign_faults += 1; + st.foreign[free.unwrap()] = (tid, page); + libc::mprotect(page as *mut _, PAGE, libc::PROT_READ | libc::PROT_WRITE); + let uc = ctx as *mut libc::ucontext_t; + (*uc).uc_mcontext.gregs[libc::REG_EFL as usize] |= EFLAGS_TF; + return; + } + if st.in_handler > 0 { + let uc = ctx as *mut libc::ucontext_t; + let pc = (*uc).uc_mcontext.gregs[libc::REG_RIP as usize] as u64; + let top = st.frames.last().map_or(0, |f| f.init) as u64; + diag(&[0xdead, addr as u64, pc, top]); + return chain(std::ptr::addr_of!(PREV_SEGV.0).cast(), sig); + } + st.in_handler += 1; + on_segv_inner(st, sig, info, ctx); + st.in_handler -= 1; + } +} + +unsafe fn on_segv_inner( + st: &mut State, + sig: libc::c_int, + info: *mut libc::siginfo_t, + ctx: *mut libc::c_void, +) { + { + let addr = (*info).si_addr() as usize; + let page = addr & !(PAGE - 1); + if st.rset_active { + let uc = ctx as *mut libc::ucontext_t; + let gregs = &mut (*uc).uc_mcontext.gregs; + if gregs[libc::REG_ERR as usize] & 2 == 0 || st.heap.slot(addr).is_none() { + return chain(std::ptr::addr_of!(PREV_SEGV.0).cast(), sig); + } + st.rset_faults += 1; + let word = addr & !7; + if let Some(i) = cand_index(st, word) { + st.cand_writes[i] = st.cand_writes[i].saturating_add(1); + st.rset_word = word | 1; + } + if let Some(slot) = st.pending.iter_mut().find(|s| **s == 0) { + *slot = page; + } + libc::mprotect(page as *mut _, PAGE, libc::PROT_READ | libc::PROT_WRITE); + gregs[libc::REG_EFL as usize] |= EFLAGS_TF; + return; + } + let tracked = page_flags(st, page).is_some_and(|f| *f & PF_TRACKED != 0); + if !tracked || policy(st, page) == libc::PROT_READ | libc::PROT_WRITE { + let uc = ctx as *mut libc::ucontext_t; + let pc = (*uc).uc_mcontext.gregs[libc::REG_RIP as usize] as u64; + let flags = page_flags(st, page).map_or(0xff, |f| *f as u64); + let range = if st.heap.slot(addr).is_some() { + 1 + } else if st.bin.slot(addr).is_some() { + 2 + } else if tls_slot(st, addr).is_some() { + 3 + } else { + 0 + }; + diag(&[ + addr as u64, + pc, + flags, + range, + st.d_mode as u64, + st.top_mode as u64, + ]); + return chain(std::ptr::addr_of!(PREV_SEGV.0).cast(), sig); + } + let uc = ctx as *mut libc::ucontext_t; + let gregs = &mut (*uc).uc_mcontext.gregs; + let write = gregs[libc::REG_ERR as usize] & 2 != 0; + let pc = gregs[libc::REG_RIP as usize] as usize; + st.faults_total += 1; + // Open the page for one instruction (before recording: the record + // may read an object header on it). + if let Some(slot) = st.pending.iter_mut().find(|s| **s == 0) { + *slot = page; + } + libc::mprotect(page as *mut _, PAGE, libc::PROT_READ | libc::PROT_WRITE); + let word = addr & !7; + match st.stores.iter_mut().find(|s| s.0 == 0) { + Some(slot) if write => *slot = (word, *(word as *const u64), pc), + _ => note_access(st, word, write, pc), + } + gregs[libc::REG_EFL as usize] |= EFLAGS_TF; + } +} + +extern "C" fn on_trap(sig: libc::c_int, _info: *mut libc::siginfo_t, ctx: *mut libc::c_void) { + unsafe { + let st_ptr = ANCHOR.state.load(Relaxed); + if !st_ptr.is_null() { + let st = &mut *st_ptr; + let tid = libc::syscall(libc::SYS_gettid); + if tid != st.main_tid { + let mut mine = false; + for k in 0..st.foreign.len() { + if st.foreign[k].0 == tid { + let page = st.foreign[k].1; + st.foreign[k] = (0, 0); + libc::mprotect(page as *mut _, PAGE, policy(st, page)); + mine = true; + } + } + if mine { + let uc = ctx as *mut libc::ucontext_t; + (*uc).uc_mcontext.gregs[libc::REG_EFL as usize] &= !EFLAGS_TF; + return; + } + return chain(std::ptr::addr_of!(PREV_TRAP.0).cast(), sig); + } + } + if st_ptr.is_null() || (*st_ptr).pending[0] == 0 { + return chain(std::ptr::addr_of!(PREV_TRAP.0).cast(), sig); + } + let st = &mut *st_ptr; + let pending = st.pending; + if st.rset_active { + let w = std::mem::replace(&mut st.rset_word, 0); + if w != 0 { + let word = w & !1; + if heap_pointer(st, *(word as *const u64)) { + if let Some(i) = cand_index(st, word) { + st.cand_ptr_writes[i] = st.cand_ptr_writes[i].saturating_add(1); + } + } + } + for &page in pending.iter().take_while(|p| **p != 0) { + libc::mprotect(page as *mut _, PAGE, libc::PROT_READ); + } + } else { + // A store that left the word's value as it was is no write + // (nothing anyone reads changed); it still read the word. + let stores = std::mem::replace(&mut st.stores, [(0, 0, 0); 4]); + for &(word, before, pc) in stores.iter().take_while(|s| s.0 != 0) { + let changed = *(word as *const u64) != before; + if !changed { + st.silent_stores += 1; + } + st.store_before = Some(before); + note_access(st, word, changed, pc); + st.store_before = None; + } + for &page in pending.iter().take_while(|p| **p != 0) { + libc::mprotect(page as *mut _, PAGE, policy(st, page)); + } + } + st.pending = [0; 8]; + let uc = ctx as *mut libc::ucontext_t; + (*uc).uc_mcontext.gregs[libc::REG_EFL as usize] &= !EFLAGS_TF; + } +} + +/// One access to a tracked word by the top frame. +/// A word on any page, whatever its protection. +unsafe fn peek(st: &State, word: usize) -> u64 { + let page = word & !(PAGE - 1); + let closed = policy(st, page) & libc::PROT_READ == 0; + if closed { + libc::mprotect(page as *mut _, PAGE, libc::PROT_READ); + } + let v = *(word as *const u64); + if closed { + libc::mprotect(page as *mut _, PAGE, policy(st, page)); + } + v +} + +/// `h` is born by a transparent init, or is the overflow table or meta cell +/// one of those objects points at (decided once per cell). +fn transparent_cell(st: &mut State, h: usize, ty: u8) -> bool { + if st.transp.contains(h as u64) { + return true; + } + if (ty != crate::gc::GC_TYPE_OVERFLOW && ty != crate::gc::GC_TYPE_OBJECT_META) + || st.transp_no.contains(h as u64) + { + return false; + } + let user = (h + crate::gc::GC_HEADER_SIZE) as u64; + let found = st.transp_objs.iter().any(|&(a, e)| { + (a..e).step_by(8).any(|w| { + let v = unsafe { peek(st, w) }; + v == user || v == h as u64 + }) + }); + if found { + st.transp.insert(h as u64); + } else { + st.transp_no.insert(h as u64); + } + found +} + +fn note_access(st: &mut State, word: usize, write: bool, pc: usize) { + if st.r1 { + // The census's own bookkeeping (a root registration, a key's bytes) + // is no access; a representation scope or a meta install is one, of + // representation (R1). + if st.bookkeeping > st.repr_depth { + st.faults_ignored += 1; + return; + } + if st.repr_depth > 0 || (write && word == st.meta_slot) { + repr_access(st, word, write, pc); + return; + } + } + if st.bookkeeping > 0 { + st.faults_ignored += 1; + return; + } + if write && word == st.meta_slot { + st.faults_ignored += 1; + return; + } + if st.env_words.len > 0 { + if let Some(v) = st.env_words.get(word as u64) { + env_word_access(st, v, word, write, pc); + return; + } + } + // Words of the binary range count only as global slots or realm cells. + if (st.bin.slot(word).is_some() || tls_slot(st, word).is_some()) + && !(word >= st.realm.0 && word < st.realm.1) + && !st.globals.contains(word as u64) + { + st.faults_ignored += 1; + return; + } + if st.transp_globals.contains(word as u64) { + st.transparent_hits += 1; + return; + } + // Immutable value-identity cells are not locations: a string never + // changes after its birth (its cached hash is no observation), and a + // shared canonical key list is an interned trie node. Reaching one + // through a location a member of D wrote is caught at that location. + let cell = if st.heap.slot(word).is_some() { + object_start(st, word) + } else { + realm_cell(st, word) + }; + if st.heap.slot(word).is_some() || cell.is_some() { + // A word outside every born object is free space the allocator + // touched (block setup), not a location. + let inside = cell.is_some_and(|h| { + st.sizes + .get(h as u64) + .is_some_and(|s| word < h + s as usize) + }); + if !inside { + st.free_space += 1; + return; + } + if let Some(h) = cell { + let hp = h & !(PAGE - 1); + let fp = word & !(PAGE - 1); + let (ty, flags) = unsafe { + if hp != fp { + libc::mprotect(hp as *mut _, PAGE, libc::PROT_READ); + } + let g = &*(h as *const crate::gc::GcHeader); + let r = (g.obj_type, g.gc_flags); + if hp != fp { + libc::mprotect(hp as *mut _, PAGE, policy(st, hp)); + } + r + }; + if ty == crate::gc::GC_TYPE_STRING || flags & crate::gc::GC_FLAG_SHAPE_SHARED != 0 { + st.excluded += 1; + // R1: a member of D changing an excluded cell in place (an + // in-place append) another init bore: rolled back. + if st.r1 && write && st.d_mode && !born_in_d(st, h) { + note_repw(st, word); + } + return; + } + if !st.transparent.is_empty() && transparent_cell(st, h, ty) { + st.transparent_hits += 1; + return; + } + } + } + let top = st.frames.last().map_or(0, |f| f.init) as usize; + st.inits[top].faults += 1; + if st.gkey_rule && st.gshape == 0 { + st.gshape = crate::object::global_this::thread_realm_global_addr(); + } + if st.gkey_rule && word == st.gshape && st.gshape != 0 { + // §2.10.13: the realm global's shape word is the global's key set. + let key = st.gkey; + let (kind, locs): (usize, [(u64, bool); 2]) = match (key != 0, write) { + (true, false) => (0, [(key, false), (GKEY_UNKEYED, false)]), + (true, true) => (1, [(key, true), (GKEY_ANY, true)]), + (false, false) => (2, [(GKEY_ANY, false), (GKEY_ANY, false)]), + (false, true) => (3, [(GKEY_ANY, true), (GKEY_UNKEYED, true)]), + }; + st.gkey_counts[kind] += 1; + let had = st.inits[top].conflict.is_some(); + for (loc, w) in locs { + note_loc(st, top, word, loc, w, pc); + } + if !had + && key != 0 + && st.inits[top].conflict.is_some() + && st.gkey_conflicts.len() < GKEY_CONFLICTS_MAX + { + // Capacity is reserved: no allocation in the fault handler. + let e = (top as u32, key, st.gkey_name, st.gkey_len); + st.gkey_conflicts.push(e); + } + return; + } + note_loc(st, top, word, word as u64, write, pc); +} + +/// A rolled-back word (R1): the image holds it as before D, so its value +/// before this (the first) write is kept for the digest's content names. +fn note_repw(st: &mut State, word: usize) { + st.repw_writes += 1; + if st.repw.insert(word as u64) { + // A store classified after its step names the value before it; at + // the fault the word still holds it (its page is open). + let old = st + .store_before + .unwrap_or_else(|| unsafe { *(word as *const u64) }); + st.repw_old.push((word as u64, old)); + } +} + +/// The object starting at `h` was born while a member of D ran. +fn born_in_d(st: &State, h: usize) -> bool { + st.born_by + .get(h as u64) + .is_some_and(|i| i != 0 && st.inits[i as usize].in_d) +} + +/// R1 (design §2.10.16): an access inside a representation scope or a meta +/// install. Representation decides ordering, never who owns a write. A +/// member of D writing an object it (or another member) bore writes D state +/// (`st.w`): imaged inits that later touch those words are rule 2/3 +/// readers. A member of D writing any other object's word: the word joins +/// the rollback set (`repw`, the digest and `.wd` leave it out), no edge. +/// An imaged init's representation access is an access only of D state. +fn repr_access(st: &mut State, word: usize, write: bool, pc: usize) { + let cell = if st.heap.slot(word).is_some() { + object_start(st, word) + } else { + realm_cell(st, word) + }; + let Some(h) = cell.filter(|&h| { + st.sizes + .get(h as u64) + .is_some_and(|s| word < h + s as usize) + }) else { + st.faults_ignored += 1; + return; + }; + // A string or a shared canonical key list is a value (R2): a member of + // D extending one another init bore in place is rolled back; no other + // access of it is an access. + // The faulting page is open for the access; another header page is + // opened for the read only (as in `note_access`). + let (ty, flags) = unsafe { + let hp = h & !(PAGE - 1); + if hp != word & !(PAGE - 1) { + libc::mprotect(hp as *mut _, PAGE, libc::PROT_READ); + } + let g = &*(h as *const crate::gc::GcHeader); + let r = (g.obj_type, g.gc_flags); + if hp != word & !(PAGE - 1) { + libc::mprotect(hp as *mut _, PAGE, policy(st, hp)); + } + r + }; + if ty == crate::gc::GC_TYPE_STRING || flags & crate::gc::GC_FLAG_SHAPE_SHARED != 0 { + st.excluded += 1; + if st.d_mode && write && !born_in_d(st, h) { + note_repw(st, word); + } + return; + } + let top = st.frames.last().map_or(0, |f| f.init) as usize; + if st.d_mode { + if !write { + st.faults_ignored += 1; + } else if born_in_d(st, h) { + st.repw_owned += 1; + st.inits[top].faults += 1; + note_loc(st, top, word, word as u64, true, pc); + } else { + note_repw(st, word); + } + return; + } + if top != 0 && st.w.contains(word as u64) { + st.repw_reads += 1; + st.inits[top].faults += 1; + note_loc(st, top, word, word as u64, write, pc); + return; + } + st.faults_ignored += 1; +} + +/// One access of location `w` (a word, or a key id of the realm global whose +/// shape word is `word`) by init `top`: rules 2-4. +fn note_loc(st: &mut State, top: usize, word: usize, w: u64, write: bool, pc: usize) { + if st.top_mode { + if st.d_mode || top == 0 || !top_pre_existing(st, word) { + return; + } + let i = &mut st.inits[top]; + if i.conflict.is_none() { + i.conflict = Some(Conflict { + rule: if write { 3 } else { 2 }, + word, + pc, + write, + cause: 0, + cause_pc: 0, + }); + } + i.in_d = true; + unsafe { extend_top(st) }; + st.d_mode = true; + unsafe { apply_protection(st) }; + return; + } + if st.d_mode { + // A member (seeded, or joined earlier in this run) still names the + // first earlier member it conflicts with: the edge of the cascade + // tree for an init the seed put in D from its start. + if top != 0 && st.inits[top].conflict.is_none() { + let hit = if st.w.contains(w) { + st.wby.get(w).map(|c| (if write { 3 } else { 2 }, c)) + } else if write && st.r.contains(w) { + st.rby.get(w).map(|c| (3, c)) + } else { + None + }; + if let Some((rule, (cause, cause_pc))) = hit.filter(|h| h.1 .0 != top as u32) { + st.inits[top].conflict = Some(Conflict { + rule, + word, + pc, + write, + cause, + cause_pc, + }); + } + } + if write { + st.w.insert(w); + st.wby.insert(w, top as u32, pc); + flag_page(st, word & !(PAGE - 1), PF_W); + } else if !st.w.contains(w) { + if st.r.insert(w) { + st.rby.insert(w, top as u32, pc); + } + flag_page(st, word & !(PAGE - 1), PF_R); + } + return; + } + if top == 0 { + return; + } + let rule = if st.w.contains(w) { + Some(if write { 3 } else { 2 }) + } else if write && st.r.contains(w) { + Some(3) + } else if write && st.rule4_armed && st.inits[top].dynamic && pre_existing(st, word) { + Some(4) + } else { + None + }; + if let Some(rule) = rule { + let (cause, cause_pc) = match rule { + 2 => st.wby.get(w), + 3 if st.w.contains(w) => st.wby.get(w), + 3 => st.rby.get(w), + _ => None, + } + .unwrap_or((0, 0)); + let i = &mut st.inits[top]; + if i.conflict.is_none() { + i.conflict = Some(Conflict { + rule, + word, + pc, + write, + cause, + cause_pc, + }); + } + i.in_d = true; + st.d_mode = true; + unsafe { apply_protection(st) }; + // This access is the member's first recorded one. + if write { + st.w.insert(w); + st.wby.insert(w, top as u32, pc); + flag_page(st, word & !(PAGE - 1), PF_W); + } else { + if st.r.insert(w) { + st.rby.insert(w, top as u32, pc); + } + flag_page(st, word & !(PAGE - 1), PF_R); + } + } +} + +// --------------------------------------------------------------------------- +// Ground truth: getenv +// --------------------------------------------------------------------------- + +unsafe extern "C" { + static mut environ: *const *const libc::c_char; +} + +/// The process's `getenv`, interposed in the instrumented archives so the +/// census sees every environment read by any code (Rust std, the runtime's +/// knobs, C libraries), not only the entries that declare `env`. +#[no_mangle] +pub unsafe extern "C" fn getenv(name: *const libc::c_char) -> *mut libc::c_char { + if name.is_null() { + return std::ptr::null_mut(); + } + let key = std::ffi::CStr::from_ptr(name).to_bytes(); + let mut found: *mut libc::c_char = std::ptr::null_mut(); + let mut e = environ; + if !e.is_null() { + while !(*e).is_null() { + let entry = std::ffi::CStr::from_ptr(*e).to_bytes(); + if entry.len() > key.len() && entry.starts_with(key) && entry[key.len()] == b'=' { + found = (*e).add(key.len() + 1) as *mut libc::c_char; + break; + } + e = e.add(1); + } + } + if on() { + let st = state(); + let top = st.frames.last().map_or(0, |f| f.init); + if st.env_reads.len() < 1 << 20 { + let value = (!found.is_null()).then(|| { + std::ffi::CStr::from_ptr(found) + .to_string_lossy() + .into_owned() + }); + st.env_reads + .push((top, String::from_utf8_lossy(key).into_owned(), value)); + } + } + found +} + +// --------------------------------------------------------------------------- +// Boundary and report +// --------------------------------------------------------------------------- + +/// True when the collection must not run (R5 keeps the heap in place). +pub(super) fn skip_collection() -> bool { + let p = ANCHOR.state.load(Relaxed); + !p.is_null() && unsafe { (*p).rset_active } +} + +pub(super) fn boundary(at: &'static str) { + let st = state(); + if st.boundary_seq != 0 { + return; + } + charge(st); + st.seq += 1; + st.boundary_seq = st.seq; + st.boundary_at = at; + ANCHOR.on.store(false, Relaxed); + st.d_mode = false; + unsafe { + for_each_tracked_range(st, |b, e| { + libc::mprotect(b as *mut _, e - b, libc::PROT_READ | libc::PROT_WRITE); + }); + } + crate::arena::refresh_inline_allocation_limit(); + write_report(st); + // The check run proves the image at the boundary; what runs after it + // reads bindings of the skipped members of D, so the run ends here. + if !st.skip.is_empty() { + unsafe { libc::_exit(0) }; + } + if st.rset { + unsafe { arm_rset(st) }; + return; + } + crate::gc::gc_unsuppress(); + // Births sorted by address for the census attribution. + st.births.as_mut_slice().sort_unstable_by_key(|b| b.addr); +} + +/// R5: the imaged cells (births of inits outside D, the runtime's included) +/// are write-protected for the rest of the run; collection stays off so they +/// keep their addresses; new births go to fresh blocks. +unsafe fn arm_rset(st: &mut State) { + let mut cand: Vec<(usize, usize)> = st + .births + .as_slice() + .iter() + .filter(|b| b.init == 0 || !st.inits[b.init as usize].in_d) + .map(|b| (b.addr, b.addr + b.size as usize)) + .collect(); + cand.sort_unstable(); + st.cand_writes = vec![0; cand.len()]; + st.cand_ptr_writes = vec![0; cand.len()]; + st.cand = cand; + crate::arena::arena_start_fresh_general_block(); + let mut last = 0usize; + for i in 0..st.cand.len() { + let (a, e) = st.cand[i]; + let mut p = a & !(PAGE - 1); + while p < e { + if p != last { + libc::mprotect(p as *mut _, PAGE, libc::PROT_READ); + last = p; + } + p += PAGE; + } + } + st.rset_active = true; + libc::atexit(rset_report); +} + +fn cand_index(st: &State, word: usize) -> Option { + let i = st.cand.partition_point(|&(a, _)| a <= word); + (i > 0 && word < st.cand[i - 1].1).then(|| i - 1) +} + +fn heap_pointer(st: &State, v: u64) -> bool { + let hi = v >> 48; + if hi != 0 && hi < 0x7FF9 { + return false; + } + let a = (v & 0xFFFF_FFFF_FFFF) as usize; + st.heap.slot(a).is_some() +} + +extern "C" fn rset_report() { + let st = state(); + if !st.rset_active { + return; + } + st.rset_active = false; + use std::io::Write; + let Ok(mut f) = std::fs::File::create(format!("{}.rset.tsv", st.out)) else { + return; + }; + let cells = st.cand_writes.iter().filter(|&&n| n > 0).count(); + let ptr_cells = st.cand_ptr_writes.iter().filter(|&&n| n > 0).count(); + let writes: u64 = st.cand_writes.iter().map(|&n| n as u64).sum(); + let ptr_writes: u64 = st.cand_ptr_writes.iter().map(|&n| n as u64).sum(); + let _ = writeln!( + f, + "#rset\tcandidates={}\tcells_written={cells}\twrites={writes}\tptr_cells={ptr_cells}\tptr_writes={ptr_writes}\tfaults={}", + st.cand.len(), + st.rset_faults + ); + // Writes per cell (histogram) and the most written cells, by birth. + let mut hist = [0u64; 24]; + for &n in &st.cand_writes { + if n > 0 { + hist[(32 - n.leading_zeros()) as usize] += 1; + } + } + for (b, n) in hist.iter().enumerate() { + if *n > 0 { + let _ = writeln!(f, "#hist\t<{}\t{n}", 1u64 << b); + } + } + let mut top: Vec = (0..st.cand.len()) + .filter(|&i| st.cand_writes[i] > 0) + .collect(); + top.sort_unstable_by_key(|&i| std::cmp::Reverse(st.cand_writes[i])); + let births = st.births.as_slice(); + for &i in top.iter().take(200) { + let (a, e) = st.cand[i]; + let init = births.iter().find(|b| b.addr == a).map_or(0, |b| b.init); + let ty = unsafe { (*(a as *const crate::gc::GcHeader)).obj_type }; + let _ = writeln!( + f, + "{a:x}\t{}\t{:x}\t{}\t{}\t{}\t{}", + init, + st.inits[init as usize].addr, + crate::gc::gc_type_info(ty).map_or("?", |i| i.name), + e - a, + st.cand_writes[i], + st.cand_ptr_writes[i] + ); + } +} + +/// The boundary collection's census visits a live object. +pub(super) fn census_live(header: *const u8, obj_type: u8, size: usize) { + let p = ANCHOR.state.load(Relaxed); + if p.is_null() { + return; + } + let st = unsafe { &mut *p }; + if st.boundary_seq == 0 || st.boundary_done { + return; + } + let a = header as usize; + let b = st.births.as_slice(); + let i = b.partition_point(|x| x.addr <= a); + if i == 0 || a >= b[i - 1].addr + b[i - 1].size as usize { + st.live_unattributed[0] += 1; + st.live_unattributed[1] += size as u64; + return; + } + let init = b[i - 1].init; + let row = st.live_by.entry((init, obj_type)).or_insert([0; 2]); + row[0] += 1; + row[1] += size as u64; +} + +pub(super) fn after_boundary_collection() { + let p = ANCHOR.state.load(Relaxed); + if p.is_null() { + return; + } + let st = unsafe { &mut *p }; + if st.boundary_seq == 0 || st.boundary_done { + return; + } + st.boundary_done = true; + use std::io::Write; + let path = format!("{}.live.tsv", st.out); + if let Ok(mut f) = std::fs::File::create(&path) { + let _ = writeln!( + f, + "#unattributed\t{}\t{}", + st.live_unattributed[0], st.live_unattributed[1] + ); + let mut rows: Vec<_> = st.live_by.iter().collect(); + rows.sort(); + for ((id, t), row) in rows { + let name = crate::gc::gc_type_info(*t).map_or("?", |i| i.name); + let _ = writeln!(f, "{id}\t{name}\t{}\t{}", row[0], row[1]); + } + } +} + +fn write_report(st: &mut State) { + use std::io::Write; + let Ok(mut f) = std::fs::File::create(&st.out) else { + return; + }; + let _ = writeln!( + f, + "#snap\tboundary={}\tseq={}\tentry_body_seq={}\tinits={}\tbirths={}\tw={}\tr={}\tfaults={}\tignored={}\thost_outside={}\twall_ms={}\tperf={}", + st.boundary_at, + st.boundary_seq, + st.entry_body_seq, + st.inits.len() - 1, + st.births.len, + st.w.len, + st.r.len, + st.faults_total, + st.faults_ignored, + st.host_outside, + st.t0.elapsed().as_millis(), + st.perf_fd >= 0, + ); + let _ = writeln!( + f, + "#meta\trule={}\tinstalls={}\towned={}\tmarks={}", + st.meta_rule as u8, st.meta_installs, st.meta_owned, st.marks + ); + let _ = writeln!( + f, + "#gkey\trule={}\tglobal={:x}\tkeyed_reads={}\tkeyed_writes={}\tunkeyed_reads={}\tunkeyed_writes={}\tkeyed_conflicts={}", + st.gkey_rule as u8, + st.gshape, + st.gkey_counts[0], + st.gkey_counts[1], + st.gkey_counts[2], + st.gkey_counts[3], + st.gkey_conflicts.len() + ); + for (init, key, name, len) in st.gkey_conflicts.iter() { + let _ = writeln!( + f, + "#gkeyconflict\t{init}\t{key:x}\t{}", + tsv_text(&String::from_utf8_lossy(&name[..*len as usize])) + ); + } + let _ = writeln!( + f, + "#id\taddr\tparent\treach\tdynamic\tfrom_entry_body\tenter\texit\topen\tthrew\tseeded\tin_d\tir\tbirths\tbirth_bytes\tfaults\thost_count\thost_class\thost_what\thost_pcs\tconflict_rule\tconflict_word\tconflict_write\tconflict_pc\tconflict_owner\tconflict_cause\tconflict_cause_pc\tdyn_fns\tevals" + ); + let births = { + let mut v: Vec<(usize, usize, u32)> = st + .births + .as_slice() + .iter() + .map(|b| (b.addr, b.addr + b.size as usize, b.init)) + .collect(); + v.sort_by_key(|b| b.0); + v + }; + let owner = |word: usize| -> String { + let i = births.partition_point(|&(a, _, _)| a <= word); + if i > 0 && word < births[i - 1].1 { + let (a, e, init) = births[i - 1]; + let ty = unsafe { (*(a as *const crate::gc::GcHeader)).obj_type }; + format!( + "birth:{init}@{a:x}:{}:{}+{}", + crate::gc::gc_type_info(ty).map_or("?", |i| i.name), + e - a, + word - a + ) + } else { + "static".into() + } + }; + for (id, i) in st.inits.iter().enumerate() { + let (hc, hw, hp) = match i.host { + Some(h) => ( + h.class.to_string(), + h.what.to_string(), + h.pcs + .iter() + .map(|p| format!("{p:x}")) + .collect::>() + .join(","), + ), + None => ("".into(), "".into(), "".into()), + }; + let (cr, cw, cwr, cpc, co, cc, ccpc) = match i.conflict { + Some(c) => ( + c.rule.to_string(), + format!("{:x}", c.word), + (c.write as u8).to_string(), + format!("{:x}", c.pc), + owner(c.word), + c.cause.to_string(), + format!("{:x}", c.cause_pc), + ), + None => Default::default(), + }; + let _ = writeln!( + f, + "{id}\t{:x}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{hc}\t{hw}\t{hp}\t{cr}\t{cw}\t{cwr}\t{cpc}\t{co}\t{cc}\t{ccpc}\t{}\t{}", + i.addr, + i.parent, + i.reach as u8, + i.dynamic as u8, + i.from_entry_body as u8, + i.enter_seq, + i.exit_seq, + i.open as u8, + i.threw as u8, + i.seeded as u8, + i.in_d as u8, + i.ir, + i.births, + i.birth_bytes, + i.faults, + i.host_count, + i.dyn_fns, + i.evals, + ); + } + // Step 6: a digest of the imaged heap per init. A birth is named by its + // init's evaluation address and its rank among that init's births; a + // word that holds a pointer into the reservation hashes as the name of + // the object it points into (plus the offset), a word a member of D + // wrote is left out (the image holds it as it was before), any other + // word hashes as itself. + // + // The digest reads each logged birth as the object it was born as and + // follows its words (a program's word count, a meta record, a weak + // storage's entries): the log is the image only while the census holds + // collection, so nothing moved or died. The weights run + // (`PERRY_SNAP_NOPROTECT=1`) collects as production does: its log keeps + // births whose blocks were freed or handed out again, so it writes no + // digest (its report is the Ir per init above). + if !st.noprotect { + let log = st.births.as_slice(); + let mut rank: std::collections::HashMap = std::collections::HashMap::new(); + let mut names: Vec<(usize, usize, u64)> = Vec::with_capacity(log.len()); + let mut ranks: Vec = Vec::with_capacity(log.len()); + // The env object and its storage are the env block, filled at + // startup (an `env_object` cell): one name, never hashed by value. + let mut env_store = st.env_storage.clone(); + env_store.sort_unstable(); + let is_env = |a: usize| env_store.binary_search_by_key(&a, |r| r.0).is_ok(); + // R1: a rolled-back word reads as it was before D (first write). + let rolled_map: std::collections::HashMap = { + let mut m = std::collections::HashMap::new(); + for &(w, v) in st.repw_old.as_slice() { + m.entry(w).or_insert(v); + } + m + }; + let rolled = |w: usize| -> u64 { + rolled_map + .get(&(w as u64)) + .copied() + .unwrap_or_else(|| unsafe { *(w as *const u64) }) + }; + let fnv = |h: u64, x: u64| (h ^ x).wrapping_mul(0x100000001B3); + // R2 (design §2.10.16): a shared immutable cell is a value. 1 a + // string (its bytes), 2 a regex program (its program words), 3 a + // canonical key list (its keys, named in a second pass), 4 a + // program's automaton storage (derived state of its program). + let vkind = |b: &Birth| -> u8 { + if !st.r2 { + return 0; + } + let g = unsafe { &*(b.addr as *const crate::gc::GcHeader) }; + match g.obj_type { + crate::gc::GC_TYPE_STRING => 1, + crate::gc::GC_TYPE_REGEX_PROGRAM => 2, + crate::gc::GC_TYPE_ARRAY if g.gc_flags & crate::gc::GC_FLAG_SHAPE_SHARED != 0 => 3, + crate::gc::GC_TYPE_REGEX_CACHE => 4, + _ => 0, + } + }; + let mut vk: Vec = Vec::with_capacity(log.len()); + let mut keylists: Vec = Vec::new(); + for b in log { + let k = rank.entry(b.init).or_insert(0); + let a = st.inits[b.init as usize].addr as u64; + let kind = if is_env(b.addr) { 0 } else { vkind(b) }; + vk.push(kind); + let user = b.addr + crate::gc::GC_HEADER_SIZE; + let end = b.addr + b.size as usize; + // The runtime's births (init 0) happen when an init first uses a + // runtime object, so their order depends on D: they are named by + // type and size only. + let name = if is_env(b.addr) { + ENV_NAME + } else if kind == 1 { + let byte_len = (rolled(user) >> 32) as usize; + let p = + crate::string::string_data(user as *const crate::string::StringHeader) as usize; + let n = byte_len.min(end.saturating_sub(p)); + let bytes = unsafe { std::slice::from_raw_parts(p as *const u8, n) }; + bytes + .iter() + .fold(fnv(0x5354_5249_4e47_0000, n as u64), |h, &c| { + fnv(h, c as u64) + }) + } else if kind == 2 { + let words = unsafe { crate::regex::perex_owner::snap_program_words(user) }; + words + .iter() + .fold(fnv(0x5052_4f47_0000_0000, words.len() as u64), |h, &x| { + fnv(h, x as u64) + }) + } else if kind == 3 { + keylists.push(b.addr); + 0 + } else if kind == 4 { + // A regex program's lazy automaton storage (`GC_TYPE_REGEX_CACHE`) + // is derived state of its program (owner ruling 2026-11-03): it + // holds no program value, only states the matcher built from the + // program, so it is one fixed name with no rank and no words. If + // it ever holds a program value the digest hashes it. + 0x5245_4341_4348_4500 + } else if b.init == 0 { + let ty = unsafe { (*(b.addr as *const crate::gc::GcHeader)).obj_type } as u64; + 0x5254_0000_0000_0000 ^ (ty << 32) ^ b.size as u64 + } else { + a.wrapping_mul(0x100000001B3) ^ *k + }; + names.push((b.addr, b.addr + b.size as usize, name)); + ranks.push(*k); + if kind == 0 { + *k += 1; + } + } + let birth_names: Vec = names.iter().map(|n| n.2).collect(); + let mut wd_out = if st.skip.is_empty() { + std::fs::File::create(format!("{}.wd", st.out)).ok() + } else { + None + }; + names.sort_unstable(); + // A pointer word as a name: the cell it points into plus offset and tag. + fn pname(names: &[(usize, usize, u64)], w: u64) -> u64 { + let v = (w & 0xFFFF_FFFF_FFFF) as usize; + let i = names.partition_point(|&(a, _, _)| a <= v); + if i > 0 && v < names[i - 1].1 { + let (n, o) = (names[i - 1].2, v - names[i - 1].0); + n ^ (o as u64).rotate_left(17) ^ (w >> 48).rotate_left(40) + } else { + w + } + } + // The words of a canonical key list's first `n` keys (after the + // key-attribute reserve), as the image holds them. + let key_words = |user: usize, end: usize, n: usize| -> Vec { + let attrs = unsafe { (*(user as *const crate::gc::GcHeader).sub(1))._reserved } + & crate::gc::GC_ARRAY_KEY_ATTRS + != 0; + let first = + user + std::mem::size_of::() + if attrs { 8 } else { 0 }; + (0..n) + .map(|i| first + 8 * i) + .take_while(|&w| w + 8 <= end) + .map(|w| rolled(w)) + .collect() + }; + // Key-name rule (design §2.10.18): a key is named by what it is, never + // by where it lives. A key in a named cell takes that cell's name: a + // string by its bytes (R2), a fresh `Symbol()` by its birth (it is an + // arena cell, `symbol::alloc_symbol`, born by the init that made it; + // no later code obtains it except through a censused read, so a D + // symbol held by an imaged init is visible as rule 2). A persistent + // symbol (`symbol::leak_symbol`: no birth) is a cell of the process + // immortal symbol space, admitted by the space's own range and cell + // stride (`symbol::in_symbol_space`, design 2.10.19). Its cells are + // bump allocated in creation order, so its address (and its id) is a + // creation ordinal: an init in D that creates a persistent symbol + // first shifts every later one. It is a registry (`Symbol.for`), + // well-known, Intl or runtime symbol: any later code obtains the same + // symbol by its name, whichever init created it, so it is named by + // its kind bits (`Symbol.for`, well-known) and its description + // bytes, not by its creator nor by its address. Any other foreign + // key word hashes as its tag, like every foreign word of the digest. + // `PERRY_SNAP_KEYNAME=0`: raw words. + let kname = |names: &[(usize, usize, u64)], w: u64| -> u64 { + if st.keyname && st.r2 { + // A shape's string key is its bytes. Module pools may register + // its image atom only when they run, so omitting D can leave an + // equal heap atom (or inline string) in the canonical list. + let mut short = [0; crate::value::SHORT_STRING_MAX_LEN]; + if let Some(bytes) = unsafe { + crate::string::js_string_key_bytes(crate::JSValue::from_bits(w), &mut short) + } { + let name = bytes + .iter() + .fold(fnv(0x5354_5249_4e47_0000, bytes.len() as u64), |h, &c| { + fnv(h, c as u64) + }); + // The ordinary R2 string cell's user-pointer name. + return name + ^ (crate::gc::GC_HEADER_SIZE as u64).rotate_left(17) + ^ 0x7fff_u64.rotate_left(40); + } + } + let a = (w & 0xFFFF_FFFF_FFFF) as usize; + let named = pname(names, w); + if !st.keyname || named != w || !crate::symbol::in_symbol_space(a) { + return named; + } + let g = unsafe { &*((a - crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader) }; + if g.obj_type != crate::gc::GC_TYPE_SYMBOL { + return fnv(0x464f_5245_4947_4e00, w >> 48); + } + let sym = a as *const crate::symbol::SymbolHeader; + // The description by its bytes (a string is a value, R2), wherever it lives: a + // leaked symbol keeps it as native text and its header's `description` is null, + // so it is read through `symbol_description_text` (the one reader of both homes). + let dn = match unsafe { crate::symbol::symbol_description_text(sym) } { + None => 0, + Some(b) => b + .iter() + .fold(fnv(0x5354_5249_4e47_0000, b.len() as u64), |h, &c| { + fnv(h, c as u64) + }), + }; + fnv( + fnv( + fnv(0x5359_4d42_0000_0000, w >> 48), + (unsafe { (*sym).registered } + & (perry_abi::SYMBOL_GLOBAL_REGISTRY | perry_abi::SYMBOL_WELL_KNOWN)) + as u64, + ), + dn, + ) + }; + if !keylists.is_empty() { + keylists.sort_unstable(); + let upd: Vec<(usize, u64)> = names + .iter() + .enumerate() + .filter(|(_, n)| keylists.binary_search(&n.0).is_ok()) + .map(|(i, n)| { + let user = n.0 + crate::gc::GC_HEADER_SIZE; + let len = (rolled(user) & 0xFFFF_FFFF) as usize; + if st.backing { + let root = key_words(user, n.1, len.min(1)) + .into_iter() + .fold(fnv(0x4b45_5953_524f_4f54, (len != 0) as u64), |h, w| { + fnv(h, kname(&names, w)) + }); + return (i, root); + } + let h = key_words(user, n.1, len) + .into_iter() + .fold(fnv(0x4b45_5953_0000_0000, len as u64), |h, w| { + fnv(h, kname(&names, w)) + }); + (i, h) + }) + .collect(); + for (i, h) in upd { + names[i].2 = h; + } + } + // A shape names its key list by its first `count` keys (R2): a list + // extended in place later is the same list to this shape. + let shape_keys = |keys: u64, count: usize| -> u64 { + let v = (keys & 0xFFFF_FFFF_FFFF) as usize; + let i = names.partition_point(|&(a, _, _)| a <= v); + if !st.r2 + || i == 0 + || v >= names[i - 1].1 + || v != names[i - 1].0 + crate::gc::GC_HEADER_SIZE + { + return pname(&names, keys); + } + key_words(v, names[i - 1].1, count) + .into_iter() + .fold(fnv(0x4b45_5953_0000_0000, count as u64), |h, w| { + fnv(h, kname(&names, w)) + }) + }; + // Diagnostic (the dig dump): a shape key input key by key: L (a named heap + // list) or P (pointer named by pname), then each key word's raw value and name. + let shape_keys_tok = |keys: u64, count: usize| -> String { + let v = (keys & 0xFFFF_FFFF_FFFF) as usize; + let i = names.partition_point(|&(a, _, _)| a <= v); + if !st.r2 + || i == 0 + || v >= names[i - 1].1 + || v != names[i - 1].0 + crate::gc::GC_HEADER_SIZE + { + let (n, o) = if i > 0 && v < names[i - 1].1 { + (names[i - 1].2, v - names[i - 1].0) + } else { + (0, 0) + }; + return format!("P{keys:x}={n:x}+{o}"); + } + let ws = key_words(v, names[i - 1].1, count); + let mut t = format!("L{}", ws.len()); + for w in ws { + t.push_str(&format!(",{w:x}={:x}", kname(&names, w))); + } + t + }; + // Serial rule (design §2.10.17): a dynamic prototype serial + // (`FIRST_DYNAMIC_PROTOTYPE_SERIAL` and up, drawn in allocation + // order, so an init's serial depends on how many D drew before it) + // is named by the birth name of the object whose meta record carries + // it, the way a ShapeId is named by its shape. Intrinsic serials + // (below it) and serials of no birth (image prototypes, families + // registered before user code) are fixed numbers and hash as such. + let serial_names: std::collections::HashMap = if st.serial { + names + .iter() + .filter_map(|&(a, _, n)| { + let bits = + crate::value::js_nanbox_pointer((a + crate::gc::GC_HEADER_SIZE) as i64) + .to_bits(); + let s = unsafe { crate::object::shapes::prototype_serial(bits) }; + (s >= crate::object::proto_validity::FIRST_DYNAMIC_PROTOTYPE_SERIAL + && s < crate::object::shapes::PROTO_ID_CLASS) + .then_some((s, n)) + }) + .collect() + } else { + Default::default() + }; + // A shape's prototype identity with its serial named: a plain serial + // identity, or a MIXED one (class id beside the serial). None: not a + // named dynamic serial (the identity hashes as before). + let pser = |id: u64| -> Option { + use crate::object::shapes::{ + PROTO_ID_CLASS, PROTO_ID_MIXED, PROTO_ID_MIXED_SERIAL_BITS, PROTO_ID_UNIQUE, + }; + let (hi, s) = if id < PROTO_ID_CLASS { + (0, id) + } else if (PROTO_ID_MIXED..PROTO_ID_UNIQUE).contains(&id) { + let m = (1u64 << PROTO_ID_MIXED_SERIAL_BITS) - 1; + (id & !m, id & m) + } else { + return None; + }; + if s < crate::object::proto_validity::FIRST_DYNAMIC_PROTOTYPE_SERIAL { + return None; + } + serial_names + .get(&s) + .map(|&n| fnv(fnv(0x5345_5249_414c_0000, hi), n)) + }; + let name_of = |v: usize| -> Option<(u64, usize)> { + let i = names.partition_point(|&(a, _, _)| a <= v); + (i > 0 && v < names[i - 1].1).then(|| (names[i - 1].2, v - names[i - 1].0)) + }; + let mut per: std::collections::BTreeMap = Default::default(); + // Diagnostic: `PERRY_SNAP_DIGEST_DUMP=1` writes every hashed word + // (`.dig`: init, birth name, offset, token) to diff two runs. + let mut dig_out = std::env::var_os("PERRY_SNAP_DIGEST_DUMP") + .and_then(|_| std::fs::File::create(format!("{}.dig", st.out)).ok()); + let mut dump = |init: usize, bname: u64, off: usize, tok: String| { + if let Some(o) = dig_out.as_mut() { + use std::io::Write; + let _ = writeln!(o, "{init:x}\t{bname:x}\t{off}\t{tok}"); + } + }; + let mix = |h: &mut u64, x: u64| { + *h ^= x; + *h = h.wrapping_mul(0x100000001B3); + }; + // A pointer outside the reservation and the binary points into + // malloc memory (mimalloc picks randomized arena hints, so its + // addresses differ per run even with ASLR off): it hashes as its tag. + // A raw (untagged) word that points into mimalloc's arenas is such + // a pointer too: runtime objects keep native pointers (a closure's + // Rust-side state) in raw words. + let foreign = |v: u64| -> bool { + let a = (v & 0xFFFF_FFFF_FFFF) as usize; + let hi = v >> 48; + if hi == 0 { + return in_malloc(a); + } + (hi == 0x7FFD || hi == 0x7FFC || hi == 0x7FFF || hi == 0x7FFE) + && st.heap.slot(a).is_none() + && st.bin.slot(a).is_none() + && tls_slot(st, a).is_none() + }; + for (bk, b) in log.iter().enumerate() { + if (b.init != 0 && st.inits[b.init as usize].in_d) || is_env(b.addr) { + continue; + } + // R2: a value is no init's state; pointers name it by content. + if vk[bk] != 0 { + continue; + } + let bname = birth_names[bk]; + let e = per + .entry(st.inits[b.init as usize].addr) + .or_insert((0xcbf29ce484222325, 0, 0)); + e.1 += 1; + let ty = unsafe { (*(b.addr as *const crate::gc::GcHeader)).obj_type }; + // Slack rule (§2.10.20): an ordinary object allocated exactly at + // its shape's live bound is named up to its key count; the slots + // past it are capacity (the image keeps its own run's width: the + // shape and the allocation agree, a later key spills as today). + // An Ordinary object allocated past its live bound (a floor birth + // whose zero-bound shape hides the floor slots, while a tracking + // birth in the other run is born at its learned width) is named + // the same way: the shape never addresses capacity past `live`. + // An arguments object keeps a private tail at the allocation end, + // so it is never widened. + let slack_end = if st.slack && ty == crate::gc::GC_TYPE_OBJECT { + let base = + crate::gc::GC_HEADER_SIZE + std::mem::size_of::(); + let w = unsafe { *((b.addr + crate::gc::GC_HEADER_SIZE) as *const u64) }; + crate::object::shapes::shape_descriptor_for_word(crate::codegen_abi::LayoutWord( + (w >> 32) as u32, + )) + .and_then(|d| { + let live = d.live_inline_slot_count as usize; + let size = b.size as usize; + let widened = size > base + 8 * live + && d.object_kind == crate::object::shapes::ShapeObjectKind::Ordinary + && unsafe { (*(b.addr as *const crate::gc::GcHeader))._reserved } + & crate::gc::OBJ_FLAG_ARGUMENTS + == 0; + (size == base + 8 * live || widened) + .then(|| base + 8 * live.min(d.logical_key_count as usize)) + }) + } else { + None + }; + dump( + st.inits[b.init as usize].addr, + bname, + 0, + format!("t{ty}:{}", slack_end.unwrap_or(b.size as usize)), + ); + // R3 (design §2.10.16): a WeakStorage's bucket positions and + // entry order are derived state (they hash raw addresses): it + // digests its live entry set, sorted. + if st.r3 && ty == crate::gc::GC_TYPE_WEAK_STORAGE { + let ws = (b.addr + crate::gc::GC_HEADER_SIZE) + as *const crate::weakref::storage::WeakStorage; + let mut ents: Vec<(u64, u64)> = Vec::new(); + unsafe { + let n = (*ws).len.min((*ws).capacity) as usize; + let base = (*ws).entries(); + for i in 0..n { + let ent = base.add(i); + if (*ent).key == crate::value::TAG_UNDEFINED { + continue; + } + let kw = std::ptr::addr_of!((*ent).key) as usize; + let mut skip = false; + for w in [kw, kw + 8] { + let o = w - b.addr; + if st.w.contains(w as u64) || st.repw.contains(w as u64) { + if let Some(f) = wd_out.as_mut() { + use std::io::Write; + let _ = writeln!(f, "{bname:x}\t{o}"); + } + skip = true; + } else if st.wd_names.contains(&(bname, o)) { + skip = true; + } + } + if !skip { + ents.push((pname(&names, (*ent).key), pname(&names, (*ent).value))); + } + } + } + ents.sort_unstable(); + let h = ents + .iter() + .fold(0xcbf29ce484222325u64, |h, &(k, v)| fnv(fnv(h, k), v)); + e.2 += 1; + mix(&mut e.0, h); + dump( + st.inits[b.init as usize].addr, + bname, + crate::gc::GC_HEADER_SIZE, + format!("k{h:x}:{}", ents.len()), + ); + continue; + } + // The GC header (mark and move-request bits) is no JS state. + let mut off = crate::gc::GC_HEADER_SIZE; + // A Map store's native indexes are derived state (see + // `map::snap_native_index_range`): left out. + let skip = if ty == crate::gc::GC_TYPE_MAP_STORE { + let (s, e) = crate::map::snap_native_index_range(); + (crate::gc::GC_HEADER_SIZE + s, crate::gc::GC_HEADER_SIZE + e) + } else { + (0, 0) + }; + while off + 8 <= b.size as usize { + let word = b.addr + off; + off += 8; + if word - b.addr >= skip.0 && word - b.addr < skip.1 { + continue; + } + if slack_end.is_some_and(|e| word - b.addr >= e) { + continue; + } + if st.w.contains(word as u64) || st.repw.contains(word as u64) { + if let Some(o) = wd_out.as_mut() { + use std::io::Write; + let _ = writeln!(o, "{bname:x}\t{}", off - 8); + } + continue; + } + if st.wd_names.contains(&(bname, off - 8)) { + continue; + } + let v = unsafe { *(word as *const u64) }; + e.2 += 1; + // The carriers of numbers drawn in creation order name them as + // their uses do: a meta record's dynamic prototype serial by the + // birth name of the object it belongs to (serial rule, §2.10.17; + // `pser`, as a shape's prototype input), and a fresh symbol's id + // by the symbol's own birth name (symbol identity, §2.10.18). A + // serial of no birth hashes as its number, as before. + let q = if st.serial + && ty == crate::gc::GC_TYPE_OBJECT_META + && off - 8 + == crate::gc::GC_HEADER_SIZE + + std::mem::offset_of!(crate::object::ObjectMeta, proto_serial) + { + pser(v) + } else if st.symid + && ty == crate::gc::GC_TYPE_SYMBOL + && off - 8 + == crate::gc::GC_HEADER_SIZE + + std::mem::offset_of!(crate::symbol::SymbolHeader, id) + { + Some(fnv(0x5359_4d49_4400_0000, bname)) + } else { + None + }; + if let Some(n) = q { + mix(&mut e.0, n); + dump( + st.inits[b.init as usize].addr, + bname, + off - 8, + format!("q{n:x}"), + ); + continue; + } + // An ordinary object's first header word holds its class id + // and its ShapeId (a closure's: capture count and ShapeId). A ShapeId is the shape's NAME: ids are + // handed out in creation order, and the env object's keys + // (filled at startup) take ids too, so the same shape gets + // another id when the env block differs. It hashes as the + // shape it names: key list, counts, flags, prototype link, + // kind and representation. + // An arguments object's tail word (`ArgumentsTail`: the length + // in the low half, the birth ShapeId in the high half) is the + // same kind of carrier: it hashes as the length and the shape. + let tail_word = st.tail_shape + && ty == perry_abi::GC_TYPE_OBJECT + && off - 8 + 16 == b.size as usize + && unsafe { (*(b.addr as *const crate::gc::GcHeader))._reserved } + & crate::gc::OBJ_FLAG_ARGUMENTS + != 0; + if tail_word + || off - 8 == crate::gc::GC_HEADER_SIZE + && (ty == perry_abi::GC_TYPE_OBJECT || ty == perry_abi::GC_TYPE_CLOSURE) + { + let sid = (v >> 32) as u32; + if let Some(d) = crate::object::shapes::shape_descriptor_for_word( + crate::codegen_abi::LayoutWord(sid), + ) { + let pn = |w: u64| { + name_of((w & 0xFFFF_FFFF_FFFF) as usize).map_or(w, |(n, o)| { + n ^ (o as u64).rotate_left(17) ^ (w >> 48).rotate_left(40) + }) + }; + let parts = [ + shape_keys(d.keys, d.logical_key_count as usize), + d.logical_key_count as u64, + if st.slack && ty == perry_abi::GC_TYPE_OBJECT { + d.live_inline_slot_count.min(d.logical_key_count) as u64 + } else { + d.live_inline_slot_count as u64 + }, + d.integrity_flags as u64, + pser(d.proto_id).unwrap_or_else(|| pn(d.proto_id)), + d.object_kind as u8 as u64, + d.hole_count as u64, + d.summary as u64, + // A deprecated F64 lane (`10`) is learned record state, like the + // width above: field_rep::identity reads it as `F64`, and stores + // by D deprecate it in the census only (design §2.10.20). + if st.slack { + crate::object::field_rep::identity_with_special(d.rep) + } else { + d.rep + }, + d.special_constfn_mask as u64, + ]; + let sname = parts.iter().fold(0xcbf29ce484222325u64, |h, &x| { + (h ^ x).wrapping_mul(0x100000001B3) + }); + mix(&mut e.0, v & 0xFFFF_FFFF); + mix(&mut e.0, sname); + // The dump shows the inputs too (keys, prototype, the rest), so a + // differing shape word names its differing field. + let rest = parts[5..] + .iter() + .fold(parts[1] ^ parts[2] << 20 ^ parts[3] << 40, |h, &x| { + (h ^ x).wrapping_mul(0x100000001B3) + }); + let tok = format!( + "s{:x}:{sname:x}|k{:x}|p{:x}|r{rest:x}|K{}", + v & 0xFFFF_FFFF, + parts[0], + parts[4], + shape_keys_tok(d.keys, d.logical_key_count as usize) + ); + dump(st.inits[b.init as usize].addr, bname, off - 8, tok); + continue; + } + } + let hi = v >> 48; + if hi == 0 || hi >= 0x7FF9 { + if let Some((n, o)) = name_of((v & 0xFFFF_FFFF_FFFF) as usize) { + mix(&mut e.0, hi ^ 0xA5A5); + mix(&mut e.0, n); + mix(&mut e.0, o as u64); + dump( + st.inits[b.init as usize].addr, + bname, + off - 8, + format!("p{hi:x}:{n:x}+{o}"), + ); + continue; + } + } + if foreign(v) { + mix(&mut e.0, v >> 48); + dump( + st.inits[b.init as usize].addr, + bname, + off - 8, + format!("f{:x}", v >> 48), + ); + continue; + } + mix(&mut e.0, v); + dump( + st.inits[b.init as usize].addr, + bname, + off - 8, + format!("v{v:x}"), + ); + } + } + // The module-global slots each imaged init registered. + for (k, &slot) in st.globals_log.iter().enumerate() { + let owner = st.slot_owner.get(k).copied().unwrap_or(0); + if owner == 0 || st.inits[owner as usize].in_d { + continue; + } + // A slot a member of D wrote (the CJS support module's + // `pendingParent`, set by a deferred require inside a D body) is + // named in `.wd` like a birth word D wrote: the check, where D + // never runs, leaves it out too. + if st.w.contains(slot as u64) { + if st.wd_slot { + if let Some(o) = wd_out.as_mut() { + use std::io::Write; + let _ = writeln!(o, "0\t{slot}"); + } + } + continue; + } + if st.wd_slot && st.wd_names.contains(&(0, slot)) { + continue; + } + let e = per + .entry(st.inits[owner as usize].addr) + .or_insert((0xcbf29ce484222325, 0, 0)); + let v = unsafe { *(slot as *const u64) }; + e.2 += 1; + let hi = v >> 48; + if hi == 0 || hi >= 0x7FF9 { + if let Some((n, o)) = name_of((v & 0xFFFF_FFFF_FFFF) as usize) { + mix(&mut e.0, hi ^ 0xA5A5); + mix(&mut e.0, n); + mix(&mut e.0, o as u64); + dump( + st.inits[owner as usize].addr, + 0, + slot, + format!("p{hi:x}:{n:x}+{o}"), + ); + continue; + } + } + if foreign(v) { + mix(&mut e.0, v >> 48); + dump( + st.inits[owner as usize].addr, + 0, + slot, + format!("f{:x}", v >> 48), + ); + continue; + } + mix(&mut e.0, v); + dump(st.inits[owner as usize].addr, 0, slot, format!("v{v:x}")); + } + let mut total = 0xcbf29ce484222325u64; + for (addr, (h, objs, words)) in &per { + mix(&mut total, *h); + let _ = writeln!(f, "#digest\t{addr:x}\t{h:016x}\t{objs}\t{words}"); + } + let _ = writeln!(f, "#digest_total\t{total:016x}\t{}", per.len()); + write_env_report(st, &mut f, &ranks); + } + let _ = writeln!(f, "#excluded\t{}", st.excluded); + let _ = writeln!( + f, + "#repw\trule={}\twords={}\twrites={}\towned={}\treads={}\tr2={}\tr3={}\tserial={}\tkeyname={}\tsymid={}", + st.r1 as u8, st.repw.len, st.repw_writes, st.repw_owned, st.repw_reads, st.r2 as u8, st.r3 as u8, + st.serial as u8, st.keyname as u8, st.symid as u8 + ); + let _ = writeln!(f, "#free_space\t{}", st.free_space); + let _ = writeln!(f, "#reborn\t{}", st.reborn); + let _ = writeln!(f, "#silent_stores\t{}", st.silent_stores); + let _ = writeln!( + f, + "#transparent\t{}\t{}\t{}", + st.transparent.len(), + st.transp.len, + st.transparent_hits + ); + let _ = writeln!(f, "#foreign_thread_faults\t{}", st.foreign_faults); + let _ = writeln!( + f, + "#untracked_slots\t{}\tof\t{}\tfirst={:x}", + st.untracked_slots, st.globals.len, st.untracked_first + ); + let _ = writeln!( + f, + "#runtime_tls_roots\t{}\tfirst={:x}\tblock={:x}-{:x}", + st.runtime_tls_roots, st.runtime_tls_first, st.tls_block.0, st.tls_block.1 + ); + let r = st.tls_ranges; + let _ = writeln!( + f, + "#tls\twhole={}\tdata={:x}-{:x}\tbss={:x}-{:x}", + st.tls_whole as u8, r[0], r[1], r[2], r[3] + ); + for (what, pcs) in &st.realm_hosts { + let pcs: Vec = pcs.iter().map(|p| format!("{p:x}")).collect(); + let _ = writeln!(f, "#realm_host\t{what}\t{}", pcs.join(",")); + } + for (init, name, _) in &st.env_reads { + let _ = writeln!(f, "#getenv\t{init}\t{name}"); + } + // The next pass's seed. + if let Ok(mut d) = std::fs::File::create(format!("{}.d", st.out)) { + for i in st.inits.iter().skip(1) { + if i.in_d { + let _ = writeln!(d, "{:x}", i.addr); + } + } + } +} + +// --------------------------------------------------------------------------- +// SNAP3: env cells, the image guard (design §2.10.3-§2.10.4) +// --------------------------------------------------------------------------- + +/// What an env observation was (`EnvObs::kind`). +const OBS_OBJECT: u8 = 0; +/// A store-site read of a set var: a cell while no imaged init reads the +/// string's body. +const OBS_STORE: u8 = 1; +const OBS_OBSERVE: u8 = 2; +const OBS_HAS: u8 = 3; +/// A store-site read of an unset var (no pointer to find: a guard member). +const OBS_STORE_UNSET: u8 = 4; +/// A store-site string whose body an imaged init read. +const OBS_STORE_READ: u8 = 5; + +/// `env_words` value of an env block word (the env object's storage). +const ENV_BLOCK: u32 = u32::MAX; +/// `env_words` value of a module-global slot a member of D registered in the +/// census pass (the check run poisons it). +const POISON: u32 = u32::MAX - 1; +/// At most this many poisoned reads are recorded (`check_fail`). +const CHECK_FAIL_MAX: usize = 4096; + +/// `#checkfail\tpoison\t\t\t\t\t` to +/// `.checkfail` (signal context: no allocation). The report at the +/// boundary repeats the line with the slot's logical name. +fn checkfail_line(st: &State, init: u32, word: usize, pc: usize, write: bool) { + if st.checkfail_fd < 0 { + return; + } + let mut buf = [0u8; 160]; + let mut n = 0; + let mut put = |b: &[u8], n: &mut usize| { + for &c in b { + buf[*n] = c; + *n += 1; + } + }; + put(b"#checkfail\tpoison\t", &mut n); + let mut dec = [0u8; 10]; + let mut k = dec.len(); + let mut x = init; + loop { + k -= 1; + dec[k] = b'0' + (x % 10) as u8; + x /= 10; + if x == 0 { + break; + } + } + put(&dec[k..], &mut n); + let addr = st.inits.get(init as usize).map_or(0, |i| i.addr); + for w in [addr, word, pc] { + put(b"\t", &mut n); + let mut hex = [0u8; 16]; + let mut k = hex.len(); + let mut x = w; + loop { + k -= 1; + hex[k] = b"0123456789abcdef"[x & 15]; + x >>= 4; + if x == 0 { + break; + } + } + put(&hex[k..], &mut n); + } + put(if write { b"\t1\n" } else { b"\t0\n" }, &mut n); + unsafe { libc::write(st.checkfail_fd, buf.as_ptr().cast(), n) }; +} + +/// The check run ends before the boundary (an exception, `process.exit`): +/// write the report there, so the run always ends with one. +extern "C" fn check_exit() { + if state().boundary_seq == 0 { + boundary("exit"); + } +} + +struct EnvObs { + init: u32, + kind: u8, + name: String, + value: Option, + /// The string a store-site read returned (header address). + ptr: usize, + /// The imaged init that read a store-site string's body, and where. + by: u32, + by_pc: usize, +} + +fn top_init(st: &State) -> u32 { + st.frames.last().map_or(0, |f| f.init) +} + +/// Watch `[lo, hi)` as env words with value `v`: their pages stay closed +/// whatever runs. +fn watch_env_words(st: &mut State, lo: usize, hi: usize, v: u32) { + let mut w = lo & !7; + while w < hi { + st.env_words.insert(w as u64, v); + w += 8; + } + let mut p = lo & !(PAGE - 1); + while p < hi { + flag_page(st, p, PF_ENV); + if !st.noprotect { + unsafe { libc::mprotect(p as *mut _, PAGE, libc::PROT_NONE) }; + } + p += PAGE; + } +} + +pub(super) fn births_len() -> usize { + state().births.len +} + +pub(super) fn env_site(kind: i32) { + let st = state(); + st.env_site = Some((top_init(st), kind as u8)); +} + +pub(super) fn env_object(bits: u64) { + let st = state(); + if !st.env_rule { + return host(Effect::HostRead, "env"); + } + st.env_obj = bits; + let top = top_init(st); + if top != 0 && st.env_obj_last != top { + st.env_obj_last = top; + st.env_obs.push(EnvObs { + init: top, + kind: OBS_OBJECT, + name: String::new(), + value: None, + ptr: 0, + by: 0, + by_pc: 0, + }); + } +} + +pub(super) fn env_storage(begin: usize, obj: usize) { + let st = state(); + let h = obj - crate::gc::GC_HEADER_SIZE; + let new: Vec<(usize, usize)> = st.births.as_slice()[begin.min(st.births.len)..] + .iter() + .map(|b| (b.addr, b.addr + b.size as usize)) + .collect(); + for (a, e) in new { + st.env_storage.push((a, e)); + // The env object's header (class, shape, meta) is what dispatch and + // identity read; its slots are the env block. + let lo = if a == h { + obj + std::mem::size_of::() + } else { + a + crate::gc::GC_HEADER_SIZE + }; + watch_env_words(st, lo, e, ENV_BLOCK); + } +} + +pub(super) fn env_var(name: &str, value: Option<&str>, ptr: usize) { + let st = state(); + let top = top_init(st); + let site = st.env_site.take().filter(|s| s.0 == top).map(|s| s.1); + if !st.env_rule { + return host(Effect::HostRead, "env"); + } + let mut kind = match (site, ptr) { + (Some(1), 0) => OBS_STORE_UNSET, + (Some(1), _) => OBS_STORE, + _ => OBS_OBSERVE, + }; + let idx = st.env_obs.len() as u32; + if kind == OBS_STORE { + let h = ptr - crate::gc::GC_HEADER_SIZE; + match st.sizes.get(h as u64) { + Some(size) => watch_env_words(st, ptr, h + size as usize, idx), + // Not a logged birth: nothing to watch, so the value is observed. + None => kind = OBS_OBSERVE, + } + } + st.env_obs.push(EnvObs { + init: top, + kind, + name: name.to_owned(), + value: value.map(str::to_owned), + ptr, + by: 0, + by_pc: 0, + }); +} + +pub(super) fn env_has(name: &str, present: bool) { + let st = state(); + if !st.env_rule { + return host(Effect::HostRead, "env"); + } + let top = top_init(st); + st.env_obs.push(EnvObs { + init: top, + kind: OBS_HAS, + name: name.to_owned(), + value: present.then(|| "".to_owned()), + ptr: 0, + by: 0, + by_pc: 0, + }); +} + +/// An access to a watched env word (`note_access`, signal context: no +/// allocation). +fn env_word_access(st: &mut State, v: u32, word: usize, write: bool, pc: usize) { + let top = top_init(st); + if v == POISON { + if st.check_fail.len() < CHECK_FAIL_MAX { + st.check_fail.push((top, word, pc, write)); + checkfail_line(st, top, word, pc, write); + } + return; + } + if top == 0 || st.inits[top as usize].in_d { + return; + } + if v == ENV_BLOCK { + // Enumeration or a write of the env block: rule 1. + let i = &mut st.inits[top as usize]; + i.host_count += 1; + i.in_d = true; + if i.host.is_none_or(|h| h.class == Effect::TargetFact as u8) { + i.host = Some(Hit { + class: Effect::HostRead as u8, + what: if write { "env_write" } else { "env_enum" }, + pcs: [pc, 0, 0, 0, 0, 0], + }); + } + if st.top_mode { + unsafe { extend_top(st) }; + } + set_mode_for_top(st); + return; + } + if let Some(o) = st.env_obs.get_mut(v as usize) { + // The word holding `capacity` and the `refcount` alias hint is the + // string's ownership state, not its value: a store barrier demotes + // the string to shared there (`js_string_addref`). The value words + // (lengths, flags, bytes) are what an observation reads; the hint is + // reported with the cell so the startup fill sets it the same way. + if word == (o.ptr + std::mem::offset_of!(crate::StringHeader, refcount)) & !7 { + return; + } + if o.kind == OBS_STORE { + o.kind = OBS_STORE_READ; + o.by = top; + o.by_pc = pc; + } + } +} + +/// The check run: the evaluation now on top is a member of D. +pub(super) fn skip_body() -> bool { + let st = state(); + if st.skip.is_empty() { + return false; + } + match st.frames.last() { + Some(f) if f.kind == FrameKind::Init => { + let i = &mut st.inits[f.init as usize]; + if !i.entry_module && st.skip.binary_search(&i.addr).is_ok() { + let first = !i.skipped; + i.skipped = true; + if first { + // The imaged inits the census entered inside this body, + // evaluated here in entry order (the image holds them + // initialized: a later importer finds them done). + let addr = i.addr; + let kids: Vec<(usize, usize)> = st + .dkids + .iter() + .filter(|k| k.0 == addr) + .map(|k| (k.1, k.2)) + .collect(); + for (kid, guard) in kids { + crate::module_require::snap_evaluate_init(kid, guard); + } + } + return true; + } + false + } + _ => false, + } +} + +fn read_hex_lines(path: &str) -> Vec { + std::fs::read_to_string(path) + .map(|t| { + t.lines() + .filter_map(|l| { + usize::from_str_radix(l.split('\t').next()?.trim().trim_start_matches("0x"), 16) + .ok() + }) + .collect() + }) + .unwrap_or_default() +} + +/// A logical name for a module-global slot: its symbol (or the image +/// offset) from `dladdr`, never its address. +fn slot_name(slot: usize) -> String { + unsafe { + let mut info: libc::Dl_info = std::mem::zeroed(); + if libc::dladdr(slot as *const _, &mut info) != 0 { + let base = info.dli_fbase as usize; + if !info.dli_sname.is_null() && !info.dli_saddr.is_null() { + let s = std::ffi::CStr::from_ptr(info.dli_sname).to_string_lossy(); + return format!("g:{s}+{}", slot - info.dli_saddr as usize); + } + return format!("g:+{:x}", slot - base); + } + } + format!("g:?{slot:x}") +} + +fn tsv_text(s: &str) -> String { + s.chars() + .map(|c| { + if c == '\t' || c == '\n' || c == '\r' { + ' ' + } else { + c + } + }) + .collect() +} + +/// The `#cell`, `#guard`, `#knob` lines (design §2.10.8) and the per-init +/// slot owners for the check run. +/// The digest name of the env object and its storage. +const ENV_NAME: u64 = 0x454E_5600_0000_0000; + +/// Whether `a` lies in mimalloc's heap memory (raw native pointers). +fn in_malloc(a: usize) -> bool { + #[cfg(feature = "alloc-mimalloc")] + { + a >= 0x1_0000 && unsafe { libmimalloc_sys::mi_is_in_heap_region(a as *const _) } + } + #[cfg(not(feature = "alloc-mimalloc"))] + { + let _ = a; + false + } +} + +fn write_env_report(st: &State, f: &mut std::fs::File, rank: &[u64]) { + use std::io::Write; + let in_d = |i: u32| st.inits[i as usize].in_d; + let addr_of = |i: u32| st.inits[i as usize].addr; + // Cells: every image word that holds the env object or a store-site + // string. The image is the births of inits outside D and the global + // slots they registered; the env block itself and words a member of D + // wrote are not image words. + let mut targets: std::collections::HashMap> = Default::default(); + if st.env_obj != 0 { + targets + .entry(st.env_obj & 0xFFFF_FFFF_FFFF) + .or_default() + .push(-1); + } + // A var some imaged init observes is a guard member: the guard makes + // every imaged string of it right, so its store-site reads need no cell. + let guarded: std::collections::HashSet<&str> = st + .env_obs + .iter() + .filter(|o| { + matches!( + o.kind, + OBS_OBSERVE | OBS_HAS | OBS_STORE_UNSET | OBS_STORE_READ + ) && (o.init == 0 || !in_d(o.init)) + }) + .map(|o| o.name.as_str()) + .collect(); + for (k, o) in st.env_obs.iter().enumerate() { + if o.kind == OBS_STORE && !in_d(o.init) && !guarded.contains(o.name.as_str()) { + targets + .entry(o.ptr as u64 & 0xFFFF_FFFF_FFFF) + .or_default() + .push(k as i64); + } + } + let mut found = vec![0u32; st.env_obs.len()]; + let mut env_obj_cells = 0u32; + let mut cell = |what: &[i64], name: String, f: &mut std::fs::File| { + for &t in what { + if t < 0 { + env_obj_cells += 1; + let _ = writeln!(f, "#cell\tenv_object\t{name}\t\t"); + } else { + let o = &st.env_obs[t as usize]; + found[t as usize] += 1; + // The alias hint at the boundary (0 shared, 1 unique). + let alias = unsafe { (*(o.ptr as *const crate::StringHeader)).refcount }; + let _ = writeln!( + f, + "#cell\tenv_var\t{name}\t{}\t{}\talias={alias}", + o.name, o.init + ); + } + } + }; + let probe = |v: u64| -> Option<&Vec> { + let hi = v >> 48; + if hi == 0 || hi >= 0x7FF9 { + targets.get(&(v & 0xFFFF_FFFF_FFFF)) + } else { + None + } + }; + if !targets.is_empty() { + let mut storage = st.env_storage.clone(); + storage.sort_unstable(); + let log = st.births.as_slice(); + for (k, b) in log.iter().enumerate() { + if (b.init != 0 && in_d(b.init)) + || storage.binary_search_by_key(&b.addr, |r| r.0).is_ok() + { + continue; + } + let mut off = crate::gc::GC_HEADER_SIZE; + while off + 8 <= b.size as usize { + let word = b.addr + off; + if !st.w.contains(word as u64) { + let v = unsafe { *(word as *const u64) }; + if let Some(t) = probe(v) { + let t = t.clone(); + cell( + &t, + format!("b:{:x}#{}+{}", addr_of(b.init), rank[k], off), + f, + ); + } + } + off += 8; + } + } + for (k, &slot) in st.globals_log.iter().enumerate() { + let owner = st.slot_owner.get(k).copied().unwrap_or(0); + if (owner != 0 && in_d(owner)) || st.w.contains(slot as u64) { + continue; + } + let v = unsafe { *(slot as *const u64) }; + if let Some(t) = probe(v) { + let t = t.clone(); + cell(&t, slot_name(slot), f); + } + } + } + let _ = writeln!(f, "#env_object_cells\t{env_obj_cells}"); + // The guard: every other observation by an imaged init (the runtime's + // own, init 0, included: the IPC probe decides the process object). + let mut seen: std::collections::HashSet<(String, u32, &'static str)> = Default::default(); + let mut guard = |name: &str, + value: Option<&str>, + init: u32, + consumer: &'static str, + by: (u32, usize), + f: &mut std::fs::File| { + if init != 0 && in_d(init) { + return; + } + if seen.insert((name.to_owned(), init, consumer)) { + let _ = writeln!( + f, + "#guard\t{}\t{}\t{init}\t{:x}\t{consumer}\t{}\t{:x}", + tsv_text(name), + value.map_or("".to_owned(), tsv_text), + addr_of(init), + by.0, + by.1 + ); + } + }; + for (k, o) in st.env_obs.iter().enumerate() { + match o.kind { + OBS_OBSERVE => guard(&o.name, o.value.as_deref(), o.init, "observe", (0, 0), f), + OBS_HAS => guard(&o.name, o.value.as_deref(), o.init, "has", (0, 0), f), + OBS_STORE_UNSET => guard(&o.name, None, o.init, "store_unset", (0, 0), f), + OBS_STORE_READ => guard( + &o.name, + o.value.as_deref(), + o.init, + "store_read", + (o.by, o.by_pc), + f, + ), + OBS_STORE if found[k] == 0 && !in_d(o.init) && !guarded.contains(o.name.as_str()) => { + let _ = writeln!(f, "#cell_unstored\t{}\t{}", o.name, o.init); + } + _ => {} + } + } + for (init, name, value) in &st.env_reads { + if name.starts_with("PERRY_") || name.starts_with("RUST_") { + let _ = writeln!(f, "#knob\t{name}\t{init}"); + continue; + } + let explained = st + .env_obs + .iter() + .any(|o| o.init == *init && o.kind != OBS_OBJECT && o.name == *name); + if !explained { + guard(name, value.as_deref(), *init, "libc", (0, 0), f); + } + } + let takers = st.env_obs.iter().filter(|o| o.kind == OBS_OBJECT).count(); + let _ = writeln!( + f, + "#env\trule={}\tq7={}\tobs={}\tobject_takers={takers}\tcheck={}", + st.env_rule as u8, + st.q7 as u8, + st.env_obs.len(), + !st.skip.is_empty() as u8 + ); + for &(init, word, pc, write) in &st.check_fail { + let _ = writeln!( + f, + "#checkfail\tpoison\t{init}\t{:x}\t{}\t{pc:x}\t{}", + addr_of(init), + slot_name(word), + write as u8 + ); + } + if !st.skip.is_empty() { + for (id, i) in st.inits.iter().enumerate().skip(1) { + if i.in_d && !i.skipped && i.addr != ENTRY_BODY_ADDR && !i.entry_module { + let _ = writeln!(f, "#checkfail\tjoined_d\t{id}\t{:x}", i.addr); + } + } + } + // The next check run's order: per member of D whose body the check skips, + // the imaged inits entered inside it (directly or through members of D it + // entered), in entry order. + if st.skip.is_empty() { + if let Ok(mut d) = std::fs::File::create(format!("{}.dkids", st.out)) { + let skipped = |id: u32| { + let i = &st.inits[id as usize]; + id != 0 && i.in_d && !i.entry_module && i.addr != ENTRY_BODY_ADDR && i.addr != 0 + }; + for (id, i) in st.inits.iter().enumerate().skip(1) { + if in_d(id as u32) || i.addr == 0 || i.addr == ENTRY_BODY_ADDR { + continue; + } + let mut p = i.parent; + while skipped(p) { + let _ = writeln!(d, "{:x}\t{:x}\t{:x}", addr_of(p), i.addr, i.guard); + p = st.inits[p as usize].parent; + } + } + } + } + // The next check run's poison: the global slots members of D registered. + if st.skip.is_empty() { + if let Ok(mut d) = std::fs::File::create(format!("{}.dslots", st.out)) { + for (k, &slot) in st.globals_log.iter().enumerate() { + let owner = st.slot_owner.get(k).copied().unwrap_or(0); + // The entry module runs in the check run: its slots are not poisoned. + if owner != 0 && in_d(owner) && !st.inits[owner as usize].entry_module { + let _ = writeln!(d, "{slot:x}\t{owner}"); + } + } + } + } +} diff --git a/crates/perry-runtime/src/symbol.rs b/crates/perry-runtime/src/symbol.rs index ddaeb98977f..5e1c3aa3fff 100644 --- a/crates/perry-runtime/src/symbol.rs +++ b/crates/perry-runtime/src/symbol.rs @@ -756,6 +756,17 @@ pub(crate) fn store_class_static_symbol_root(class_id: u32, sym_key: usize, valu store_object_symbol_property_root(owner, sym_key, value_bits); } +/// Is the raw word `addr` a cell of the process immortal symbol space +/// ([`space::holds`]: range and cell stride, no read)? For the snapshot +/// census key-name rule, which names a persistent symbol by what it is +/// (kind and description), never by its cell address: cells are bump +/// allocated in creation order, so an address is a creation ordinal. +#[cfg(all(perry_gc_instruments, target_os = "linux", target_arch = "x86_64"))] +#[inline(always)] +pub(crate) fn in_symbol_space(addr: usize) -> bool { + space::holds(addr) +} + #[cfg(test)] mod wellknown_desc_tests { use super::*; diff --git a/crates/perry-runtime/src/timer.rs b/crates/perry-runtime/src/timer.rs index 133bb8e2744..f2a61178e6f 100644 --- a/crates/perry-runtime/src/timer.rs +++ b/crates/perry-runtime/src/timer.rs @@ -782,6 +782,7 @@ pub extern "C" fn js_set_timeout_callback(callback: i64, delay_ms: f64) -> i64 { #[no_mangle] pub extern "C" fn js_set_immediate_callback(callback: i64) -> i64 { + crate::effect!(host_write, timer); schedule_js_timer( callback, 0.0, @@ -964,6 +965,7 @@ pub unsafe extern "C" fn js_set_immediate_callback_args( args_ptr: *const f64, n_args: i32, ) -> i64 { + crate::effect!(host_write, timer); schedule_js_timer( callback, 0.0, diff --git a/crates/perry-runtime/src/timer/store.rs b/crates/perry-runtime/src/timer/store.rs index c1481344bb8..ad00ea169ee 100644 --- a/crates/perry-runtime/src/timer/store.rs +++ b/crates/perry-runtime/src/timer/store.rs @@ -411,6 +411,7 @@ impl AgentTimers { /// Insert a timers-phase entry. pub(super) fn insert_timer(&mut self, entry: Entry) -> usize { + crate::effect!(host_write, timer); debug_assert!(entry.class.is_timer()); let index = self.alloc(entry); self.heap_push(index); diff --git a/crates/perry-runtime/src/tty.rs b/crates/perry-runtime/src/tty.rs index caa1c8cddb2..e6a8671fb62 100644 --- a/crates/perry-runtime/src/tty.rs +++ b/crates/perry-runtime/src/tty.rs @@ -80,11 +80,13 @@ pub(crate) fn scan_tty_resize_callback_root_mut(visitor: &mut crate::gc::Runtime #[cfg(unix)] fn isatty_impl(fd: i32) -> bool { + crate::effect!(host_read, tty); unsafe { libc::isatty(fd) != 0 } } #[cfg(unix)] fn winsize_impl(fd: i32) -> Option<(i32, i32)> { + crate::effect!(host_read, tty); unsafe { let mut ws: libc::winsize = std::mem::zeroed(); if libc::ioctl(fd, libc::TIOCGWINSZ, &mut ws) == 0 && ws.ws_col > 0 && ws.ws_row > 0 { @@ -97,6 +99,7 @@ fn winsize_impl(fd: i32) -> Option<(i32, i32)> { #[cfg(windows)] fn isatty_impl(fd: i32) -> bool { + crate::effect!(host_read, tty); // A std handle counts as a TTY when `GetConsoleMode` succeeds on it — // the same probe `std::io::IsTerminal` uses (see `builtins/console.rs`) // and the same criterion libuv uses to classify a handle as `UV_TTY`. @@ -106,6 +109,7 @@ fn isatty_impl(fd: i32) -> bool { #[cfg(windows)] fn winsize_impl(fd: i32) -> Option<(i32, i32)> { + crate::effect!(host_read, tty); // GetConsoleScreenBufferInfo srWindow extents (right-left+1, // bottom-top+1) for the stdout/stderr console handle. crate::win_console::window_size(fd) @@ -113,11 +117,13 @@ fn winsize_impl(fd: i32) -> Option<(i32, i32)> { #[cfg(not(any(unix, windows)))] fn isatty_impl(_fd: i32) -> bool { + crate::effect!(host_read, tty); false } #[cfg(not(any(unix, windows)))] fn winsize_impl(_fd: i32) -> Option<(i32, i32)> { + crate::effect!(host_read, tty); None } diff --git a/crates/perry-stdlib/src/crypto/random.rs b/crates/perry-stdlib/src/crypto/random.rs index 2be13cc7843..11e035f9192 100644 --- a/crates/perry-stdlib/src/crypto/random.rs +++ b/crates/perry-stdlib/src/crypto/random.rs @@ -55,6 +55,7 @@ fn validate_random_bytes_size(size: f64) -> usize { pub extern "C" fn js_crypto_random_bytes_buffer( size: f64, ) -> *mut perry_runtime::buffer::BufferHeader { + perry_runtime::effect!(host_read, random); let size = validate_random_bytes_size(size); let (value, pin) = perry_runtime::buffer::bytes::new_bytes( perry_runtime::buffer::bytes::Brand::Buffer, @@ -83,6 +84,7 @@ pub extern "C" fn js_crypto_random_bytes_buffer( /// callback dispatch is deferred. #[no_mangle] pub unsafe extern "C" fn js_crypto_random_bytes_async(size: f64, callback_bits: f64) -> f64 { + perry_runtime::effect!(host_read, random); let buf = js_crypto_random_bytes_buffer(size); let value = if buf.is_null() { f64::from_bits(JSValue::undefined().bits()) @@ -113,6 +115,7 @@ pub unsafe extern "C" fn js_crypto_random_bytes_async(size: f64, callback_bits: /// crypto.randomBytes(size).toString('hex') -> string #[no_mangle] pub extern "C" fn js_crypto_random_bytes_hex(size: f64) -> *mut StringHeader { + perry_runtime::effect!(host_read, random); let size = validate_random_bytes_size(size); let mut bytes = vec![0u8; size]; @@ -130,6 +133,7 @@ pub extern "C" fn js_crypto_random_bytes_hex(size: f64) -> *mut StringHeader { /// `{ disableEntropyCache: true }`, which draws fresh OS entropy. #[no_mangle] pub unsafe extern "C" fn js_crypto_random_uuid(options_bits: f64) -> *mut StringHeader { + perry_runtime::effect!(host_read, random); let uuid = if validate_random_uuid_options(options_bits) { perry_uuid::v4_uncached() } else { @@ -149,6 +153,7 @@ pub unsafe extern "C" fn js_crypto_random_uuid(options_bits: f64) -> *mut String /// accepted for shape parity but does not change the generated value. #[no_mangle] pub extern "C" fn js_crypto_random_uuidv7() -> *mut StringHeader { + perry_runtime::effect!(host_read, random); let uuid = perry_uuid::v7(); let uuid_bytes = uuid.as_bytes(); js_string_from_bytes(uuid_bytes.as_ptr(), uuid_bytes.len() as u32) @@ -247,6 +252,7 @@ fn validate_random_int_arg(value: f64, arg_name: &str) -> i64 { /// Returns a uniformly distributed integer in `[min, max)`. #[no_mangle] pub extern "C" fn js_crypto_random_int(min_bits: f64, max_bits: f64) -> f64 { + perry_runtime::effect!(host_read, random); let min = validate_random_int_arg(min_bits, "min"); let max = validate_random_int_arg(max_bits, "max"); if max <= min { diff --git a/crates/perry-wpfacts/src/units/driver.rs b/crates/perry-wpfacts/src/units/driver.rs index d2c683abfa9..8fb9ed3c699 100644 --- a/crates/perry-wpfacts/src/units/driver.rs +++ b/crates/perry-wpfacts/src/units/driver.rs @@ -1145,6 +1145,11 @@ pub(crate) fn run(sorted: &[(&str, &Module)], cfg: &FactsConfig) -> ProgramFacts if cfg.unit_io { pf.unit_io = unit_io(sorted, &solves, &order, &per_unit, &input); } + // SNAP census (dump only, `PERRY_SNAP_FACTS=`): what each + // unit's code may write and read outside itself, at the fixpoint. + if let Some(path) = std::env::var_os("PERRY_SNAP_FACTS") { + snap_facts_dump(std::path::Path::new(&path), sorted, &solves, &per_unit); + } // The cells' writes at the fixpoint, each once per unit writing it // (each unit holds its own), for the tests to read their size. #[cfg(test)] @@ -1402,6 +1407,47 @@ pub(super) fn solve_level( }) } +/// SNAP census dump: per unit, its writes of other units' bindings, its +/// stores into other units' layouts, its shared-object effects (⊤ included), +/// its cells, and what it reads of them. One line per fact, tab-separated. +fn snap_facts_dump( + path: &std::path::Path, + sorted: &[(&str, &Module)], + solves: &[Solve], + per_unit: &[Option], +) { + use std::io::Write; + let Ok(f) = std::fs::File::create(path) else { + return; + }; + let mut f = std::io::BufWriter::new(f); + for (si, s) in solves.iter().enumerate() { + let Some(u) = per_unit[si].as_ref() else { + continue; + }; + let _ = writeln!(f, "unit\t{}\t{}", sorted[s.mi].0, s.key); + for ((p, n), _) in &u.writes_out { + let _ = writeln!(f, "write\t{p}\t{n}"); + } + for (d, k, _) in &u.stores_out { + let _ = writeln!(f, "store\t{d}\t{k}"); + } + for w in &u.shared_out { + let _ = writeln!(f, "shared_w\t{w:?}"); + } + for (k, _) in &u.cells_out { + let _ = writeln!(f, "cell_w\t{k}"); + } + for k in &u.cells_read { + let _ = writeln!(f, "cell_r\t{k}"); + } + for (d, _) in &u.layouts_read { + let _ = writeln!(f, "layout_r\t{d}"); + } + let _ = writeln!(f, "shared_r\t{:?}", u.shared_reads); + } +} + /// `FactsConfig::unit_io` (design §13.3): per unit, a digest of every input /// it reads (the contracts of the units it is ordered after, its closed /// functions' argument contracts, the store effects on and the census diff --git a/crates/perry/src/commands/compile/run_pipeline.rs b/crates/perry/src/commands/compile/run_pipeline.rs index 52e13bdb856..c21f5b07970 100644 --- a/crates/perry/src/commands/compile/run_pipeline.rs +++ b/crates/perry/src/commands/compile/run_pipeline.rs @@ -8098,6 +8098,30 @@ pub fn run_with_parse_cache( } } + // SNAP census compiles: the TLS anchors around the program objects (begin + // first, end after every other compiled object, before the pads and the + // runtime archive), so the program's TLS slots get pages of their own. + if std::env::var_os("PERRY_SNAP_CENSUS").is_some_and(|v| !v.is_empty()) { + for end in [false, true] { + let bytes = + perry_codegen::stubs::generate_snap_tls_anchor_object(end, target.as_deref())?; + let path = object_output_dir.join(if end { + "_perry_snap_tls_end.o" + } else { + "_perry_snap_tls_begin.o" + }); + fs::write(&path, &bytes)?; + obj_cleanup_paths.push(path.clone()); + if end { + obj_paths.push(path); + obj_fingerprints.push(None); + } else { + obj_paths.insert(0, path); + obj_fingerprints.insert(0, None); + } + } + } + match format { OutputFormat::Text => { if ctx.needs_stdlib { diff --git a/scripts/addr_class_allowlist.txt b/scripts/addr_class_allowlist.txt index 6019b6d54d8..65b47fb5ebb 100644 --- a/scripts/addr_class_allowlist.txt +++ b/scripts/addr_class_allowlist.txt @@ -198,3 +198,5 @@ crates/perry-runtime/src/array/indexing_read_cold.rs | as *const crate::gc::GcHe crates/perry-runtime/src/closure/alloc.rs | (*(raw.sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader))._reserved | existing fixed-capture birth debug assertion: raw is the fresh user pointer returned by closure_alloc_storage_no_collect in the same scope, with no collection before the assertion; the compact bound birth no longer has this assertion crates/perry-runtime/src/json_tape.rs | let header = (hdr as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; | is_lazy_array: hdr is a live GC user pointer by contract (callers hold the array being classified); reads its own header kind, the same discipline as the allocator walks. crates/perry-runtime/src/json_tape.rs | let header = (hdr as *mut u8).sub(crate::gc::GC_HEADER_SIZE) as *mut crate::gc::GcHeader; | lazy-to-ordinary flip: hdr is the live lazy array being materialized in place; the header kind is rewritten to GC_TYPE_ARRAY. +# SNAP census (census-only, cfg(perry_gc_instruments)): headers of births the census recorded at allocation, read while it holds collection. +crates/perry-runtime/src/snap_census/runner.rs | as *const crate::gc::GcHeader | Census digest (instr builds only): every address is a GC birth the census recorded at allocation (or the header before such a birth user pointer, or a runtime-leaked symbol header in the malloc range), read before the boundary while the census holds collection, so no object moved or died. diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index f51da4cd8b9..e2862c74af5 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -250,7 +250,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Re-audited 2026-10-23 against the pinned-source diffs and current collection phases. Raw header-address snapshot exists only in a synchronous full run_to_completion: step_mark_propagation records it after tracing, then block persistence and weak/ephemeron processing only mark or clear existing edges. Full finalization skips the relocating minor prelude. step_sweep consumes the snapshot before census reporting and before reclamation; budgeted cycles and minors skip its creation and consumption. No JS callback, object relocation, or mutator window occurs between the boundaries. The added full-sweep honest-pointer verifier only reads heap slots and allocator metadata; it neither calls JS nor relocates. Scanner registration and optional feature installation occur in gc_init before this window. Trigger-watermark admission, promotion-policy estimates and test-only throw sabotage do not change the synchronous mark-to-sweep sequence; the production throw check aborts rather than calling JS or unwinding. Native Rust allocations do not collect. Removing the retired PIC census row changes reporting only. Re-audited b7f3d56fed: policy.rs adds old_gen_growth_since_last_full_bytes, a read-only query of existing pressure and post-full baseline counters for idle-reducer admission. It does not call JS, collect or relocate, and neither snapshot boundary nor the synchronous mark-to-sweep sequence changes. Re-audited DPA-R: js_gc_init_with_layouts binds immutable image metadata before runtime initialization; the compatibility wrapper and linker keepalive anchor do not touch collection, either snapshot boundary, or the synchronous mark-to-sweep sequence. Re-audited WRT2b: the added image-root scan precedes mark propagation and snapshot creation; image cells are immortal MARKED|PINNED and do not enter the moving arena set. The honest-pointer verifier enumerates those immutable image addresses read-only. The new module declarations change no snapshot boundary, JS callback, relocation, or synchronous mark-to-sweep window. WRT3a re-audit: these source files equal e6e7cc419 and the parent; the synchronous census_pass1_if_armed to census_take_if_armed_at_full_sweep_start window remains inside run_to_completion, before reclamation, with no mutator resumption or relocation. Full runtime tests exercise this current source. Release-readiness re-audit 2026-10-28: census.rs only narrows the proc_pid_rusage reporter to macOS. Current gc_init installs roots and intrinsic layouts before collection; js_gc_init_with_layouts binds image metadata before gc_init. The unchanged run_to_completion, mark-completion capture, and pre-reclamation step_sweep consumption still provide the synchronous non-moving window. Re-audited release CI head: burst_boundary now arms the existing precise-poll flag and records the arena baseline. It does not collect or call JavaScript; collection remains deferred until a later precise poll outside the synchronous mark-to-sweep snapshot window. Re-audited #13405: cycle.rs routes birth-color changes through set_birth_extra_flags; it stores the same flags and refreshes the existing inline allocation bound. The refresh only reads existing TLS/arena state and changes the bound, without allocation, JS callbacks, collection or relocation. The synchronous mark-to-sweep custody window is unchanged. Re-audited 2026-10-29 against #13443 (trace.rs -> mark.rs, AtomicFinalize split out to cycle/atomic_finalize.rs, GC_STATS consolidated into gc/progress.rs), the #13457/#13463/#13479 GC slices, and the #13405/#13471/#13472 pacing changes. Both boundaries and both predicates are unchanged and still name the same condition: pass 1 needs minor.is_none() && !is_budgeted(), pass 2 needs full_trace (= minor.is_none()) && !is_budgeted(). The window now runs BlockPersistence -> AtomicFinalize -> the sweep entry before note_sweep_started, and each phase was read rather than assumed: BlockPersistence only marks; the only relocating subphase of AtomicFinalize, MinorPrelude, is entered solely from the minor branch of the ephemeron clear (cycle/weak_ephemerons.rs), while a full goes WeakProcessing -> DisableBarrier -> Sweep, so a full cycle never evacuates between the boundaries; no phase inside the window invokes JavaScript. The const-generic DIAGNOSTICS split of #13443 gates only trace_phase_* recording and the read-only honest-pointer and minor-unmarked verifiers. The new leaf-escape and in-place-promotion machinery (#13463/#13479, #13439) is reached from allocation, store and copying-minor promotion sites, never from this window. census.rs changed only its diagnostic gate predicate. The rest of the interval was read too: HeapChange::begin(Sweep) only clears the weak intern cache and advances the scope counter, the sweep-entry gap drain only traces seeds the barrier shaded, and finish_full_trace invokes perry-stdlib provider hooks that release native payloads owned by the stdlib rather than GC objects. The snapshot's own Vec is a native Rust allocation, which does not collect. Re-audited #13582 (x86-64 Linux GC walk): arena/inline.rs only puts a naked offering shim in front of js_inline_arena_slow_alloc, which passes its caller's pc/sp/fp to a Rust frame that stores a pointer to that origin in the thread-local walk-origin offer cell for the duration of the unchanged slow-path body and clears it on return. It is an allocation entry, not reached from inside the census window; it calls no JS, relocates nothing and changes neither snapshot boundary nor the synchronous mark-to-sweep sequence. Re-audited 2026-11-02 (RELPREP) against d70032de4..ab8fc9070. census.rs: the reporter's visit_object now accepts both object kinds (gc_object_type_pat!) and reads the meta word and slots through the A1 header accessors (meta_read!, object_slot_at, meta_form_alloc_bytes); it only reads heap words and counts, allocates nothing on the GC heap and changes neither census_pass1_if_armed nor census_take_if_armed_at_full_sweep_start. cycle.rs, mod.rs and allocation_pacing.rs changed formatting and a cfg-gated re-export of poll_armed only. policy.rs: on x86-64 Linux js_gc_loop_safepoint and js_gc_collect became naked entry shims that pass their caller's frame to the armed body as a WalkOrigin (EntryWalkOrigin::enter), and the leaf trigger arm (leaf_window_due) subtracts survivor bytes before comparing Eden with the Eden window. Both are collection ENTRIES and trigger arithmetic, reached before run_to_completion starts, never inside the synchronous mark-to-sweep window; no JS call, relocation or mutator resumption was added between the boundaries. Re-audited 2026-11-03 (RPX) against the diffs since ab8fc9070: census.rs swaps two sort_unstable calls for cold_sort::sort_by (same ordering of the same address Vec, no callback); barrier/mod.rs switches the rewrite-slot visitor to its shared (non-generic) form, same read-only walk; policy.rs, allocation_pacing.rs and verify_honest.rs gain #[optimize(size)] attributes and one rustfmt wrap. No JS callback, relocation or mutator window enters the synchronous mark-to-sweep sequence. Re-audited 2026-11-03 (handle-id reuse): gc/mod.rs gains thread_gc_initialized, a read-only query of the GC_INIT_DONE thread-local used by perry-ffi to tell a collecting thread from a worker; it is called from handle retirement, not from the census window, and calls no JS, allocates nothing on the GC heap and relocates nothing. full_trace.rs adds the pool liveness hook and release step consulted by an armed full trace and weakref.rs asks the pool liveness predicate for handle-valued weak keys, both outside the snapshot boundaries; the synchronous mark-to-sweep sequence is unchanged. Re-audited 2026-11-03 (RELPREP) policy.rs against 68a93a1478f0/f144e6d3d10d/2a03be364e4d/f134e5b860dd: a trigger check turned away inside a GcSuppressScope is now OWED (GC_FLAG_TRIGGER_OWED) and runs at the outermost scope's drop through gc_check_trigger, which returns while GC_FLAG_IN_ALLOC is held (a synchronous full collection holds it), so no check can collect between the pass-1 snapshot and the sweep; the finalization hunk only removes a post-GC delivery. Neither snapshot boundary nor the synchronous mark-to-sweep sequence changes.", + "why": "Re-audited 2026-10-23 against the pinned-source diffs and current collection phases. Raw header-address snapshot exists only in a synchronous full run_to_completion: step_mark_propagation records it after tracing, then block persistence and weak/ephemeron processing only mark or clear existing edges. Full finalization skips the relocating minor prelude. step_sweep consumes the snapshot before census reporting and before reclamation; budgeted cycles and minors skip its creation and consumption. No JS callback, object relocation, or mutator window occurs between the boundaries. The added full-sweep honest-pointer verifier only reads heap slots and allocator metadata; it neither calls JS nor relocates. Scanner registration and optional feature installation occur in gc_init before this window. Trigger-watermark admission, promotion-policy estimates and test-only throw sabotage do not change the synchronous mark-to-sweep sequence; the production throw check aborts rather than calling JS or unwinding. Native Rust allocations do not collect. Removing the retired PIC census row changes reporting only. Re-audited b7f3d56fed: policy.rs adds old_gen_growth_since_last_full_bytes, a read-only query of existing pressure and post-full baseline counters for idle-reducer admission. It does not call JS, collect or relocate, and neither snapshot boundary nor the synchronous mark-to-sweep sequence changes. Re-audited DPA-R: js_gc_init_with_layouts binds immutable image metadata before runtime initialization; the compatibility wrapper and linker keepalive anchor do not touch collection, either snapshot boundary, or the synchronous mark-to-sweep sequence. Re-audited WRT2b: the added image-root scan precedes mark propagation and snapshot creation; image cells are immortal MARKED|PINNED and do not enter the moving arena set. The honest-pointer verifier enumerates those immutable image addresses read-only. The new module declarations change no snapshot boundary, JS callback, relocation, or synchronous mark-to-sweep window. WRT3a re-audit: these source files equal e6e7cc419 and the parent; the synchronous census_pass1_if_armed to census_take_if_armed_at_full_sweep_start window remains inside run_to_completion, before reclamation, with no mutator resumption or relocation. Full runtime tests exercise this current source. Release-readiness re-audit 2026-10-28: census.rs only narrows the proc_pid_rusage reporter to macOS. Current gc_init installs roots and intrinsic layouts before collection; js_gc_init_with_layouts binds image metadata before gc_init. The unchanged run_to_completion, mark-completion capture, and pre-reclamation step_sweep consumption still provide the synchronous non-moving window. Re-audited release CI head: burst_boundary now arms the existing precise-poll flag and records the arena baseline. It does not collect or call JavaScript; collection remains deferred until a later precise poll outside the synchronous mark-to-sweep snapshot window. Re-audited #13405: cycle.rs routes birth-color changes through set_birth_extra_flags; it stores the same flags and refreshes the existing inline allocation bound. The refresh only reads existing TLS/arena state and changes the bound, without allocation, JS callbacks, collection or relocation. The synchronous mark-to-sweep custody window is unchanged. Re-audited 2026-10-29 against #13443 (trace.rs -> mark.rs, AtomicFinalize split out to cycle/atomic_finalize.rs, GC_STATS consolidated into gc/progress.rs), the #13457/#13463/#13479 GC slices, and the #13405/#13471/#13472 pacing changes. Both boundaries and both predicates are unchanged and still name the same condition: pass 1 needs minor.is_none() && !is_budgeted(), pass 2 needs full_trace (= minor.is_none()) && !is_budgeted(). The window now runs BlockPersistence -> AtomicFinalize -> the sweep entry before note_sweep_started, and each phase was read rather than assumed: BlockPersistence only marks; the only relocating subphase of AtomicFinalize, MinorPrelude, is entered solely from the minor branch of the ephemeron clear (cycle/weak_ephemerons.rs), while a full goes WeakProcessing -> DisableBarrier -> Sweep, so a full cycle never evacuates between the boundaries; no phase inside the window invokes JavaScript. The const-generic DIAGNOSTICS split of #13443 gates only trace_phase_* recording and the read-only honest-pointer and minor-unmarked verifiers. The new leaf-escape and in-place-promotion machinery (#13463/#13479, #13439) is reached from allocation, store and copying-minor promotion sites, never from this window. census.rs changed only its diagnostic gate predicate. The rest of the interval was read too: HeapChange::begin(Sweep) only clears the weak intern cache and advances the scope counter, the sweep-entry gap drain only traces seeds the barrier shaded, and finish_full_trace invokes perry-stdlib provider hooks that release native payloads owned by the stdlib rather than GC objects. The snapshot's own Vec is a native Rust allocation, which does not collect. Re-audited #13582 (x86-64 Linux GC walk): arena/inline.rs only puts a naked offering shim in front of js_inline_arena_slow_alloc, which passes its caller's pc/sp/fp to a Rust frame that stores a pointer to that origin in the thread-local walk-origin offer cell for the duration of the unchanged slow-path body and clears it on return. It is an allocation entry, not reached from inside the census window; it calls no JS, relocates nothing and changes neither snapshot boundary nor the synchronous mark-to-sweep sequence. Re-audited 2026-11-02 (RELPREP) against d70032de4..ab8fc9070. census.rs: the reporter's visit_object now accepts both object kinds (gc_object_type_pat!) and reads the meta word and slots through the A1 header accessors (meta_read!, object_slot_at, meta_form_alloc_bytes); it only reads heap words and counts, allocates nothing on the GC heap and changes neither census_pass1_if_armed nor census_take_if_armed_at_full_sweep_start. cycle.rs, mod.rs and allocation_pacing.rs changed formatting and a cfg-gated re-export of poll_armed only. policy.rs: on x86-64 Linux js_gc_loop_safepoint and js_gc_collect became naked entry shims that pass their caller's frame to the armed body as a WalkOrigin (EntryWalkOrigin::enter), and the leaf trigger arm (leaf_window_due) subtracts survivor bytes before comparing Eden with the Eden window. Both are collection ENTRIES and trigger arithmetic, reached before run_to_completion starts, never inside the synchronous mark-to-sweep window; no JS call, relocation or mutator resumption was added between the boundaries. Re-audited 2026-11-03 (RPX) against the diffs since ab8fc9070: census.rs swaps two sort_unstable calls for cold_sort::sort_by (same ordering of the same address Vec, no callback); barrier/mod.rs switches the rewrite-slot visitor to its shared (non-generic) form, same read-only walk; policy.rs, allocation_pacing.rs and verify_honest.rs gain #[optimize(size)] attributes and one rustfmt wrap. No JS callback, relocation or mutator window enters the synchronous mark-to-sweep sequence. Re-audited 2026-11-03 (handle-id reuse): gc/mod.rs gains thread_gc_initialized, a read-only query of the GC_INIT_DONE thread-local used by perry-ffi to tell a collecting thread from a worker; it is called from handle retirement, not from the census window, and calls no JS, allocates nothing on the GC heap and relocates nothing. full_trace.rs adds the pool liveness hook and release step consulted by an armed full trace and weakref.rs asks the pool liveness predicate for handle-valued weak keys, both outside the snapshot boundaries; the synchronous mark-to-sweep sequence is unchanged. Re-audited 2026-11-03 (RELPREP) policy.rs against 68a93a1478f0/f144e6d3d10d/2a03be364e4d/f134e5b860dd: a trigger check turned away inside a GcSuppressScope is now OWED (GC_FLAG_TRIGGER_OWED) and runs at the outermost scope's drop through gc_check_trigger, which returns while GC_FLAG_IN_ALLOC is held (a synchronous full collection holds it), so no check can collect between the pass-1 snapshot and the sweep; the finalization hunk only removes a post-GC delivery. Neither snapshot boundary nor the synchronous mark-to-sweep sequence changes. Re-audited 2026-11-03 for the SNAP census hooks (instruments builds only; production calls are empty inline stubs): gc/census.rs Census accounting calls snap_census::census_live(header, type, size), which only counts the header against the census's own birth list in a malloc map (no mark, no move, no GC allocation, no JS); arena/inline.rs allocation_limit returns 0 while snap_census closes the inline bump, which routes mutator allocations to the slow path, outside this window; gc/mod.rs re-exports roots::for_each_global_root_slot under the instruments cfg, no code in the window. Re-audited 2026-11-05 (SNAP merge with main) policy.rs: the SNAP census hooks add an early return in gc_unsuppress while snap_census holds collection (a flag clear, no collection) and an early return plus explicit_gc_done bookkeeping in js_gc_collect_body (the manual-collect entry, reached before run_to_completion); all instruments-only stubs in production, no JS, relocation or mutator resumption inside the synchronous mark-to-sweep window, which main left unchanged.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -265,12 +265,12 @@ "function": "run_to_completion" }, "sources": { - "crates/perry-runtime/src/gc/census.rs": "6458bd6c728fec0981ec93fdd8a0669a410df17af9d618ea981342008e95d97a", + "crates/perry-runtime/src/gc/census.rs": "865d2ce07cccb76e77bb3c55fc4a562cc0623888c2d57ebbe456a829e287e09b", "crates/perry-runtime/src/gc/cycle.rs": "4a4b57e793afbb28821f3bce1444cfb06a41728c57a680825a4223b3e25f35a6", "crates/perry-runtime/src/gc/barrier/mod.rs": "47ca32e42119473fcb42b1f5b9ba7d1b51c602fe6682d1ef8cf520ed11b57d40", - "crates/perry-runtime/src/arena/inline.rs": "c4fa63b0fbd3aa00d47a3e856c34c4050ef96edc8c50444158d2cfc817d6f863", - "crates/perry-runtime/src/gc/mod.rs": "03fe4120680ac9a2e91f4e22f35a6838f3a012aa4160a114c236720e6f185a2b", - "crates/perry-runtime/src/gc/policy.rs": "c70c2f71775477b5d8bfbf4ee795e2b1924d072a199bfc914d36050a25205868", + "crates/perry-runtime/src/arena/inline.rs": "437911ec0c0386b6d01b1226e073b58bb5df63659d71cdacc70825649caf03cc", + "crates/perry-runtime/src/gc/mod.rs": "af972a03e965e8709fc82e44bf177033c644e898fa58e28213ea11e0e4ea7db8", + "crates/perry-runtime/src/gc/policy.rs": "36062973623b8e6f0ace9c48fbd4925cdbb51c4733c8d95e088c6f747330ecb6", "crates/perry-runtime/src/gc/progress.rs": "d2cf96a624dff35e405ac15fa37fe6ba31e7e158668581428fb3765f2c7160c3", "crates/perry-runtime/src/gc/allocation_pacing.rs": "31567ecfd0560763dfe306503c90454c979851c5875465f50b0ed0eae4730a31", "crates/perry-runtime/src/gc/ephemeron.rs": "fd477a0729117b514c7a41fd0c48ba430afaf40f26f9c85dab6890ef1171f437",