Realm production A2a: Set bucket contract, array element storage, realm-diff gate

changelog.d/13793-realm-diff-gate.md

@@ -0,0 +1 @@
+The realm image has one Set index sizing, shared by the emitter and the runtime threshold crossing, and static array elements have one home in the realm manifest (array storage). A realm-diff gate script compares the realm dumper with checked-in references, in full mode and with a test-only restricted realm reach.

crates/perry-abi/src/set_cells.rs

@@ -78,6 +78,37 @@ impl OwnedSetIndex {
         Self::with_capacity((size * 2).max(16).next_power_of_two())
     }
 
+    /// The index a Set has after these insertions, in order: none while
+    /// the Set is small enough to scan, otherwise the index born when the
+    /// size first exceeds `SMALL_SET_SCAN_MAX` (sized for that size) followed
+    /// by the remaining inserts. Each item is `(hash, element slot)`. The
+    /// runtimes threshold crossing and the compilers image emission both
+    /// build through this, so an image Set has the buckets a heap birth of
+    /// the same entries has.
+    pub fn from_insertions(insertions: impl IntoIterator<Item = (u32, u32)>) -> Option<Self> {
+        const FIRST: usize = SMALL_SET_SCAN_MAX as usize + 1;
+        let mut pending = [(0u32, 0u32); FIRST];
+        let mut count = 0;
+        let mut index = None::<Self>;
+        for (hash, slot) in insertions {
+            match &mut index {
+                Some(index) => index.insert_hash(hash, slot),
+                None => {
+                    pending[count] = (hash, slot);
+                    count += 1;
+                    if count == FIRST {
+                        let mut born = Self::new(FIRST);
+                        for &(hash, slot) in &pending {
+                            born.insert_hash(hash, slot);
+                        }
+                        index = Some(born);
+                    }
+                }
+            }
+        }
+        index
+    }
+
     pub fn buckets(&self) -> &[Bucket] {
         // Native instances own a Box slice; image instances point at a
         // program-lifetime readonly slice with this same declared extent.

crates/perry-codegen/src/codegen/realm_image/arrays.rs

@@ -5,7 +5,7 @@ use super::atoms::RealmAtoms;
 use super::objects::ObjectFacts;
 use super::properties::emit_values;
 use super::*;
-use perry_hir::realm_manifest::{RealmDeclaration, RealmProperty, RealmValue};
+use perry_hir::realm_manifest::{RealmDeclaration, RealmProperty, RealmStorageFacts, RealmValue};
 
 pub(crate) struct NamedArrayFacts<'a> {
     pub(crate) object: &'a ObjectFacts,
@@ -43,21 +43,19 @@ impl<'a> ArrayParts<'a> {
         // Integrity restrictions are represented by the receiver header, as
         // on heap arrays; do not materialize redundant element descriptors.
         let frozen = header.reserved & perry_abi::OBJ_FLAG_FROZEN != 0;
-        let sealed = header.reserved & perry_abi::OBJ_FLAG_SEALED != 0;
         if length.attributes != u8::from(!frozen) || length.get.is_some() || length.set.is_some() {
             return Err(format!(
                 "realm array {} needs length descriptor storage",
                 cell.name
             ));
         }
-        let default_attrs = if frozen {
-            2
-        } else if sealed {
-            3
-        } else {
-            7
+        // The elements have one home: the cell's array storage. An index
+        // key among the holder's properties is a second home and an error.
+        let elements = match &cell.storage {
+            Some(RealmStorageFacts::Array { elements }) => elements.iter().collect::<Vec<_>>(),
+            None => Vec::new(),
+            Some(_) => return Err(format!("realm array {} has non-array storage", cell.name)),
         };
-        let mut indexed = std::collections::BTreeMap::new();
         let mut named = Vec::new();
         for property in &cell.properties {
             if !property.symbol && property.key == "length" {
@@ -67,36 +65,22 @@ impl<'a> ArrayParts<'a> {
                 .then(|| property.key.parse::<u32>().ok())
                 .flatten()
                 .filter(|i| *i != u32::MAX && i.to_string() == property.key);
-            if let Some(index) = index {
-                let value = property
-                    .value
-                    .as_ref()
-                    .filter(|_| property.get.is_none() && property.set.is_none())
-                    .ok_or_else(|| {
-                        format!("realm array {} needs an indexed accessor holder", cell.name)
-                    })?;
-                if property.attributes != default_attrs || index >= n as u32 {
-                    return Err(format!(
-                        "realm array {} needs indexed descriptor storage",
-                        cell.name
-                    ));
-                }
-                if indexed.insert(index, value).is_some() {
-                    return Err("duplicate realm array index".into());
-                }
-            } else {
-                named.push(property.clone());
+            if index.is_some() {
+                return Err(format!(
+                    "realm array {} declares element {} as a property, not storage",
+                    cell.name, property.key
+                ));
             }
+            named.push(property.clone());
         }
         // Current source-owned static arrays are dense. A future sparse birth
         // must describe its holes explicitly, not silently emit undefined.
-        if indexed.len() != n as usize {
+        if elements.len() != n as usize {
             return Err(format!(
                 "realm array {} lacks dense element birth facts",
                 cell.name
             ));
         }
-        let elements = indexed.into_values().collect();
         Ok(Self {
             length: n as u32,
             elements,

crates/perry-codegen/src/codegen/realm_image/sets.rs

@@ -19,20 +19,19 @@ fn index(entries: &[RealmValue]) -> Result<Option<perry_abi::OwnedSetIndex>, Str
             ))
         })
         .collect::<Result<Vec<_>, String>>()?;
-    let first = perry_abi::SMALL_SET_SCAN_MAX as usize + 1;
-    if entries.len() < first {
-        return Ok(None);
-    }
-    let mut index = perry_abi::OwnedSetIndex::new(first);
-    // Match the birth's insertion history, including the shared builder's
-    // rehash order; sizing once from final length gives different buckets.
-    for (slot, hash) in hashes.into_iter().enumerate() {
-        index.insert_hash(
-            hash,
-            u32::try_from(slot).map_err(|_| "realm Set too large")?,
-        );
-    }
-    Ok(Some(index))
+    // The runtime's threshold crossing builds a heap Set's index through the
+    // same function, so the image buckets are the ones a birth would have.
+    let insertions = hashes
+        .into_iter()
+        .enumerate()
+        .map(|(slot, hash)| {
+            Ok((
+                hash,
+                u32::try_from(slot).map_err(|_| "realm Set too large")?,
+            ))
+        })
+        .collect::<Result<Vec<_>, String>>()?;
+    Ok(perry_abi::OwnedSetIndex::from_insertions(insertions))
 }
 
 pub(crate) fn emit_set(
@@ -201,7 +200,11 @@ mod tests {
             } else {
                 "e-p:64:64-i64:64"
             });
-            for cell in plan.cells.values().filter(|c| c.storage.is_some()) {
+            for cell in plan
+                .cells
+                .values()
+                .filter(|c| matches!(c.storage, Some(RealmStorageFacts::Set { .. })))
+            {
                 let name = crate::realm_plan::cell_symbol(&cell.name);
                 let ty = parsed
                     .get_global(&name)

crates/perry-hir/src/realm_reach.rs

@@ -59,6 +59,17 @@ impl RealmReach {
         }
     }
 
+    /// Test-only (the realm-diff gate's `PERRY_REALM_REACH_NAMES` input):
+    /// R becomes exactly `names`, with every top reason dropped, so a program
+    /// that reflects over `globalThis` (the realm dumper) can be built with a
+    /// partial image. A top program's own names are not kept: a scan closes a
+    /// top R to the whole catalog, so they are no longer distinguishable.
+    /// [`close`](Self::close) then applies the birth relation as usual.
+    pub fn restrict_to(&mut self, names: impl IntoIterator<Item = String>) {
+        self.top.clear();
+        self.names = names.into_iter().collect();
+    }
+
     pub fn close(&mut self, entries: &[RealmEntry]) {
         if self.is_full() {
             self.names

crates/perry-hir/src/realm_reach_tests.rs

@@ -181,6 +181,26 @@ fn local_names_and_types_are_not_global_references() {
     assert!(!r.names.contains("Set"));
 }
 
+#[test]
+fn restricting_drops_top_and_closes_the_given_names() {
+    // `with` makes R top, and the scan closes it to the whole catalog.
+    let mut r = reach("new Map(); with ({}) { x; }");
+    assert!(r.is_full(), "{r:?}");
+    assert_eq!(r.names.len(), CATALOG.len());
+    r.restrict_to(["Buffer".to_string()]);
+    assert!(!r.is_full());
+    r.close(CATALOG);
+    // The given names and their births; nothing else.
+    let expected = ["Buffer", "Uint8Array", "%TypedArray%", "ArrayBuffer"];
+    assert_eq!(
+        r.names,
+        expected
+            .iter()
+            .map(|n| n.to_string())
+            .collect::<BTreeSet<_>>()
+    );
+}
+
 #[test]
 fn constant_code_is_program_code_and_birth_closure_is_transitive() {
     let r = reach("Function('return Buffer')(); eval('Map');");

crates/perry-runtime-manifest/src/lib.rs

@@ -248,4 +248,6 @@ pub mod realm_json;
 pub use perry_abi as abi;
 pub mod realm_layouts;
 pub mod realm_source;
+#[cfg(test)]
+mod realm_owners;
 mod realm_storage;

crates/perry-runtime-manifest/src/realm.rs

@@ -108,14 +108,23 @@ pub struct ConstructorFacts {
 #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
 #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
 pub enum RealmStorageFacts {
-    Set { entries: Vec<RealmValue> },
+    Set {
+        entries: Vec<RealmValue>,
+    },
+    /// The dense elements of a static array birth, in index order. Reflection
+    /// reads them as index properties; the array holder carries only
+    /// and its named properties, so the elements have one home.
+    Array {
+        elements: Vec<RealmValue>,
+    },
     RuntimeLayout(RuntimeLayoutFacts),
 }
 
 impl RealmStorageFacts {
     pub fn values(&self) -> &[RealmValue] {
         match self {
             Self::Set { entries } => entries,
+            Self::Array { elements } => elements,
             Self::RuntimeLayout(_) => &[],
         }
     }

crates/perry-runtime-manifest/src/realm_array.rs

@@ -328,30 +328,13 @@ pub(super) fn realm(expression: &str, entry: &[Token<'_>], source: &str) -> Resu
             ))
         }
     };
-    // Integrity makes each own property non-configurable, and freezing
-    // makes it non-writable; length is never enumerable or configurable.
-    let element_attrs = match integrity {
-        Integrity::Extensible | Integrity::NonExtensible => 7,
-        Integrity::Sealed => 3,
-        Integrity::Frozen => 2,
-    };
+    // Integrity lives in the header (the elements' attributes follow from
+    // it); length is never enumerable or configurable.
     let length_attrs = if integrity == Integrity::Frozen { 0 } else { 1 };
-    let mut properties = items
-        .iter()
-        .enumerate()
-        .map(|(i, item)| {
-            object([
-                ("key", string(i.to_string())),
-                ("symbol", Value::Bool(false)),
-                ("attributes", number(element_attrs)),
-                (
-                    "value",
-                    object([("kind", string("string")), ("value", string(item.clone()))]),
-                ),
-            ])
-        })
-        .collect::<Vec<_>>();
-    properties.push(object([
+    // The elements are storage, the one home the array emitter reads. The
+    // holder's key order lists its properties; the index keys come first by
+    // the array's own-keys rule and are not repeated here.
+    let properties = vec![object([
         ("key", string("length")),
         ("symbol", Value::Bool(false)),
         ("attributes", number(length_attrs)),
@@ -365,15 +348,23 @@ pub(super) fn realm(expression: &str, entry: &[Token<'_>], source: &str) -> Resu
                 ),
             ]),
         ),
-    ]));
-    properties.sort_by_key(|p| p.get("key").and_then(Value::string).unwrap().to_owned());
-    let mut keys = (0..items.len())
-        .map(|i| string(i.to_string()))
-        .collect::<Vec<_>>();
-    keys.push(string("length"));
+    ])];
     let mut map = base("namespace", elements);
     map.insert("properties".into(), Value::Array(properties));
-    map.insert("own_key_order".into(), Value::Array(keys));
+    map.insert("own_key_order".into(), Value::Array(vec![string("length")]));
+    if !items.is_empty() {
+        let elements = items
+            .iter()
+            .map(|item| object([("kind", string("string")), ("value", string(item.clone()))]))
+            .collect();
+        map.insert(
+            "storage".into(),
+            object([
+                ("kind", string("array")),
+                ("elements", Value::Array(elements)),
+            ]),
+        );
+    }
     Ok(Value::Object(map))
 }
 

crates/perry-runtime-manifest/src/realm_birth.rs

@@ -811,7 +811,29 @@ mod tests {
         .unwrap();
         assert!(d.get("prototype").is_none());
         assert_eq!(d.get("births").unwrap().encode(), "[]");
-        assert_eq!(d.get("header").unwrap().get("reserved"), Some(&number(64)));
+        // A null [[Prototype]] is the shape's identity (#13666): the birth
+        // records it, and the header word is the one an ordinary birth of
+        // the same members gets. No header bit stands for it.
+        let header = d.get("header").unwrap();
+        assert_eq!(
+            header.get("@birth").unwrap().get("null_prototype"),
+            Some(&Value::Bool(true))
+        );
+        let ordinary = birth(
+            &tokens("crate::object::js_object_alloc(0,0)").unwrap(),
+            &tokens(r#""node:module._cache", kind=namespace, members={}"#).unwrap(),
+        )
+        .unwrap();
+        assert_eq!(
+            ordinary.get("prototype").and_then(Value::string),
+            Some("Object.prototype")
+        );
+        let ordinary = ordinary.get("header").unwrap();
+        assert_eq!(
+            ordinary.get("@birth").unwrap().get("null_prototype"),
+            Some(&Value::Bool(false))
+        );
+        assert_eq!(header.get("reserved"), ordinary.get("reserved"));
         let d = birth(
             &tokens("js_object_alloc(0,limit)").unwrap(),
             &tokens("process.env, kind=exotic, members={}").unwrap(),