perf(json): right-size tape object spill buffers

Author: Ralph Küpper

changelog.d/8016-json-tape-exact-spill.md

@@ -0,0 +1,21 @@
+### perf(json): right-size tape-materialized object spill buffers (#7267)
+
+JSON tape materialization previously allocated every object at the two-slot
+inline floor, then let the first overflowing property create a general-purpose
+array with 16 slots of growth headroom. Five-field records therefore carried a
+16-slot side allocation even though the tape already knew their final width.
+
+Both the recursive lazy materializer and the iterative deep-input materializer
+now reserve an exact-width spill buffer before storing fields. The recursive
+path counts only the current object's keys by hopping across nested container
+links; the iterative path reuses the key count it has already collected. The
+primary object deliberately remains at `INLINE_SLOT_FLOOR`: sizing its inline
+allocation to every key was benchmarked in #7267 and regressed the named field
+access workload.
+
+On `benchmarks/json_polyglot/bench_field_access.ts`, eight interleaved
+`perry-dev` runs with `PERRY_NO_AUTO_OPTIMIZE=1` reduced median time from
+1266.5 ms to 1138.5 ms (-10.1%) and peak RSS from 309.3 MiB to 294.3 MiB
+(-4.8%), with identical checksums. A direct-parser control was unchanged
+(934.5 ms vs 935.5 ms median), isolating the improvement to tape-backed object
+materialization.

crates/perry-runtime/src/array/alloc.rs

@@ -140,6 +140,34 @@ pub extern "C" fn js_array_alloc_with_length(capacity: u32) -> *mut ArrayHeader
     ptr
 }
 
+/// Allocate an exact-sized holey array for runtime-owned side storage.
+///
+/// Unlike [`js_array_alloc_with_length`], this does not add
+/// [`MIN_ARRAY_CAPACITY`] growth headroom. Callers must know their final width;
+/// the spill buffer used by JSON tape materialization does, and padding every
+/// parsed object to 16 side slots would otherwise dominate the object itself.
+pub(crate) fn js_array_alloc_with_length_exact(capacity: u32) -> *mut ArrayHeader {
+    let ptr = arena_alloc_gc(
+        array_byte_size(capacity as usize),
+        8,
+        crate::gc::GC_TYPE_ARRAY,
+    ) as *mut ArrayHeader;
+
+    unsafe {
+        (*ptr).length = capacity;
+        (*ptr).capacity = capacity;
+        let elements_ptr = (ptr as *mut u8).add(std::mem::size_of::<ArrayHeader>()) as *mut u64;
+        for i in 0..capacity as usize {
+            // GC_STORE_AUDIT(POINTER_FREE): TAG_HOLE is a non-pointer sentinel for fresh array slots.
+            std::ptr::write(elements_ptr.add(i), crate::value::TAG_HOLE);
+        }
+        clear_array_numeric_layout(ptr);
+        crate::gc::layout_init_pointer_free(ptr as *mut u8);
+    }
+
+    ptr
+}
+
 /// Runtime path for `Array(value)` / `new Array(value)`.
 ///
 /// A single Number argument is interpreted as an array length and must be a

crates/perry-runtime/src/array/mod.rs

@@ -34,7 +34,9 @@ mod subclass_tests;
 #[cfg(test)]
 mod tests;
 
-pub(crate) use self::alloc::{array_length_range_error, js_array_alloc_pointer_elements};
+pub(crate) use self::alloc::{
+    array_length_range_error, js_array_alloc_pointer_elements, js_array_alloc_with_length_exact,
+};
 pub use self::alloc::{
     js_array_alloc, js_array_alloc_literal, js_array_alloc_with_length,
     js_array_alloc_with_length_longlived, js_array_constructor_single, js_array_create,

crates/perry-runtime/src/json_tape.rs

@@ -669,9 +669,12 @@ unsafe fn materialize_object(
     idx: &mut usize,
     end_idx: usize,
 ) -> JSValue {
+    let field_count = count_object_fields(source, *idx, end_idx);
     let obj = crate::object::js_object_alloc(0, 0);
     let obj_handle = scope.root_raw_mut_ptr(obj);
     json_tape_safepoint(JsonTapeSafepoint::MaterializeObjectRooted, obj as usize);
+    let obj = obj_handle.get_raw_mut_ptr::<crate::object::ObjectHeader>();
+    crate::object::reserve_object_spill(obj as usize, field_count);
     while *idx < end_idx {
         let Some(key_entry) = source.entry(*idx) else {
             break;
@@ -699,6 +702,32 @@ unsafe fn materialize_object(
     JSValue::object_ptr(obj as *mut u8)
 }
 
+/// Count only this object's keys, hopping over nested values through their
+/// matching-container links. The walk allocates nothing, so it is safe for a
+/// lazy source whose backing pointers may be refreshed through a GC handle.
+unsafe fn count_object_fields(source: &TapeSource<'_, '_>, mut idx: usize, end_idx: usize) -> u32 {
+    let mut count = 0u32;
+    while idx < end_idx {
+        let Some(key) = source.entry(idx) else {
+            break;
+        };
+        if key.kind != KIND_KEY {
+            break;
+        }
+        count = count.saturating_add(1);
+        idx += 1;
+        let Some(value) = source.entry(idx) else {
+            break;
+        };
+        if value.kind == KIND_OBJ_START || value.kind == KIND_ARR_START {
+            idx = value.link as usize + 1;
+        } else {
+            idx += 1;
+        }
+    }
+    count
+}
+
 unsafe fn materialize_array(
     source: &TapeSource<'_, '_>,
     scope: &crate::gc::RuntimeHandleScope,

crates/perry-runtime/src/json_tape/iterative.rs

@@ -40,7 +40,9 @@ unsafe fn finish_frame(frame: BuildFrame) -> Option<JSValue> {
             if keys.len() != values.len() {
                 return None;
             }
+            let field_count = u32::try_from(keys.len()).ok()?;
             let object = crate::object::js_object_alloc(0, 0);
+            crate::object::reserve_object_spill(object as usize, field_count);
             for (key, value) in keys.into_iter().zip(values) {
                 crate::object::js_object_set_field_by_name(
                     object,

crates/perry-runtime/src/json_tape_tests.rs

@@ -115,6 +115,47 @@ fn tape_top_level_scalars() {
     assert_eq!(build_tape(b"null").unwrap().entries.len(), 1);
 }
 
+#[test]
+fn recursive_materializer_reserves_exact_spill_per_object_depth() {
+    let input = br#"{"a":1,"nested":{"n0":0,"n1":1,"n2":2,"n3":3,"n4":4},"b":2}"#;
+    let tape = build_tape(input).expect("valid tape");
+    let nested_key = crate::string::js_string_from_bytes(b"nested".as_ptr(), 6);
+
+    crate::gc::gc_suppress();
+    let value = unsafe { materialize(&tape, input) };
+    let object = (value.bits() & crate::value::POINTER_MASK) as *const crate::ObjectHeader;
+    let nested = crate::object::js_object_get_field_by_name(object, nested_key);
+    let nested = (nested.bits() & crate::value::POINTER_MASK) as *const crate::ObjectHeader;
+
+    unsafe {
+        assert_eq!(
+            (*object).field_count,
+            crate::object::INLINE_SLOT_FLOOR as u32,
+            "known width must not enlarge the primary object"
+        );
+        let spill =
+            crate::object::test_spill_buffer_addr(object as usize) as *const crate::ArrayHeader;
+        assert!(!spill.is_null());
+        assert_eq!((*spill).capacity, 3, "count only outer-object keys");
+        assert_eq!((*spill).length, 3);
+
+        assert_eq!(
+            (*nested).field_count,
+            crate::object::INLINE_SLOT_FLOOR as u32
+        );
+        let nested_spill =
+            crate::object::test_spill_buffer_addr(nested as usize) as *const crate::ArrayHeader;
+        assert!(!nested_spill.is_null());
+        assert_eq!(
+            (*nested_spill).capacity,
+            5,
+            "reserve the nested width exactly"
+        );
+        assert_eq!((*nested_spill).length, 5);
+    }
+    crate::gc::gc_unsuppress();
+}
+
 #[test]
 fn iterative_materializer_preserves_nested_objects_arrays_and_duplicate_keys() {
     let input = br#"{"a":[1,true,"x"],"a":{"b":2}}"#;
@@ -136,6 +177,28 @@ fn iterative_materializer_preserves_nested_objects_arrays_and_duplicate_keys() {
     crate::json::parse_root_restore(saved_roots);
 }
 
+#[test]
+fn iterative_materializer_reserves_exact_spill_without_widening_object() {
+    let input = br#"{"f0":0,"f1":1,"f2":2,"f3":3,"f4":4}"#;
+    let tape = build_tape(input).expect("valid tape");
+
+    crate::gc::gc_suppress();
+    let value = unsafe { materialize_iterative(&tape.entries, input) }.expect("materializes");
+    let object = (value.bits() & crate::value::POINTER_MASK) as *const crate::ObjectHeader;
+    unsafe {
+        assert_eq!(
+            (*object).field_count,
+            crate::object::INLINE_SLOT_FLOOR as u32
+        );
+        let spill =
+            crate::object::test_spill_buffer_addr(object as usize) as *const crate::ArrayHeader;
+        assert!(!spill.is_null());
+        assert_eq!((*spill).capacity, 5);
+        assert_eq!((*spill).length, 5);
+    }
+    crate::gc::gc_unsuppress();
+}
+
 /// `TapeEntry` is 12 bytes (u32 + u8 + padding + u32). Keeping
 /// this compact matters for tape-size parity with parse output:
 /// a 1 MB JSON blob with ~20k tokens should build a ~240 KB tape,

crates/perry-runtime/src/object/mod.rs

@@ -130,6 +130,7 @@ mod regex_proto_thunks;
 mod spill;
 pub(crate) use spill::{
     learned_inline_field_count, learned_inline_fields_hot_addr, overflow_get, overflow_set,
+    reserve_object_spill,
 };
 #[cfg(test)]
 use spill::{object_spill_enabled, spill_capable_owner, spill_get, SPILL_MAX_FIELD_INDEX};

crates/perry-runtime/src/object/spill.rs

@@ -152,6 +152,56 @@ pub(crate) fn spill_set(obj_ptr: usize, field_index: usize, vbits: u64) {
     }
 }
 
+/// Reserve exact-width overflow storage for a freshly allocated object whose
+/// final key count is already known.
+///
+/// Dynamic objects normally discover their width one property at a time, so
+/// the first overflow store uses an array with general-purpose growth
+/// headroom. JSON tapes already encode the matching container boundary and can
+/// count an object's top-level keys before materializing it. Keeping the
+/// primary object at [`INLINE_SLOT_FLOOR`] avoids the measured regression from
+/// widening every object, while an exact spill avoids padding every record's
+/// side allocation to [`crate::array::MIN_ARRAY_CAPACITY`].
+pub(crate) fn reserve_object_spill(obj_ptr: usize, field_count: u32) {
+    if !object_spill_enabled()
+        || field_count as usize > SPILL_MAX_FIELD_INDEX
+        || unsafe { !spill_capable_owner(obj_ptr) }
+    {
+        return;
+    }
+
+    unsafe {
+        let obj = obj_ptr as *mut ObjectHeader;
+        let inline_capacity =
+            std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32);
+        if field_count <= inline_capacity {
+            return;
+        }
+
+        let scope = crate::gc::RuntimeHandleScope::new();
+        let obj_handle = scope.root_raw_mut_ptr(obj);
+        object_meta_ensure(obj);
+
+        let obj = obj_handle.get_raw_mut_ptr::<ObjectHeader>();
+        let meta = (*obj).meta;
+        if (*meta).spill != 0 {
+            return;
+        }
+
+        let spill = crate::array::js_array_alloc_with_length_exact(field_count);
+        let obj = obj_handle.get_raw_mut_ptr::<ObjectHeader>();
+        let meta = (*obj).meta;
+        if (*meta).spill == 0 {
+            (*meta).spill = spill as u64;
+            crate::gc::runtime_write_barrier_slot(
+                meta as usize,
+                &(*meta).spill as *const _ as usize,
+                spill as u64,
+            );
+        }
+    }
+}
+
 #[cfg(test)]
 pub(crate) type SpillSafepointHook = fn(usize);