@@ -47,6 +47,101 @@ pub fn arena_alloc(size: usize, align: usize) -> *mut u8 {
4747 }
4848}
4949
50+ /// [`arena_alloc_gc`] with its **collection point removed**: the request is
51+ /// served by bumping the nursery block that is already open, or the call
52+ /// returns null. It never runs `gc_check_trigger()`, never reserves a fresh
53+ /// block and never births into old-gen.
54+ ///
55+ /// ★ The value here is not the handful of instructions saved on the slow
56+ /// branch — it is the *guarantee*. A runtime helper holding raw heap pointers
57+ /// it has not rooted can allocate through this and, on a non-null return,
58+ /// KNOW that nothing moved: the only collection point on the arena path is
59+ /// precisely the one this refuses to reach. That turns "root every operand
60+ /// into the transient handle stack, then re-read every one of them
61+ /// afterwards" into "read them once", for the overwhelmingly common case
62+ /// where a 1 MB block has room.
63+ ///
64+ /// On null the caller MUST fall back: root its operands, re-issue through
65+ /// [`arena_alloc_gc`], and re-read the operands from their handles. Nothing
66+ /// has collected at that point either — a null is a refusal, not an event —
67+ /// so the operands are still readable where the caller last saw them.
68+ ///
69+ /// Deliberately written out rather than sharing a body with `arena_alloc_gc`:
70+ /// that function is `#[inline(always)]` into every allocation site in the
71+ /// program (including user IR, through the bitcode-link path), and it is not
72+ /// worth risking its codegen to save twenty lines here. The two divergences
73+ /// are both refusals — an oversized request and a non-empty hot free list
74+ /// both return null instead of being served — so this can only ever hand back
75+ /// memory `arena_alloc_gc` would have handed back identically.
76+ #[ inline( always) ]
77+ pub ( crate ) fn arena_alloc_gc_no_collect ( size : usize , align : usize , obj_type : u8 ) -> * mut u8 {
78+ use crate :: gc:: { GcHeader , GC_FLAG_ARENA , GC_HEADER_SIZE } ;
79+
80+ let total = gc_padded_total_size ( size, align) ;
81+ // Old-gen birth walks page lists and can reserve — outside the contract.
82+ if crate :: gc:: is_large_object_total_size_for_type ( total, obj_type) {
83+ return std:: ptr:: null_mut ( ) ;
84+ }
85+ // The free-list arm of `arena_alloc_gc` cannot collect either, but nothing
86+ // in the tree ever sets this latch, so serving it here would be untested
87+ // code on a hot path. Refuse and let the caller take the rooted path.
88+ if crate :: gc:: hot_arena_free_list_nonempty ( ) . get ( ) {
89+ return std:: ptr:: null_mut ( ) ;
90+ }
91+
92+ let raw = arena_alloc_no_collect ( total, align) ;
93+ if raw. is_null ( ) {
94+ return std:: ptr:: null_mut ( ) ;
95+ }
96+
97+ unsafe {
98+ let header = raw as * mut GcHeader ;
99+ ( * header) . obj_type = obj_type;
100+ ( * header) . gc_flags = GC_FLAG_ARENA | crate :: gc:: gc_birth_extra_flags ( ) ;
101+ crate :: gc:: gc_note_black_birth ( header) ;
102+ ( * header) . _reserved = 0 ;
103+ ( * header) . size = total as u32 ;
104+ }
105+
106+ unsafe { raw. add ( GC_HEADER_SIZE ) }
107+ }
108+
109+ /// [`arena_alloc`] minus its collection point: serve the request from the
110+ /// block that is already open, or return null.
111+ ///
112+ /// The inline-state sync/resync mirrors `arena_alloc`'s, so a successful
113+ /// allocation is indistinguishable from one taken through it. A refusal
114+ /// leaves every offset exactly where it was, so the caller's fallback through
115+ /// `arena_alloc` behaves as if this had never been called.
116+ #[ inline( always) ]
117+ fn arena_alloc_no_collect ( size : usize , align : usize ) -> * mut u8 {
118+ unsafe {
119+ let inline_ptr = crate :: arena:: hot_inline_state ( ) ;
120+ let arena_ptr = crate :: arena:: hot_arena ( ) ;
121+ if !( * inline_ptr) . data . is_null ( ) {
122+ let offset = ( * inline_ptr) . offset ;
123+ let arena = & mut * arena_ptr;
124+ let current = arena. current ;
125+ arena. blocks [ current] . offset = offset;
126+ }
127+ let Some ( ptr) = crate :: arena:: arena_cell_try_alloc_current ( arena_ptr, size, align) else {
128+ return std:: ptr:: null_mut ( ) ;
129+ } ;
130+ if !( * inline_ptr) . data . is_null ( ) {
131+ let ( data, offset, block_size) = {
132+ let arena = & * arena_ptr;
133+ let block = & arena. blocks [ arena. current ] ;
134+ ( block. data , block. offset , block. size )
135+ } ;
136+ let inline = & mut * inline_ptr;
137+ inline. data = data;
138+ inline. offset = offset;
139+ inline. size = block_size;
140+ }
141+ ptr
142+ }
143+ }
144+
50145/// Allocate from the longlived arena (issue #179). Unlike `arena_alloc`,
51146/// this never touches the inline allocator state — the longlived arena
52147/// is reserved for explicit-call allocations from cache builders
0 commit comments