mmtk/policy/
largeobjectspace.rs

1use atomic::Ordering;
2
3use crate::plan::tracing::{ObjectQueue, OptionObjectQueue};
4use crate::policy::sft::GCWorkerMutRef;
5use crate::policy::sft::SFT;
6use crate::policy::space::{CommonSpace, Space};
7use crate::util::alloc::allocator::AllocationOptions;
8use crate::util::constants::BYTES_IN_PAGE;
9use crate::util::heap::{FreeListPageResource, PageResource};
10use crate::util::metadata;
11use crate::util::metadata::side_metadata::spec_defs::LOS_PAGE_REUSE_COUNT;
12use crate::util::metadata::MetadataSpec;
13use crate::util::object_enum::ClosureObjectEnumerator;
14use crate::util::object_enum::ObjectEnumerator;
15use crate::util::opaque_pointer::*;
16use crate::util::rc::RefCountHelper;
17use crate::util::treadmill::TreadMill;
18use crate::util::{Address, ObjectReference};
19use crate::vm::ObjectModel;
20use crate::vm::VMBinding;
21use std::sync::atomic::AtomicBool;
22use std::sync::atomic::AtomicUsize;
23
24#[allow(unused)]
25const PAGE_MASK: usize = !(BYTES_IN_PAGE - 1);
26
27const MARK_BIT: u8 = 0b01;
28const NURSERY_BIT: u8 = 0b10;
29#[allow(unused)]
30const LOS_BIT_MASK: u8 = 0b11;
31
32/// The states of the [`ObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC`] metadata in tracing-based GC.
33///
34/// # The states
35///
36/// LOS objects can be in one of the three states at any time:
37///
38/// | State               | `NURSERY_BIT` | `MARK_BIT`         |
39/// |---------------------|---------------|--------------------|
40/// | Nursery             | 1             | disregarded        |
41/// | MatureUnmarked      | 0             | `!= mark_state`    |
42/// | MatureMarked        | 0             | `== mark_state`    |
43///
44/// Note that there is no "nursery marked" state.  Nursery objects are promoted when marked.
45///
46/// When encoded as the two-bit metadata, both `0b10` and `0b11` represents the Nursery state.  When
47/// the [`NURSERY_BIT`] is 0, it is mature, and the [`MARK_BIT`] represents "marked" if it is equal
48/// to [`LargeObjectSpace::mark_state`].
49///
50/// When flipping the meaning of the mark bit, MatureUnmarked becomes MatureMarked, and MatureMarked
51/// becomes MatureUnmarked. However, the Nursery state remains the Nursery state because the mark
52/// bit is ignored.
53///
54/// # In tracing collectors
55///
56/// In tracing collectors, allowed state transitions are:
57///
58/// ```text
59///  allocate (normal)          allocate (as live)
60///  │                          |
61///  │   ┌──┐flip mark state    |
62///  │   │  │                   |
63/// ┌▼───▼──┴─┐         ┌───────▼──────┐ flip mark state ┌────────────────┐
64/// │         │ mark    │              ├────────────────►│                │
65/// │ Nursery │────────►│ MatureMarked │                 │ MatureUnmarked │
66/// │         │         │              │◄────────────────┤                │
67/// └─────────┘         └──────────────┘ mark            └────────────────┘
68/// ```
69///
70/// Newly allocated objects can be either in the `Nursery` state (for normal allocations) or the
71/// `MatureMarked` state (when allocating as live for concurrent GC).  Note that right before
72/// flipping mark state (at the beginning of a full-heap GC), all objects must be either in the
73/// `Nursery` state or the `MatureMarked` state.  There must not be unmarked mature objects,
74/// otherwise it is an error.  After flipping, all objects become unmarked (Nursery remains Nursery,
75/// and MatureMarked becomes MatureUnmarked).
76///
77/// # In RC collectors
78///
79/// In RC collectors (currently just LXR), the `NURSERY_BIT` is unused, and the `Nursery` state is
80/// unused.
81///
82/// ```text
83///         allocate
84///         |
85/// ┌───────▼──────┐ flip mark state ┌────────────────┐
86/// │              ├────────────────►│                │
87/// │ MatureMarked │                 │ MatureUnmarked │
88/// │              │◄────────────────┤                │
89/// └──────────────┘ mark            └────────────────┘
90/// ```
91///
92/// Newly allocated objects are always in the `MatureMarked` state.  There are two reasons:
93///
94/// 1.  When backup tracing is *not* in progress, all (surviving) objects are in the `MatureMarked`
95///     state.  Allocating new objects in the `MatureMarked` state just makes them look like any
96///     other objects.  During RC collections, only the reference counts are used, and the
97///     `MatureMarked` state is disregared.
98/// 2.  When backup tracing starts (the InitialMark pause or the Full pause), the meaning of the
99///     mark bit is flipped, and all `MatureMarked` objects become `MatureUnmarked`.  Objects
100///     allocated during concurrent marking are in the `MatureMarked` state, too.  They will be
101///     conservatively considered as "marked", just like SATB-based concurrent tracing collectors.
102///     At the end of tracing, objects that are in the `MatureUnmarked` state have been dead since
103///     tracing started, i.e. they have been dead in the snapshot in the beginning (SATB).
104mod mark_nursery_bits_states {
105    use super::NURSERY_BIT;
106
107    /// Return true if the mark-nursery state represents the `Nursery` state.
108    pub fn is_nursery(state: u8) -> bool {
109        state & NURSERY_BIT == NURSERY_BIT
110    }
111
112    /// Return true if the mark-nursery state represents a marked state (i.e. the `MatureMarked`
113    /// state). All other states are considered unmarked.
114    pub fn is_marked(state: u8, mark_state: u8) -> bool {
115        state == mark_state
116    }
117}
118
119/// This type implements a policy for large objects. Each instance corresponds
120/// to one Treadmill space.
121pub struct LargeObjectSpace<VM: VMBinding> {
122    common: CommonSpace<VM>,
123    pr: FreeListPageResource<VM>,
124    mark_state: u8,
125    in_nursery_gc: bool,
126    treadmill: TreadMill,
127    clear_log_bit_on_sweep: bool,
128    pub num_pages_released_lazy: AtomicUsize,
129    pub rc_enabled: bool,
130    pub(crate) rc: RefCountHelper<VM>,
131    pub is_end_of_satb_or_full_gc: bool,
132    /// Whether newly allocated LOS objects should bump `LOS_PAGE_REUSE_COUNT` on their pages, so
133    /// remembered-set entries recorded against a page's previous occupant are invalidated. Only
134    /// needed while concurrent marking can be validating a remembered set; set/cleared by the
135    /// owning plan (currently only LXR) as concurrent marking starts/ends.
136    pub(crate) bump_page_reuse_count: AtomicBool,
137}
138
139impl<VM: VMBinding> SFT for LargeObjectSpace<VM> {
140    fn name(&self) -> &'static str {
141        self.get_name()
142    }
143    fn is_live(&self, object: ObjectReference) -> bool {
144        if self.rc_enabled {
145            if self.is_end_of_satb_or_full_gc {
146                return self.is_marked(object) && self.rc.count(object) > 0;
147            }
148            return self.rc.count(object) > 0;
149        }
150        self.is_marked(object)
151    }
152    fn is_reachable(&self, object: ObjectReference) -> bool {
153        if self.rc_enabled {
154            self.test_mark_bit(object, self.mark_state) && self.rc.count(object) > 0
155        } else {
156            self.is_live(object)
157        }
158    }
159    #[cfg(feature = "object_pinning")]
160    fn pin_object(&self, _object: ObjectReference) -> bool {
161        false
162    }
163    #[cfg(feature = "object_pinning")]
164    fn unpin_object(&self, _object: ObjectReference) -> bool {
165        false
166    }
167    #[cfg(feature = "object_pinning")]
168    fn is_object_pinned(&self, _object: ObjectReference) -> bool {
169        true
170    }
171    fn is_movable(&self) -> bool {
172        false
173    }
174    #[cfg(feature = "sanity")]
175    fn is_sane(&self) -> bool {
176        true
177    }
178
179    fn initialize_object_metadata(&self, object: ObjectReference, bytes: usize) {
180        // VO bit: Set for all objects.
181        #[cfg(feature = "vo_bit")]
182        crate::util::metadata::vo_bit::set_vo_bit(object);
183        #[cfg(all(feature = "vo_bit", debug_assertions))]
184        {
185            use crate::util::constants::LOG_BYTES_IN_PAGE;
186            let vo_addr = object.to_raw_address();
187            let offset_from_page_start = vo_addr & ((1 << LOG_BYTES_IN_PAGE) - 1) as usize;
188            debug_assert!(
189                offset_from_page_start < crate::util::metadata::vo_bit::VO_BIT_WORD_TO_REGION,
190                "The raw address of ObjectReference is not in the first 512 bytes of a page. The internal pointer searching for LOS won't work."
191            );
192        }
193
194        if self.rc_enabled {
195            // Add to treadmill nursery
196            self.treadmill.add_to_treadmill(object, true);
197            // Initialize the object to the MatureMarked state.
198            VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC.store_atomic::<VM, u8>(
199                object,
200                self.mark_state,
201                None,
202                Ordering::SeqCst,
203            );
204            // Initialize metadata
205            if self.bump_page_reuse_count.load(Ordering::Acquire) {
206                for off in (0..bytes).step_by(BYTES_IN_PAGE) {
207                    let a = object.to_raw_address() + off;
208                    let count = LOS_PAGE_REUSE_COUNT.load_atomic::<u8>(a, Ordering::SeqCst);
209                    let new_count = if count == u8::MAX { 0 } else { count + 1 };
210                    LOS_PAGE_REUSE_COUNT.store_atomic::<u8>(a, new_count, Ordering::SeqCst);
211                }
212            }
213            return;
214        }
215
216        let allocate_as_live = self.should_allocate_as_live();
217        let into_nursery = !allocate_as_live;
218
219        {
220            let mark_nursery_state = if into_nursery {
221                // If we allocate the object into nursery,
222                // the initial state will be Nursery.
223                // It is considered the Nursery state as long as the NURSERY_BIT is set,
224                // regardless of the mark state.
225                NURSERY_BIT
226            } else {
227                // If we allocate an object as live,
228                // the initial state will be MatureMarked.
229                // The NURSERY_BIT bit is not set,
230                // and the mark bit is equal to `self.mark_state`.
231                self.mark_state
232            };
233
234            VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC.store_atomic::<VM, u8>(
235                object,
236                mark_nursery_state,
237                None,
238                Ordering::SeqCst,
239            );
240        }
241
242        // global unlog bit: Set if `unlog_allocated_object`.  Ensure it is not set otherwise.
243        if self.common.unlog_allocated_object {
244            debug_assert!(self.common.needs_log_bit);
245            debug_assert!(
246                !allocate_as_live,
247                "Currently only ConcurrentImmix can allocate as live, and it doesn't unlog allocated objects in LOS."
248            );
249
250            VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.mark_as_unlogged::<VM>(object, Ordering::SeqCst);
251        } else {
252            #[cfg(debug_assertions)]
253            if self.common.needs_log_bit {
254                debug_assert_eq!(
255                    VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.load_atomic::<VM, u8>(
256                        object,
257                        None,
258                        Ordering::Acquire
259                    ),
260                    0
261                );
262            }
263        }
264
265        // Add to the treadmill.  Nursery and mature objects need to be added to different sets.
266        self.treadmill.add_to_treadmill(object, into_nursery);
267    }
268
269    #[cfg(feature = "vo_bit")]
270    fn is_mmtk_object(&self, addr: Address) -> Option<ObjectReference> {
271        crate::util::metadata::vo_bit::is_vo_bit_set_for_addr(addr)
272    }
273    #[cfg(feature = "vo_bit")]
274    fn find_object_from_internal_pointer(
275        &self,
276        ptr: Address,
277        max_search_bytes: usize,
278    ) -> Option<ObjectReference> {
279        use crate::{util::metadata::vo_bit, MMAPPER};
280
281        let mmap_granularity = MMAPPER.granularity();
282
283        // We need to check if metadata address is mapped or not.  But we make use of the granularity of
284        // the `Mmapper` to reduce the number of checks.  This records the start of a grain that is
285        // tested to be mapped.
286        let mut mapped_grain = Address::MAX;
287
288        // For large object space, it is a bit special. We only need to check VO bit for each page.
289        let mut cur_page = ptr.align_down(BYTES_IN_PAGE);
290        let low_page = ptr
291            .saturating_sub(max_search_bytes)
292            .align_down(BYTES_IN_PAGE);
293        while cur_page >= low_page {
294            if cur_page < mapped_grain {
295                if !cur_page.is_mapped() {
296                    // If the page start is not mapped, there can't be an object in it.
297                    return None;
298                }
299                // This is mapped. No need to check for this chunk.
300                mapped_grain = cur_page.align_down(mmap_granularity);
301            }
302            // For performance, we only check the first word which maps to the first 512 bytes in the page.
303            // In almost all the cases, it should be sufficient.
304            // However, if the raw address of ObjectReference is not in the first 512 bytes, this won't work.
305            // We assert this when we set VO bit for LOS.
306            if vo_bit::get_raw_vo_bit_word(cur_page) != 0 {
307                // Find the exact address that has vo bit set
308                for offset in 0..vo_bit::VO_BIT_WORD_TO_REGION {
309                    let addr = cur_page + offset;
310                    if unsafe { vo_bit::is_vo_addr(addr) } {
311                        return vo_bit::is_internal_ptr_from_vo_bit::<VM>(addr, ptr);
312                    }
313                }
314                unreachable!(
315                    "We found vo bit in the raw word, but we cannot find the exact address"
316                );
317            }
318
319            cur_page -= BYTES_IN_PAGE;
320        }
321        None
322    }
323    fn sft_trace_object(
324        &self,
325        queue: &mut OptionObjectQueue,
326        object: ObjectReference,
327        _worker: GCWorkerMutRef,
328    ) -> ObjectReference {
329        self.trace_object(queue, object)
330    }
331
332    fn debug_print_object_info(&self, object: ObjectReference) {
333        let mark_nursery_state = VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC
334            .load_atomic::<VM, u8>(object, None, Ordering::SeqCst);
335        let mark_state = self.mark_state;
336        println!("mark_nursery_state = 0b{:02b}", mark_nursery_state);
337        println!("LOS mark state = {}", mark_state);
338        println!("marked = {}", self.is_marked(object));
339        if self.rc_enabled {
340            let rc = self.rc.count(object);
341            println!("RC = {}", rc);
342        } else {
343            let is_in_nursery = mark_nursery_bits_states::is_nursery(mark_nursery_state);
344            println!("is in nursery = {}", is_in_nursery);
345        }
346        self.common.debug_print_object_global_info(object);
347    }
348}
349
350impl<VM: VMBinding> Space<VM> for LargeObjectSpace<VM> {
351    fn as_space(&self) -> &dyn Space<VM> {
352        self
353    }
354    fn as_sft(&self) -> &(dyn SFT + Sync + 'static) {
355        self
356    }
357    fn get_page_resource(&self) -> &dyn PageResource<VM> {
358        &self.pr
359    }
360    fn maybe_get_page_resource_mut(&mut self) -> Option<&mut dyn PageResource<VM>> {
361        Some(&mut self.pr)
362    }
363
364    fn initialize_sft(&self, sft_map: &mut dyn crate::policy::sft_map::SFTMap) {
365        self.common().initialize_sft(self.as_sft(), sft_map)
366    }
367
368    fn common(&self) -> &CommonSpace<VM> {
369        &self.common
370    }
371
372    fn release_multiple_pages(&mut self, start: Address) {
373        self.pr.release_pages(start);
374    }
375
376    fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) {
377        // `MMTK::enumerate_objects` is not allowed during GC, so the collection nursery and the
378        // from space must be empty.  In `ConcurrentImmix`, mutators may run during GC and call
379        // `MMTK::enumerate_objects`.  It has undefined behavior according to the current API, so
380        // the assertion failure is expected.
381        assert!(
382            self.treadmill.is_collect_nursery_empty(),
383            "Collection nursery is not empty"
384        );
385        assert!(
386            self.treadmill.is_from_space_empty(),
387            "From-space is not empty"
388        );
389
390        // Visit objects in the allocation nursery and the to-space, which contain young and old
391        // objects, respectively, during mutator time.
392        self.treadmill.enumerate_objects(enumerator, false);
393    }
394
395    fn clear_side_log_bits(&self) {
396        let mut enumerator = ClosureObjectEnumerator::<_, VM>::new(|object| {
397            VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.clear::<VM>(object, Ordering::SeqCst);
398        });
399        // Visit all objects.  It can be ordered arbitrarily with `Self::Release` which sweeps dead
400        // objects (removing them from the treadmill) and clears their unlog bits, too.
401        self.treadmill.enumerate_objects(&mut enumerator, true);
402    }
403
404    fn set_side_log_bits(&self) {
405        let mut enumerator = ClosureObjectEnumerator::<_, VM>::new(|object| {
406            VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.mark_as_unlogged::<VM>(object, Ordering::SeqCst);
407        });
408        // Visit all objects.
409        self.treadmill.enumerate_objects(&mut enumerator, true);
410    }
411}
412
413use crate::scheduler::GCWorker;
414use crate::util::copy::CopySemantics;
415
416impl<VM: VMBinding> crate::policy::gc_work::PolicyTraceObject<VM> for LargeObjectSpace<VM> {
417    fn trace_object<Q: ObjectQueue, const KIND: crate::policy::gc_work::TraceKind>(
418        &self,
419        queue: &mut Q,
420        object: ObjectReference,
421        _copy: Option<CopySemantics>,
422        _worker: &mut GCWorker<VM>,
423    ) -> ObjectReference {
424        self.trace_object(queue, object)
425    }
426    fn may_move_objects<const KIND: crate::policy::gc_work::TraceKind>() -> bool {
427        false
428    }
429}
430
431impl<VM: VMBinding> LargeObjectSpace<VM> {
432    pub fn new(
433        args: crate::policy::space::PlanCreateSpaceArgs<VM>,
434        protect_memory_on_release: bool,
435        clear_log_bit_on_sweep: bool,
436    ) -> Self {
437        let is_discontiguous = args.vmrequest.is_discontiguous();
438        let vm_map = args.vm_map;
439        let rc_enabled = args.constraints.rc_enabled;
440        let specs = if rc_enabled {
441            vec![
442                *VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC,
443                MetadataSpec::OnSide(LOS_PAGE_REUSE_COUNT),
444            ]
445        } else {
446            vec![*VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC]
447        };
448        let policy_args =
449            args.into_policy_args(false, false, metadata::extract_side_metadata(&specs));
450        let common = CommonSpace::new(policy_args);
451        let mut pr = if is_discontiguous {
452            FreeListPageResource::new_discontiguous(vm_map)
453        } else {
454            FreeListPageResource::new_contiguous(common.start, common.extent, vm_map)
455        };
456        pr.protect_memory_on_release = if protect_memory_on_release {
457            Some(common.mmap_protection())
458        } else {
459            None
460        };
461        LargeObjectSpace {
462            pr,
463            common,
464            mark_state: 0,
465            in_nursery_gc: false,
466            treadmill: TreadMill::new(),
467            clear_log_bit_on_sweep,
468            num_pages_released_lazy: Default::default(),
469            rc_enabled: false,
470            rc: RefCountHelper::NEW,
471            is_end_of_satb_or_full_gc: false,
472            bump_page_reuse_count: AtomicBool::new(false),
473        }
474    }
475
476    fn release_object(&self, object: ObjectReference) -> usize {
477        let start = get_super_page(object.to_object_start::<VM>());
478        #[cfg(feature = "vo_bit")]
479        crate::util::metadata::vo_bit::unset_vo_bit(object);
480        if self.rc_enabled {
481            debug_assert_eq!(self.rc.count(object), 0);
482            let pages = self.pr.get_pages(start);
483            // TODO: Currently this code path assumes the collector is LXR and it uses field log bit.
484            // When we can use object log bit for LXR, we should merge with `sweep_large_pages`
485            // and clear object log bit instead.
486            VM::VMObjectModel::GLOBAL_FIELD_UNLOG_BIT_SPEC
487                .as_spec()
488                .extract_side_spec()
489                .bzero_metadata(start, pages * BYTES_IN_PAGE);
490        }
491        self.pr.release_pages(start)
492    }
493
494    pub fn release_rc_nursery_objects(&self) {
495        debug_assert!(self.rc_enabled);
496        // promote nursery objects or release dead nursery
497        for o in self.treadmill.collect_alloc_nursery() {
498            if self.rc.count(o) == 0 {
499                self.release_object(o);
500            } else {
501                self.treadmill.add_to_treadmill(o, false);
502            }
503        }
504    }
505
506    pub fn prepare(&mut self, full_heap: bool) {
507        if full_heap {
508            self.mark_state = MARK_BIT - self.mark_state;
509        }
510        self.num_pages_released_lazy.store(0, Ordering::Relaxed);
511        if self.rc_enabled {
512            return;
513        }
514        self.treadmill.flip(full_heap);
515        self.in_nursery_gc = !full_heap;
516    }
517
518    pub fn release(&mut self, full_heap: bool) {
519        if self.rc_enabled {
520            self.release_rc_nursery_objects();
521            return;
522        }
523        // We swapped the allocation nursery and the collection nursery when GC starts, and we don't
524        // add objects to the allocation nursery during GC.  It should have remained empty during
525        // the whole GC.
526        debug_assert!(self.treadmill.is_alloc_nursery_empty());
527
528        self.sweep_large_pages(true);
529        debug_assert!(self.treadmill.is_collect_nursery_empty());
530        if full_heap {
531            self.sweep_large_pages(false);
532            debug_assert!(self.treadmill.is_from_space_empty());
533        }
534    }
535
536    // Allow nested-if for this function to make it clear that test_and_mark() is only executed
537    // for the outer condition is met.
538    #[allow(clippy::collapsible_if)]
539    pub fn trace_object<Q: ObjectQueue>(
540        &self,
541        queue: &mut Q,
542        object: ObjectReference,
543    ) -> ObjectReference {
544        #[cfg(feature = "vo_bit")]
545        debug_assert!(
546            crate::util::metadata::vo_bit::is_vo_bit_set(object),
547            "{:x}: VO bit not set",
548            object
549        );
550
551        if self.rc_enabled {
552            if self.test_and_mark(object).is_ok() {
553                queue.enqueue(object);
554            }
555            return object;
556        }
557
558        // We don't check if the current GC is a nursery GC,
559        // and we don't check if `object` is in nursery.
560        // test_and_mark will always transition the state to MatureMarked.
561        // If the object is already in the MatureMarked state, test_and_mark will not do anything.
562        if let Ok(old_state) = self.test_and_mark(object) {
563            let was_nursery_object = mark_nursery_bits_states::is_nursery(old_state);
564            trace!(
565                "Marked LOS object {}.  It {} a nursery object",
566                object,
567                if was_nursery_object { "was" } else { "was not" }
568            );
569            // If the object was a nursery object, we move it to `to_space` in the treadmill.
570            self.treadmill.copy(object, was_nursery_object);
571            // We just moved the object out of the logical nursery, mark it as unlogged.
572            // We also unlog mature objects as their unlog bit may have been unset before the
573            // full-heap GC
574            if self.common.unlog_traced_object {
575                VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC
576                    .mark_as_unlogged::<VM>(object, Ordering::SeqCst);
577            }
578            queue.enqueue(object);
579        } else {
580            trace!("LOS object {} is already marked", object);
581        }
582
583        object
584    }
585
586    fn sweep_large_pages(&mut self, sweep_nursery: bool) {
587        let sweep = |object: ObjectReference| {
588            // Clear log bits for dead objects to prevent a new nursery object having the unlog bit set
589            // TODO: This code path assumes the log bit is object log bit instead of field log bit.
590            // When generational plans and ConcurrentImmix support field log bit,
591            // we can push this clean-up operation into `Self::release_object`.
592            if self.clear_log_bit_on_sweep {
593                VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.clear::<VM>(object, Ordering::SeqCst);
594            }
595            self.release_object(object);
596        };
597        if sweep_nursery {
598            for object in self.treadmill.collect_nursery() {
599                sweep(object);
600            }
601        } else {
602            for object in self.treadmill.collect_mature() {
603                sweep(object)
604            }
605        }
606    }
607
608    /// Enumerate objects in the to-space.  It is a workaround for OVC which currently needs
609    /// to enumerate reachable objects for during reference forwarding.
610    pub(crate) fn enumerate_to_space_objects(&self, enumerator: &mut dyn ObjectEnumerator) {
611        // This function is intended to enumerate objects in the to-space.
612        // The alloc nursery should have remained empty during the GC.
613        debug_assert!(self.treadmill.is_alloc_nursery_empty());
614        // We only need to visit the to_space, which contains all objects determined to be live.
615        self.treadmill.enumerate_objects(enumerator, false);
616    }
617
618    /// Allocate an object
619    pub fn allocate_pages(
620        &self,
621        tls: VMThread,
622        pages: usize,
623        alloc_options: AllocationOptions,
624    ) -> Address {
625        self.acquire(tls, pages, alloc_options)
626    }
627
628    /// Attempt to mark `object`.  Return `true` if this invocation marked the object.
629    pub fn attempt_mark(&self, object: ObjectReference) -> bool {
630        self.test_and_mark(object).is_ok()
631    }
632
633    pub fn rc_free(&self, o: ObjectReference) {
634        if self.treadmill.remove_mature(o) {
635            let pages = self.release_object(o);
636            self.num_pages_released_lazy
637                .fetch_add(pages, Ordering::Relaxed);
638        }
639    }
640
641    /// Test if the nursery-mark state is in the marked state.  If not, it will atomically change
642    /// the state to marked (`MatureMarked`, which implies clearing the nursery bit).  Returns
643    /// `Ok(old_state)` if this invocation marked the object, or `Err(old_state)` if the object is
644    /// already in the marked state.  In either case, `old_state` is the old state before the atomic
645    /// operation.
646    fn test_and_mark(&self, object: ObjectReference) -> Result<u8, u8> {
647        let mark_state = self.mark_state;
648        VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC.fetch_update_metadata::<VM, u8, _>(
649            object,
650            Ordering::SeqCst,
651            Ordering::SeqCst,
652            |old_value| {
653                (!mark_nursery_bits_states::is_marked(old_value, mark_state)).then_some(mark_state)
654            },
655        )
656    }
657
658    /// Test if the mark bit of `LOCAL_LOS_MARK_NURSERY_SPEC` is equal to `value`.
659    fn test_mark_bit(&self, object: ObjectReference, value: u8) -> bool {
660        VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC.load_atomic::<VM, u8>(
661            object,
662            None,
663            Ordering::SeqCst,
664        ) & MARK_BIT
665            == value
666    }
667
668    pub fn sweep_rc_mature_objects_after_satb(&self, is_live: &impl Fn(ObjectReference) -> bool) {
669        self.treadmill.retain_mature(|o| {
670            if !is_live(*o) {
671                self.rc.set(*o, 0);
672                let pages = self.release_object(*o);
673                self.num_pages_released_lazy
674                    .fetch_add(pages, Ordering::Relaxed);
675                false
676            } else {
677                true
678            }
679        });
680    }
681
682    /// Check if a given object is marked
683    pub fn is_marked(&self, object: ObjectReference) -> bool {
684        let mark_nursery_state = VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC
685            .load_atomic::<VM, u8>(object, None, Ordering::SeqCst);
686
687        mark_nursery_bits_states::is_marked(mark_nursery_state, self.mark_state)
688    }
689}
690
691fn get_super_page(cell: Address) -> Address {
692    cell.align_down(BYTES_IN_PAGE)
693}
mmtk/policy/largeobjectspace.rs

mmtk/policy/
largeobjectspace.rs
