mmtk/plan/lxr/gc_work/
rc.rs

1use super::super::LazySweepingJobsCounter;
2use super::super::SurvivalRatioPredictorLocal;
3use super::super::LXR;
4use super::super::{LAZY_DECREMENTS, MATURE_EVACUATION, NO_EVAC, NURSERY_EVACUATION};
5use super::tracing::LXRConcurrentTraceObjects;
6use super::tracing::LXRStopTheWorldProcessEdges;
7use super::tracing::LXRStopTheWorldProcessNodes;
8use super::tracing::ProcessModBufSATB;
9use super::ProcessEdgesBase;
10use crate::plan::VectorQueue;
11use crate::policy::immix::block::BlockState;
12use crate::scheduler::gc_work::RootKind;
13use crate::util::copy::CopySemantics;
14use crate::util::linear_scan::UnstraddlableRegion;
15use crate::util::metadata::side_metadata::SideMetadataSpec;
16use crate::util::rc::*;
17use crate::vm::slot::Slot;
18use crate::{
19    plan::concurrent::global::ConcurrentPlan,
20    plan::concurrent::Pause,
21    plan::global::Plan,
22    policy::{immix::block::Block, space::Space},
23    scheduler::{GCWork, GCWorker, WorkBucketStage},
24    util::{metadata::side_metadata, object_forwarding, ObjectReference},
25    vm::*,
26    MMTK,
27};
28use atomic::Ordering;
29use std::marker::PhantomData;
30use std::ops::{Deref, DerefMut};
31use std::sync::Arc;
32
33pub struct ProcessIncs<VM: VMBinding, const KIND: EdgeKind> {
34    /// Increments to process
35    incs: Vec<VM::VMSlot>,
36    /// Recursively generated new increments
37    new_incs: VectorQueue<VM::VMSlot>,
38    new_incs_count: u32,
39    pause: Pause,
40    in_cm: bool,
41    no_evac: bool,
42    pub root_kind: Option<RootKind>,
43    depth: u32,
44    lxr: &'static LXR<VM>,
45    rc: RefCountHelper<VM>,
46    survival_ratio_predictor_local: SurvivalRatioPredictorLocal,
47}
48
49unsafe impl<VM: VMBinding, const KIND: EdgeKind> Send for ProcessIncs<VM, KIND> {}
50
51impl<VM: VMBinding, const KIND: EdgeKind> ProcessIncs<VM, KIND> {
52    const CAPACITY: usize = 1024;
53    const UNLOG_BITS: SideMetadataSpec = *VM::VMObjectModel::GLOBAL_FIELD_UNLOG_BIT_SPEC
54        .as_spec()
55        .extract_side_spec();
56
57    fn __default(lxr: &'static LXR<VM>) -> Self {
58        Self {
59            incs: vec![],
60            new_incs: VectorQueue::default(),
61            new_incs_count: 0,
62            lxr,
63            pause: Pause::RefCount,
64            in_cm: false,
65            no_evac: false,
66            depth: 1,
67            rc: RefCountHelper::NEW,
68            root_kind: None,
69            survival_ratio_predictor_local: SurvivalRatioPredictorLocal::default(),
70        }
71    }
72
73    fn add_new_slot(&mut self, worker: &mut GCWorker<VM>, s: VM::VMSlot) {
74        self.new_incs.push(s);
75        self.new_incs_count += 1;
76        if self.new_incs_count as usize >= Self::CAPACITY {
77            self.flush(worker);
78        }
79    }
80
81    pub fn new(incs: Vec<VM::VMSlot>, lxr: &'static LXR<VM>) -> Self {
82        Self {
83            incs,
84            ..Self::__default(lxr)
85        }
86    }
87
88    /// Reference-count root objects reported as objects rather than as slots.
89    /// Node counterpart of `process_incs::<EDGE_KIND_ROOT>`. Nothing here may move an object.
90    fn process_root_nodes(
91        &mut self,
92        worker: &mut GCWorker<VM>,
93        nodes: Vec<ObjectReference>,
94    ) -> (Vec<ObjectReference>, Vec<ObjectReference>) {
95        let mut roots = Vec::with_capacity(nodes.len());
96        let mut uncounted = vec![];
97        for o in nodes {
98            if !self.lxr.is_rc_object(o) {
99                uncounted.push(o);
100                continue;
101            }
102            let los = self.lxr.los().in_space(o);
103            if self.inc(o) {
104                // Promote without moving.
105                self.promote(worker, o, false, los, 0);
106            }
107            roots.push(o);
108        }
109        // Promotion above can queue further increments; hand them on rather than
110        // dropping them.
111        self.flush(worker);
112        (roots, uncounted)
113    }
114
115    /// Node-shaped counterpart of [`GCWork::do_work`] below for `KIND == EDGE_KIND_ROOT`.
116    /// However, the major difference is that we cannot move any of these objects here, as
117    /// we don't know their slots.
118    pub fn do_work_root_nodes(
119        &mut self,
120        nodes: Vec<ObjectReference>,
121        worker: &mut GCWorker<VM>,
122        mmtk: &'static MMTK<VM>,
123    ) {
124        debug_assert_eq!(KIND, EDGE_KIND_ROOT);
125        let lxr = self.lxr;
126        self.pause = lxr.current_pause().unwrap();
127        self.in_cm = lxr.concurrent_work_in_progress();
128        let (roots, uncounted) = self.process_root_nodes(worker, nodes);
129        let mut to_trace = roots.clone();
130        to_trace.extend_from_slice(&uncounted);
131        if to_trace.is_empty() {
132            return;
133        }
134        // Roots arriving mid-cycle must also enter the SATB snapshot, or the cycle
135        // collector can conclude their subgraphs are unreachable.
136        if lxr.cm_enabled() && self.in_cm {
137            worker.add_work(
138                WorkBucketStage::FinishConcurrentWork,
139                ProcessModBufSATB::new(to_trace.clone()),
140            );
141        }
142        // Nodes, unlike slots, are never re-scanned, so a stop-the-world tracing pause has to
143        // mark from them here or they are never marked at all.
144        match self.pause {
145            Pause::Full => worker.add_work(
146                WorkBucketStage::PinningRootsTrace,
147                LXRStopTheWorldProcessNodes::<VM, true>::new(
148                    to_trace,
149                    mmtk,
150                    WorkBucketStage::Closure,
151                ),
152            ),
153            Pause::FinalMark => worker.add_work(
154                WorkBucketStage::PinningRootsTrace,
155                LXRStopTheWorldProcessNodes::<VM, false>::new(
156                    to_trace,
157                    mmtk,
158                    WorkBucketStage::Closure,
159                ),
160            ),
161            // Seed concurrent marking with these roots, exactly as the slot path does --
162            // otherwise node roots seed nothing until `FinalMark`.
163            Pause::InitialMark if lxr.cm_enabled() => {
164                worker.scheduler().work_buckets[WorkBucketStage::ConcurrentResumable]
165                    .add(LXRConcurrentTraceObjects::new(to_trace, mmtk));
166            }
167            // `RefCount` has no marking phase.
168            _ => {}
169        }
170        // Recorded so the matching decrements are applied in the next pause.
171        if !roots.is_empty() {
172            lxr.curr_roots.read().unwrap().push(roots);
173        }
174    }
175
176    fn promote(
177        &mut self,
178        worker: &mut GCWorker<VM>,
179        o: ObjectReference,
180        copied: bool,
181        los: bool,
182        depth: u32,
183    ) {
184        let size = o.get_size::<VM>();
185
186        if !los {
187            let block = Block::containing(o);
188            let in_nursery_block = block.get_state() == BlockState::Nursery;
189            if !copied && in_nursery_block {
190                block.set_as_in_place_promoted();
191            }
192            self.rc.promote_with_size(o, size);
193            self.survival_ratio_predictor_local
194                .record_promotion(size, copied);
195        } else {
196            // println!("promote los {:?} {}", o, self.immix().is_marked(o));
197        }
198        // Don't mark copied objects in initial mark pause. The concurrent marker will do it (and can also resursively mark the old objects).
199        if self.in_cm || self.pause == Pause::FinalMark {
200            debug_assert!(self.lxr.is_marked(o), "{:?} is not marked", o);
201        }
202        self.scan_nursery_object(worker, o, los, !copied, depth, size);
203    }
204
205    fn record_mature_evac_remset2(
206        &mut self,
207        slot_in_defrag: bool,
208        s: VM::VMSlot,
209        o: ObjectReference,
210    ) {
211        if !(MATURE_EVACUATION && (self.in_cm || self.pause == Pause::FinalMark)) {
212            return;
213        }
214        if !slot_in_defrag && self.lxr.in_defrag(o) {
215            self.lxr.mature_evac_remset.record(s, o, self.lxr);
216        }
217    }
218
219    fn record_mature_evac_remset(&mut self, s: VM::VMSlot, o: ObjectReference) {
220        if !(MATURE_EVACUATION && (self.in_cm || self.pause == Pause::FinalMark)) {
221            return;
222        }
223        self.record_mature_evac_remset2(self.lxr.address_in_defrag(s.to_address()), s, o);
224    }
225
226    fn scan_nursery_object(
227        &mut self,
228        worker: &mut GCWorker<VM>,
229        o: ObjectReference,
230        los: bool,
231        in_place_promotion: bool,
232        _depth: u32,
233        size: usize,
234    ) {
235        let heap_bytes_per_unlog_byte = if VM::VMObjectModel::COMPRESSED_PTR_ENABLED {
236            32usize
237        } else {
238            64
239        };
240        if los {
241            let start =
242                side_metadata::address_to_meta_address(&Self::UNLOG_BITS, o.to_raw_address())
243                    .to_mut_ptr::<u8>();
244            let limit = side_metadata::address_to_meta_address(
245                &Self::UNLOG_BITS,
246                (o.to_raw_address() + size).align_up(heap_bytes_per_unlog_byte),
247            )
248            .to_mut_ptr::<u8>();
249            unsafe {
250                let bytes = limit.offset_from(start) as usize;
251                std::ptr::write_bytes(start, 0xffu8, bytes);
252            }
253            o.to_raw_address().unlog_field_relaxed::<VM>();
254        } else if in_place_promotion {
255            let header_size = if VM::VMObjectModel::COMPRESSED_PTR_ENABLED {
256                12usize
257            } else {
258                16
259            };
260            let step = heap_bytes_per_unlog_byte << 2;
261            let end = o.to_raw_address() + size;
262            let aligned_end = end.align_up(step);
263            let cursor = o.to_raw_address() + header_size;
264            let mut cursor = cursor.align_down(step);
265            let mut meta = side_metadata::address_to_meta_address(&Self::UNLOG_BITS, cursor);
266            while cursor < aligned_end {
267                unsafe { meta.store(0xffffffffu32) }
268                meta += 4usize;
269                cursor += step;
270            }
271        };
272        // Promotion above arms the per-field unlog bits. Do the same for object log bits.
273        #[cfg(feature = "lxr_object_log")]
274        VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.mark_as_unlogged::<VM>(o, Ordering::SeqCst);
275        let obj_in_defrag = !los && Block::in_defrag_block(o);
276        let tls = worker.tls.0;
277        o.iterate_fields::<VM, _>(tls, |slot| {
278            let Some(target) = slot.load() else {
279                return;
280            };
281            debug_assert!(
282                target.to_raw_address().is_mapped(),
283                "Unmapped obj {:?}.{:?} -> {:?}",
284                o,
285                slot,
286                target
287            );
288            debug_assert!(
289                target.is_in_any_space(),
290                "Unmapped obj {:?}.{:?} -> {:?}",
291                o,
292                slot,
293                target
294            );
295            let rc = self.rc.count(target);
296            if rc == 0 {
297                self.add_new_slot(worker, slot);
298            } else {
299                if rc != crate::util::rc::MAX_REF_COUNT {
300                    let _ = self.rc.inc(target);
301                }
302                self.record_mature_evac_remset2(obj_in_defrag, slot, target);
303            }
304        });
305    }
306
307    #[cold]
308    fn flush(&mut self, worker: &mut GCWorker<VM>) {
309        if !self.new_incs.is_empty() {
310            let new_incs = self.new_incs.take();
311            let mut w = ProcessIncs::<VM, EDGE_KIND_NURSERY>::new(new_incs, self.lxr);
312            w.depth += 1;
313            worker.add_work(WorkBucketStage::Unconstrained, w);
314        }
315        self.new_incs_count = 0;
316    }
317
318    /// Return true if the object's ref count is incremented and the count was zero before the increment
319    fn inc(&self, o: ObjectReference) -> bool {
320        self.rc.inc(o) == Ok(0)
321    }
322
323    fn dont_evacuate(&self, o: ObjectReference, los: bool) -> bool {
324        if los {
325            return true;
326        }
327        // Skip mature object
328        if self.rc.count(o) != 0 {
329            return true;
330        }
331        // Skip recycled lines
332        if Block::containing(o).get_state() != BlockState::Nursery {
333            return true;
334        }
335        if cfg!(debug_assertions) {
336            let cls = unsafe { (o.to_raw_address() + 8usize).load::<u32>() };
337            assert!(cls != 0, "ERROR {:?} rc={}", o, self.rc.count(o));
338        }
339        false
340    }
341
342    fn process_inc_and_evacuate(
343        &mut self,
344        worker: &mut GCWorker<VM>,
345        o: ObjectReference,
346        depth: u32,
347    ) -> ObjectReference {
348        let los = self.lxr.los().in_space(o);
349        if NURSERY_EVACUATION && !los && object_forwarding::is_forwarded_or_being_forwarded::<VM>(o)
350        {
351            while object_forwarding::is_being_forwarded::<VM>(o) {
352                std::hint::spin_loop();
353            }
354            let new = if object_forwarding::is_forwarded::<VM>(o) {
355                object_forwarding::read_forwarding_pointer::<VM>(o)
356            } else {
357                o
358            };
359            let promoted = self.inc(new);
360            if promoted && new == o {
361                self.promote(worker, o, false, los, depth);
362            }
363            return new;
364        }
365        if !NURSERY_EVACUATION || self.dont_evacuate(o, los) {
366            if self.inc(o) {
367                self.promote(worker, o, false, los, depth);
368            }
369            return o;
370        }
371        let forwarding_status = object_forwarding::attempt_to_forward::<VM>(o);
372        if object_forwarding::state_is_forwarded_or_being_forwarded(forwarding_status) {
373            // Object is moved to a new location.
374            let new = object_forwarding::spin_and_get_forwarded_object::<VM>(o, forwarding_status);
375            self.inc(new);
376            new
377        } else {
378            let is_nursery = self.rc.count(o) == 0;
379            if is_nursery && !self.no_evac {
380                // Evacuate the object
381                let new = object_forwarding::try_forward_object::<VM>(
382                    o,
383                    CopySemantics::DefaultCopy,
384                    worker.get_copy_context_mut(),
385                    |_new| {
386                        #[cfg(feature = "vo_bit")]
387                        {
388                            // Set the VO bit of the new object.
389                            crate::util::metadata::vo_bit::set_vo_bit(_new);
390                            // Clear the VO bit of the old object.
391                            // Note that sweeping can also clear the VO bit when the line is freed,
392                            // but no RC inc/dec should be performed on the old object from now on.
393                            // We clear it eagerly to detect inc/dec errors.
394                            crate::util::metadata::vo_bit::unset_vo_bit(o);
395                        }
396                    },
397                );
398                if let Some(new) = new {
399                    self.inc(new);
400                    self.promote(worker, new, true, false, depth);
401                    new
402                } else {
403                    warn!("to-space overflow");
404                    // Object is not moved.
405                    let promoted = self.inc(o);
406                    object_forwarding::clear_forwarding_bits::<VM>(o);
407                    if promoted {
408                        self.promote(worker, o, false, los, depth);
409                    }
410                    NO_EVAC.store(true, Ordering::Relaxed);
411                    self.no_evac = true;
412                    o
413                }
414            } else {
415                // Object is not moved.
416                let promoted = self.inc(o);
417                object_forwarding::clear_forwarding_bits::<VM>(o);
418                if promoted {
419                    self.promote(worker, o, false, los, depth);
420                }
421                o
422            }
423        }
424    }
425
426    /// Return `None` if the increment of the slot should be delayed
427    fn unlog_and_load_rc_object<const K: EdgeKind>(
428        &mut self,
429        s: VM::VMSlot,
430    ) -> Option<ObjectReference> {
431        let o = s.load();
432        // unlog slot
433        if K == EDGE_KIND_MATURE {
434            s.to_address().unlog_field_relaxed::<VM>();
435        }
436        o
437    }
438
439    fn process_slot<const K: EdgeKind>(
440        &mut self,
441        worker: &mut GCWorker<VM>,
442        s: VM::VMSlot,
443        depth: u32,
444        add_root_to_remset: bool,
445    ) -> Option<ObjectReference> {
446        let o = match self.unlog_and_load_rc_object::<K>(s) {
447            Some(o) => o,
448            _ => {
449                return None;
450            }
451        };
452        // println!(" - inc {:?}: {:?} rc={}", s, o, self.rc.count(o));
453        let new = self.process_inc_and_evacuate(worker, o, depth);
454        // Put this into remset if this is a mature slot, or a weak root
455        if K != EDGE_KIND_ROOT || add_root_to_remset {
456            self.record_mature_evac_remset(s, new);
457        }
458        if new != o {
459            s.store(new)
460        }
461        Some(new)
462    }
463
464    fn process_incs<const K: EdgeKind>(
465        &mut self,
466        worker: &mut GCWorker<VM>,
467        mut incs: AddressBuffer<'_, VM::VMSlot>,
468        depth: u32,
469        add_root_to_remset: bool,
470    ) -> Option<Vec<ObjectReference>> {
471        if K == EDGE_KIND_ROOT {
472            // An optimization with Rust zero allocation. However, it only works if
473            // VMSlot has exactly the same size and alignment as ObjectReference, which
474            // is not guaranteed.
475            // TODO: Check performance of this optimization. If it is not significant, we can remove it to simplify the code.
476            if std::mem::size_of::<VM::VMSlot>() == std::mem::size_of::<ObjectReference>()
477                && std::mem::align_of::<VM::VMSlot>() == std::mem::align_of::<ObjectReference>()
478            {
479                let roots = incs.as_mut_ptr() as *mut ObjectReference;
480                let mut num_roots = 0usize;
481                for s in incs.iter() {
482                    if let Some(new) = self.process_slot::<K>(worker, *s, depth, add_root_to_remset)
483                    {
484                        unsafe {
485                            roots.add(num_roots).write(new);
486                        }
487                        num_roots += 1;
488                    }
489                }
490                return if num_roots != 0 {
491                    let cap = incs.capacity();
492                    std::mem::forget(incs); // roots references incs now. we dont need incs.
493                    let roots =
494                        unsafe { Vec::<ObjectReference>::from_raw_parts(roots, num_roots, cap) };
495                    Some(roots)
496                } else {
497                    None
498                };
499            }
500
501            // General case: we need to allocate a new vector and push into it.
502            let mut roots = Vec::with_capacity(incs.len());
503            for s in incs.iter() {
504                if let Some(new) = self.process_slot::<K>(worker, *s, depth, add_root_to_remset) {
505                    roots.push(new);
506                }
507            }
508            if roots.is_empty() {
509                None
510            } else {
511                Some(roots)
512            }
513        } else {
514            for s in incs.iter() {
515                self.process_slot::<K>(worker, *s, depth, false);
516            }
517            None
518        }
519    }
520}
521
522pub type EdgeKind = u8;
523pub const EDGE_KIND_ROOT: u8 = 0;
524pub const EDGE_KIND_NURSERY: u8 = 1;
525pub const EDGE_KIND_MATURE: u8 = 2;
526
527enum AddressBuffer<'a, S: Slot> {
528    Owned(Vec<S>),
529    Ref(&'a mut Vec<S>),
530}
531
532impl<S: Slot> Deref for AddressBuffer<'_, S> {
533    type Target = Vec<S>;
534    fn deref(&self) -> &Self::Target {
535        match self {
536            Self::Owned(x) => x,
537            Self::Ref(x) => x,
538        }
539    }
540}
541
542impl<S: Slot> DerefMut for AddressBuffer<'_, S> {
543    fn deref_mut(&mut self) -> &mut Self::Target {
544        match self {
545            Self::Owned(x) => x,
546            Self::Ref(x) => x,
547        }
548    }
549}
550
551impl<VM: VMBinding, const KIND: EdgeKind> GCWork<VM> for ProcessIncs<VM, KIND> {
552    fn do_work(&mut self, worker: &mut GCWorker<VM>, mmtk: &'static MMTK<VM>) {
553        self.lxr = mmtk.get_plan().downcast_ref::<LXR<VM>>().unwrap();
554        self.pause = self.lxr.current_pause().unwrap();
555        self.in_cm = self.lxr.concurrent_work_in_progress();
556        if NO_EVAC.load(Ordering::Relaxed) {
557            self.no_evac = true;
558        } else {
559            let over_space = mmtk.get_plan().get_used_pages()
560                - mmtk.get_plan().get_collection_reserved_pages()
561                > mmtk.get_plan().get_total_pages();
562            if over_space {
563                self.no_evac = true;
564                NO_EVAC.store(true, Ordering::Relaxed);
565            }
566        }
567        // Process main buffer
568        let root_slots = if KIND == EDGE_KIND_ROOT
569            && (self.pause == Pause::FinalMark || self.pause == Pause::Full)
570        {
571            self.incs.clone()
572        } else {
573            vec![]
574        };
575        let roots = {
576            let incs = std::mem::take(&mut self.incs);
577            self.process_incs::<KIND>(worker, AddressBuffer::Owned(incs), self.depth, false)
578        };
579        if let Some(roots) = roots {
580            if self.lxr.cm_enabled()
581                && self.pause == Pause::InitialMark
582                && !self.root_kind.unwrap().should_skip_mark_and_decs()
583            {
584                if cfg!(any(feature = "sanity", debug_assertions)) {
585                    for r in &roots {
586                        assert!(
587                            r.to_raw_address().is_mapped(),
588                            "Invalid object {:?}: address is not mapped",
589                            r
590                        );
591                    }
592                }
593                worker.scheduler().work_buckets[WorkBucketStage::ConcurrentResumable]
594                    .add(LXRConcurrentTraceObjects::new(roots.clone(), mmtk));
595            }
596            if self.pause == Pause::FinalMark || self.pause == Pause::Full {
597                if !root_slots.is_empty() && self.root_kind != Some(RootKind::Weak) {
598                    if self.pause == Pause::FinalMark {
599                        let mut w = LXRStopTheWorldProcessEdges::<_, false>::new(
600                            root_slots,
601                            true,
602                            mmtk,
603                            WorkBucketStage::Closure,
604                        );
605                        w.root_kind = self.root_kind;
606                        worker.add_work(WorkBucketStage::Closure, w)
607                    } else {
608                        let mut w = LXRStopTheWorldProcessEdges::<_, true>::new(
609                            root_slots,
610                            true,
611                            mmtk,
612                            WorkBucketStage::Closure,
613                        );
614                        w.root_kind = self.root_kind;
615                        worker.add_work(WorkBucketStage::Closure, w)
616                    };
617                }
618            } else if !self.root_kind.unwrap().should_skip_decs() {
619                self.lxr.curr_roots.read().unwrap().push(roots);
620            }
621        }
622        // Process recursively generated buffer
623        let mut depth = self.depth;
624        let mut incs = vec![];
625        // Incs are processed by the worker that discovered them, unless we split.
626        // Split size defaults to `usize::MAX` (never split).
627        let split_size = *self.lxr.base().options.lxr_min_packet_split_size;
628        // Split depth defaults to 16 (split after 16 recursions).
629        let split_depth = *self.lxr.base().options.lxr_min_packet_split_depth;
630        while !self.new_incs.is_empty() {
631            self.new_incs_count = 0;
632            depth += 1;
633            incs.clear();
634            self.new_incs.swap(&mut incs);
635            if depth as usize >= split_depth && incs.len() > split_size {
636                let (a, b) = incs.split_at(incs.len() / 2);
637                let mut w = ProcessIncs::<VM, EDGE_KIND_NURSERY>::new(b.to_vec(), self.lxr);
638                w.depth = depth;
639                worker.add_work(WorkBucketStage::Unconstrained, w);
640                incs = a.to_vec();
641            }
642            if !incs.is_empty() {
643                self.process_incs::<EDGE_KIND_NURSERY>(
644                    worker,
645                    AddressBuffer::Ref(&mut incs),
646                    depth,
647                    false,
648                );
649            }
650        }
651        self.survival_ratio_predictor_local.sync();
652    }
653}
654
655pub struct ProcessDecs<VM: VMBinding> {
656    /// Decrements to process
657    decs: Option<Vec<ObjectReference>>,
658    decs_arc: Option<Arc<Vec<ObjectReference>>>,
659    /// Recursively generated new decrements
660    new_decs: VectorQueue<ObjectReference>,
661    counter: LazySweepingJobsCounter,
662    mark_objects: VectorQueue<ObjectReference>,
663    mark_dead_objects: bool,
664    mature_sweeping_in_progress: bool,
665    rc: RefCountHelper<VM>,
666}
667
668impl<VM: VMBinding> ProcessDecs<VM> {
669    pub fn new(decs: Vec<ObjectReference>, counter: LazySweepingJobsCounter) -> Self {
670        Self {
671            decs: Some(decs),
672            decs_arc: None,
673            new_decs: VectorQueue::default(),
674            counter,
675            mark_objects: VectorQueue::default(),
676            mark_dead_objects: false,
677            mature_sweeping_in_progress: false,
678            rc: RefCountHelper::NEW,
679        }
680    }
681
682    pub fn new_arc(decs: Arc<Vec<ObjectReference>>, counter: LazySweepingJobsCounter) -> Self {
683        Self {
684            decs: None,
685            decs_arc: Some(decs),
686            new_decs: VectorQueue::default(),
687            counter,
688            mark_objects: VectorQueue::default(),
689            mark_dead_objects: false,
690            mature_sweeping_in_progress: false,
691            rc: RefCountHelper::NEW,
692        }
693    }
694
695    fn recursive_dec(&mut self, worker: &mut GCWorker<VM>, o: ObjectReference) {
696        self.new_decs.push(o);
697        if self.new_decs.is_full() {
698            self.flush(worker)
699        }
700    }
701
702    fn new_work(&self, worker: &mut GCWorker<VM>, w: ProcessDecs<VM>) {
703        worker.add_work(WorkBucketStage::Unconstrained, w);
704    }
705
706    fn flush(&mut self, worker: &mut GCWorker<VM>) {
707        let mmtk = worker.mmtk;
708        if !self.new_decs.is_empty() {
709            let new_decs = self.new_decs.take();
710            self.new_work(
711                worker,
712                ProcessDecs::new(new_decs, self.counter.clone_with_decs()),
713            );
714        }
715        if !self.mark_objects.is_empty() {
716            let objects = self.mark_objects.take();
717            let w = LXRConcurrentTraceObjects::new(objects, mmtk);
718            if LAZY_DECREMENTS {
719                worker.add_work(WorkBucketStage::Unconstrained, w);
720            } else {
721                worker.scheduler().work_buckets[WorkBucketStage::ConcurrentResumable].add(w);
722            }
723        }
724    }
725
726    #[cold]
727    fn process_dead_object(
728        &mut self,
729        worker: &mut GCWorker<VM>,
730        o: ObjectReference,
731        lxr: &LXR<VM>,
732    ) -> bool {
733        if self.mark_dead_objects {
734            lxr.mark(o);
735        }
736        // Recursively decrease field ref counts
737        let tls = worker.tls.0;
738        o.iterate_fields::<VM, _>(tls, |slot| {
739            if let Some(x) = slot.load() {
740                // println!(" -- rec dec {:?}.{:?} -> {:?}", o, slot, x);
741                let rc = self.rc.count(x);
742                if rc != MAX_REF_COUNT && rc != 0 {
743                    self.recursive_dec(worker, x);
744                }
745                if self.mark_dead_objects && !lxr.is_marked(x) {
746                    if cfg!(any(feature = "sanity", debug_assertions)) {
747                        assert!(
748                            x.to_raw_address().is_mapped(),
749                            "Invalid object {:?}.{:?} -> {:?}: address is not mapped",
750                            o,
751                            slot,
752                            x
753                        );
754                    }
755                    self.mark_objects.push(x);
756                    if self.mark_objects.is_full() {
757                        self.flush(worker);
758                    }
759                }
760            }
761        });
762        let in_ix_space = lxr.immix_space.in_space(o);
763        if in_ix_space {
764            // Clear the VO bit if `o` is in the immix space.
765            // Note that if the object is in the LOS,
766            // the VO bit will be cleared in `LargeObjectSpace::release_object`.
767            #[cfg(feature = "vo_bit")]
768            crate::util::metadata::vo_bit::unset_vo_bit(o);
769
770            self.rc.unmark_straddle_object(o);
771        }
772        if RefCountHelper::<VM>::SANITY {
773            unsafe { o.to_raw_address().store(0xdeadusize) };
774        }
775        if in_ix_space {
776            let block = Block::containing(o);
777            lxr.add_to_possibly_dead_mature_blocks(block, false);
778            false
779        } else {
780            true
781        }
782    }
783
784    fn process_decs(&mut self, worker: &mut GCWorker<VM>, decs: &[ObjectReference], lxr: &LXR<VM>) {
785        for o in decs.iter() {
786            if self.rc.is_dead_or_stuck(*o)
787                || (self.mature_sweeping_in_progress && !lxr.is_marked(*o))
788            {
789                continue;
790            }
791            let o = if MATURE_EVACUATION && object_forwarding::is_forwarded::<VM>(*o) {
792                object_forwarding::read_forwarding_pointer::<VM>(*o)
793            } else {
794                *o
795            };
796            let mut dead = false;
797            let mut is_los = false;
798            let mut already_run = false;
799            let result = self.rc.clone().fetch_update(o, |c| {
800                if already_run {
801                    log::warn!("fetch_update is re-run! o: {o}");
802                } else {
803                    already_run = true;
804                }
805                if c == 1 && !dead {
806                    dead = true;
807                    is_los = self.process_dead_object(worker, o, lxr);
808                }
809                debug_assert!(c <= MAX_REF_COUNT);
810                if c == 0 || c == MAX_REF_COUNT {
811                    None /* sticky */
812                } else {
813                    Some(c - 1)
814                }
815            });
816            if result == Ok(1) && is_los {
817                lxr.los().rc_free(o);
818            }
819        }
820    }
821}
822
823impl<VM: VMBinding> GCWork<VM> for ProcessDecs<VM> {
824    fn do_work(&mut self, worker: &mut GCWorker<VM>, mmtk: &'static MMTK<VM>) {
825        let lxr = mmtk.get_plan().downcast_ref::<LXR<VM>>().unwrap();
826        self.mark_dead_objects = if LAZY_DECREMENTS {
827            lxr.concurrent_work_in_progress() && lxr.previous_pause() != Some(Pause::InitialMark)
828        } else {
829            lxr.concurrent_work_in_progress() && lxr.current_pause() != Some(Pause::InitialMark)
830        };
831        self.mature_sweeping_in_progress = if LAZY_DECREMENTS {
832            lxr.previous_pause() == Some(Pause::FinalMark)
833                || lxr.current_pause() == Some(Pause::Full)
834        } else {
835            lxr.current_pause() == Some(Pause::FinalMark)
836                || lxr.current_pause() == Some(Pause::Full)
837        };
838        if let Some(decs) = std::mem::take(&mut self.decs) {
839            self.process_decs(worker, &decs, lxr);
840        } else if let Some(decs) = std::mem::take(&mut self.decs_arc) {
841            self.process_decs(worker, &decs, lxr);
842        }
843        let mut decs = vec![];
844        while !self.new_decs.is_empty() {
845            decs.clear();
846            self.new_decs.swap(&mut decs);
847            self.process_decs(worker, &decs, lxr);
848        }
849        self.flush(worker);
850    }
851}
852
853pub struct CollectSlotRoots<VM: VMBinding> {
854    base: ProcessEdgesBase<VM>,
855}
856
857impl<VM: VMBinding> CollectSlotRoots<VM> {
858    pub fn new(
859        slots: Vec<VM::VMSlot>,
860        roots: bool,
861        mmtk: &'static MMTK<VM>,
862        bucket: WorkBucketStage,
863    ) -> Self {
864        debug_assert!(roots);
865        let base = ProcessEdgesBase::new(slots, roots, mmtk, bucket);
866        Self { base }
867    }
868}
869
870impl<VM: VMBinding> GCWork<VM> for CollectSlotRoots<VM> {
871    fn do_work(&mut self, worker: &mut GCWorker<VM>, _mmtk: &'static MMTK<VM>) {
872        self.set_worker(worker);
873        if !self.slots.is_empty() {
874            let lxr = self.mmtk().get_plan().downcast_ref::<LXR<VM>>().unwrap();
875            let roots = std::mem::take(&mut self.slots);
876            let mut w = ProcessIncs::<_, EDGE_KIND_ROOT>::new(roots, lxr);
877            w.root_kind = self.root_kind;
878            GCWork::do_work(&mut w, self.worker(), self.mmtk());
879        }
880    }
881}
882
883impl<VM: VMBinding> Deref for CollectSlotRoots<VM> {
884    type Target = ProcessEdgesBase<VM>;
885    fn deref(&self) -> &Self::Target {
886        &self.base
887    }
888}
889
890impl<VM: VMBinding> DerefMut for CollectSlotRoots<VM> {
891    fn deref_mut(&mut self) -> &mut Self::Target {
892        &mut self.base
893    }
894}
895
896/// Collects roots reported as objects rather than as slots. Node counterpart of
897/// [`CollectSlotRoots`].
898pub struct CollectNodeRoots<VM: VMBinding> {
899    nodes: Vec<ObjectReference>,
900    _p: PhantomData<VM>,
901}
902
903impl<VM: VMBinding> CollectNodeRoots<VM> {
904    pub fn new(nodes: Vec<ObjectReference>) -> Self {
905        Self {
906            nodes,
907            _p: PhantomData,
908        }
909    }
910}
911
912impl<VM: VMBinding> GCWork<VM> for CollectNodeRoots<VM> {
913    fn do_work(&mut self, worker: &mut GCWorker<VM>, mmtk: &'static MMTK<VM>) {
914        if !self.nodes.is_empty() {
915            let lxr = mmtk.get_plan().downcast_ref::<LXR<VM>>().unwrap();
916            let roots = std::mem::take(&mut self.nodes);
917            let mut w = ProcessIncs::<_, EDGE_KIND_ROOT>::new(vec![], lxr);
918            // `RootsWorkFactory` passes no `RootKind` for node roots; they are strong roots.
919            w.root_kind = Some(RootKind::Strong);
920            w.do_work_root_nodes(roots, worker, mmtk);
921        }
922    }
923}
mmtk/plan/lxr/gc_work/rc.rs

mmtk/plan/lxr/gc_work/
rc.rs
