mmtk/plan/lxr/gc_work/
rc.rs

1use super::super::LazySweepingJobsCounter;
2use super::super::SurvivalRatioPredictorLocal;
3use super::super::LXR;
4use super::super::{LAZY_DECREMENTS, MATURE_EVACUATION, NO_EVAC, NURSERY_EVACUATION};
5use super::tracing::LXRConcurrentTraceObjects;
6use super::tracing::LXRStopTheWorldProcessEdges;
7use super::ProcessEdgesBase;
8use crate::plan::VectorQueue;
9use crate::policy::immix::block::BlockState;
10use crate::scheduler::gc_work::RootKind;
11use crate::util::copy::CopySemantics;
12use crate::util::linear_scan::UnstraddlableRegion;
13use crate::util::metadata::side_metadata::SideMetadataSpec;
14use crate::util::rc::*;
15use crate::vm::slot::Slot;
16use crate::{
17    plan::concurrent::global::ConcurrentPlan,
18    plan::concurrent::Pause,
19    policy::{immix::block::Block, space::Space},
20    scheduler::{GCWork, GCWorker, WorkBucketStage},
21    util::{metadata::side_metadata, object_forwarding, ObjectReference},
22    vm::*,
23    MMTK,
24};
25use atomic::Ordering;
26use std::ops::{Deref, DerefMut};
27use std::sync::Arc;
28
29pub struct ProcessIncs<VM: VMBinding, const KIND: EdgeKind> {
30    /// Increments to process
31    incs: Vec<VM::VMSlot>,
32    /// Recursively generated new increments
33    new_incs: VectorQueue<VM::VMSlot>,
34    new_incs_count: u32,
35    pause: Pause,
36    in_cm: bool,
37    no_evac: bool,
38    pub root_kind: Option<RootKind>,
39    depth: u32,
40    lxr: &'static LXR<VM>,
41    rc: RefCountHelper<VM>,
42    survival_ratio_predictor_local: SurvivalRatioPredictorLocal,
43}
44
45unsafe impl<VM: VMBinding, const KIND: EdgeKind> Send for ProcessIncs<VM, KIND> {}
46
47impl<VM: VMBinding, const KIND: EdgeKind> ProcessIncs<VM, KIND> {
48    const CAPACITY: usize = 1024;
49    const UNLOG_BITS: SideMetadataSpec = *VM::VMObjectModel::GLOBAL_FIELD_UNLOG_BIT_SPEC
50        .as_spec()
51        .extract_side_spec();
52
53    fn __default(lxr: &'static LXR<VM>) -> Self {
54        Self {
55            incs: vec![],
56            new_incs: VectorQueue::default(),
57            new_incs_count: 0,
58            lxr,
59            pause: Pause::RefCount,
60            in_cm: false,
61            no_evac: false,
62            depth: 1,
63            rc: RefCountHelper::NEW,
64            root_kind: None,
65            survival_ratio_predictor_local: SurvivalRatioPredictorLocal::default(),
66        }
67    }
68
69    fn add_new_slot(&mut self, worker: &mut GCWorker<VM>, s: VM::VMSlot) {
70        self.new_incs.push(s);
71        self.new_incs_count += 1;
72        if self.new_incs_count as usize >= Self::CAPACITY {
73            self.flush(worker);
74        }
75    }
76
77    pub fn new(incs: Vec<VM::VMSlot>, lxr: &'static LXR<VM>) -> Self {
78        Self {
79            incs,
80            ..Self::__default(lxr)
81        }
82    }
83
84    fn promote(
85        &mut self,
86        worker: &mut GCWorker<VM>,
87        o: ObjectReference,
88        copied: bool,
89        los: bool,
90        depth: u32,
91    ) {
92        let size = o.get_size::<VM>();
93
94        if !los {
95            let block = Block::containing(o);
96            let in_nursery_block = block.get_state() == BlockState::Nursery;
97            if !copied && in_nursery_block {
98                block.set_as_in_place_promoted();
99            }
100            self.rc.promote_with_size(o, size);
101            if copied {
102                self.survival_ratio_predictor_local
103                    .record_copied_promotion(size);
104            }
105        } else {
106            // println!("promote los {:?} {}", o, self.immix().is_marked(o));
107        }
108        // Don't mark copied objects in initial mark pause. The concurrent marker will do it (and can also resursively mark the old objects).
109        if self.in_cm || self.pause == Pause::FinalMark {
110            debug_assert!(self.lxr.is_marked(o), "{:?} is not marked", o);
111        }
112        self.scan_nursery_object(worker, o, los, !copied, depth, size);
113    }
114
115    fn record_mature_evac_remset2(
116        &mut self,
117        slot_in_defrag: bool,
118        s: VM::VMSlot,
119        o: ObjectReference,
120    ) {
121        if !(MATURE_EVACUATION && (self.in_cm || self.pause == Pause::FinalMark)) {
122            return;
123        }
124        if !slot_in_defrag && self.lxr.in_defrag(o) {
125            self.lxr.mature_evac_remset.record(s, o, self.lxr);
126        }
127    }
128
129    fn record_mature_evac_remset(&mut self, s: VM::VMSlot, o: ObjectReference) {
130        if !(MATURE_EVACUATION && (self.in_cm || self.pause == Pause::FinalMark)) {
131            return;
132        }
133        self.record_mature_evac_remset2(self.lxr.address_in_defrag(s.to_address()), s, o);
134    }
135
136    fn scan_nursery_object(
137        &mut self,
138        worker: &mut GCWorker<VM>,
139        o: ObjectReference,
140        los: bool,
141        in_place_promotion: bool,
142        _depth: u32,
143        size: usize,
144    ) {
145        let heap_bytes_per_unlog_byte = if VM::VMObjectModel::COMPRESSED_PTR_ENABLED {
146            32usize
147        } else {
148            64
149        };
150        if los {
151            let start =
152                side_metadata::address_to_meta_address(&Self::UNLOG_BITS, o.to_raw_address())
153                    .to_mut_ptr::<u8>();
154            let limit = side_metadata::address_to_meta_address(
155                &Self::UNLOG_BITS,
156                (o.to_raw_address() + size).align_up(heap_bytes_per_unlog_byte),
157            )
158            .to_mut_ptr::<u8>();
159            unsafe {
160                let bytes = limit.offset_from(start) as usize;
161                std::ptr::write_bytes(start, 0xffu8, bytes);
162            }
163            o.to_raw_address().unlog_field_relaxed::<VM>();
164        } else if in_place_promotion {
165            let header_size = if VM::VMObjectModel::COMPRESSED_PTR_ENABLED {
166                12usize
167            } else {
168                16
169            };
170            let step = heap_bytes_per_unlog_byte << 2;
171            let end = o.to_raw_address() + size;
172            let aligned_end = end.align_up(step);
173            let cursor = o.to_raw_address() + header_size;
174            let mut cursor = cursor.align_down(step);
175            let mut meta = side_metadata::address_to_meta_address(&Self::UNLOG_BITS, cursor);
176            while cursor < aligned_end {
177                unsafe { meta.store(0xffffffffu32) }
178                meta += 4usize;
179                cursor += step;
180            }
181        };
182        let obj_in_defrag = !los && Block::in_defrag_block(o);
183        let tls = worker.tls.0;
184        o.iterate_fields::<VM, _>(tls, |slot| {
185            let Some(target) = slot.load() else {
186                return;
187            };
188            debug_assert!(
189                target.to_raw_address().is_mapped(),
190                "Unmapped obj {:?}.{:?} -> {:?}",
191                o,
192                slot,
193                target
194            );
195            debug_assert!(
196                target.is_in_any_space(),
197                "Unmapped obj {:?}.{:?} -> {:?}",
198                o,
199                slot,
200                target
201            );
202            let rc = self.rc.count(target);
203            if rc == 0 {
204                self.add_new_slot(worker, slot);
205            } else {
206                if rc != crate::util::rc::MAX_REF_COUNT {
207                    let _ = self.rc.inc(target);
208                }
209                self.record_mature_evac_remset2(obj_in_defrag, slot, target);
210            }
211        });
212    }
213
214    #[cold]
215    fn flush(&mut self, worker: &mut GCWorker<VM>) {
216        if !self.new_incs.is_empty() {
217            let new_incs = self.new_incs.take();
218            let mut w = ProcessIncs::<VM, EDGE_KIND_NURSERY>::new(new_incs, self.lxr);
219            w.depth += 1;
220            worker.add_work(WorkBucketStage::Unconstrained, w);
221        }
222        self.new_incs_count = 0;
223    }
224
225    /// Return true if the object's ref count is incremented and the count was zero before the increment
226    fn inc(&self, o: ObjectReference) -> bool {
227        self.rc.inc(o) == Ok(0)
228    }
229
230    fn dont_evacuate(&self, o: ObjectReference, los: bool) -> bool {
231        if los {
232            return true;
233        }
234        // Skip mature object
235        if self.rc.count(o) != 0 {
236            return true;
237        }
238        // Skip recycled lines
239        if Block::containing(o).get_state() != BlockState::Nursery {
240            return true;
241        }
242        if cfg!(debug_assertions) {
243            let cls = unsafe { (o.to_raw_address() + 8usize).load::<u32>() };
244            assert!(cls != 0, "ERROR {:?} rc={}", o, self.rc.count(o));
245        }
246        false
247    }
248
249    fn process_inc_and_evacuate(
250        &mut self,
251        worker: &mut GCWorker<VM>,
252        o: ObjectReference,
253        depth: u32,
254    ) -> ObjectReference {
255        let los = self.lxr.los().in_space(o);
256        if NURSERY_EVACUATION && !los && object_forwarding::is_forwarded_or_being_forwarded::<VM>(o)
257        {
258            while object_forwarding::is_being_forwarded::<VM>(o) {
259                std::hint::spin_loop();
260            }
261            let new = if object_forwarding::is_forwarded::<VM>(o) {
262                object_forwarding::read_forwarding_pointer::<VM>(o)
263            } else {
264                o
265            };
266            let promoted = self.inc(new);
267            if promoted && new == o {
268                self.promote(worker, o, false, los, depth);
269            }
270            return new;
271        }
272        if !NURSERY_EVACUATION || self.dont_evacuate(o, los) {
273            if self.inc(o) {
274                self.promote(worker, o, false, los, depth);
275            }
276            return o;
277        }
278        let forwarding_status = object_forwarding::attempt_to_forward::<VM>(o);
279        if object_forwarding::state_is_forwarded_or_being_forwarded(forwarding_status) {
280            // Object is moved to a new location.
281            let new = object_forwarding::spin_and_get_forwarded_object::<VM>(o, forwarding_status);
282            self.inc(new);
283            new
284        } else {
285            let is_nursery = self.rc.count(o) == 0;
286            if is_nursery && !self.no_evac {
287                // Evacuate the object
288                let new = object_forwarding::try_forward_object::<VM>(
289                    o,
290                    CopySemantics::DefaultCopy,
291                    worker.get_copy_context_mut(),
292                    |_new| {
293                        #[cfg(feature = "vo_bit")]
294                        {
295                            // Set the VO bit of the new object.
296                            crate::util::metadata::vo_bit::set_vo_bit(_new);
297                            // Clear the VO bit of the old object.
298                            // Note that sweeping can also clear the VO bit when the line is freed,
299                            // but no RC inc/dec should be performed on the old object from now on.
300                            // We clear it eagerly to detect inc/dec errors.
301                            crate::util::metadata::vo_bit::unset_vo_bit(o);
302                        }
303                    },
304                );
305                if let Some(new) = new {
306                    self.inc(new);
307                    self.promote(worker, new, true, false, depth);
308                    new
309                } else {
310                    warn!("to-space overflow");
311                    // Object is not moved.
312                    let promoted = self.inc(o);
313                    object_forwarding::clear_forwarding_bits::<VM>(o);
314                    if promoted {
315                        self.promote(worker, o, false, los, depth);
316                    }
317                    NO_EVAC.store(true, Ordering::Relaxed);
318                    self.no_evac = true;
319                    o
320                }
321            } else {
322                // Object is not moved.
323                let promoted = self.inc(o);
324                object_forwarding::clear_forwarding_bits::<VM>(o);
325                if promoted {
326                    self.promote(worker, o, false, los, depth);
327                }
328                o
329            }
330        }
331    }
332
333    /// Return `None` if the increment of the slot should be delayed
334    fn unlog_and_load_rc_object<const K: EdgeKind>(
335        &mut self,
336        s: VM::VMSlot,
337    ) -> Option<ObjectReference> {
338        let o = s.load();
339        // unlog slot
340        if K == EDGE_KIND_MATURE {
341            s.to_address().unlog_field_relaxed::<VM>();
342        }
343        o
344    }
345
346    fn process_slot<const K: EdgeKind>(
347        &mut self,
348        worker: &mut GCWorker<VM>,
349        s: VM::VMSlot,
350        depth: u32,
351        add_root_to_remset: bool,
352    ) -> Option<ObjectReference> {
353        let o = match self.unlog_and_load_rc_object::<K>(s) {
354            Some(o) => o,
355            _ => {
356                return None;
357            }
358        };
359        // println!(" - inc {:?}: {:?} rc={}", s, o, self.rc.count(o));
360        let new = self.process_inc_and_evacuate(worker, o, depth);
361        // Put this into remset if this is a mature slot, or a weak root
362        if K != EDGE_KIND_ROOT || add_root_to_remset {
363            self.record_mature_evac_remset(s, new);
364        }
365        if new != o {
366            s.store(new)
367        }
368        Some(new)
369    }
370
371    fn process_incs<const K: EdgeKind>(
372        &mut self,
373        worker: &mut GCWorker<VM>,
374        mut incs: AddressBuffer<'_, VM::VMSlot>,
375        depth: u32,
376        add_root_to_remset: bool,
377    ) -> Option<Vec<ObjectReference>> {
378        if K == EDGE_KIND_ROOT {
379            let roots = incs.as_mut_ptr() as *mut ObjectReference;
380            let mut num_roots = 0usize;
381            for s in incs.iter() {
382                if let Some(new) = self.process_slot::<K>(worker, *s, depth, add_root_to_remset) {
383                    unsafe {
384                        roots.add(num_roots).write(new);
385                    }
386                    num_roots += 1;
387                }
388            }
389            if num_roots != 0 {
390                let cap = incs.capacity();
391                std::mem::forget(incs); // roots references incs now. we dont need incs.
392                let roots =
393                    unsafe { Vec::<ObjectReference>::from_raw_parts(roots, num_roots, cap) };
394                Some(roots)
395            } else {
396                None
397            }
398        } else {
399            for s in incs.iter() {
400                self.process_slot::<K>(worker, *s, depth, false);
401            }
402            None
403        }
404    }
405}
406
407pub type EdgeKind = u8;
408pub const EDGE_KIND_ROOT: u8 = 0;
409pub const EDGE_KIND_NURSERY: u8 = 1;
410pub const EDGE_KIND_MATURE: u8 = 2;
411
412enum AddressBuffer<'a, S: Slot> {
413    Owned(Vec<S>),
414    Ref(&'a mut Vec<S>),
415}
416
417impl<S: Slot> Deref for AddressBuffer<'_, S> {
418    type Target = Vec<S>;
419    fn deref(&self) -> &Self::Target {
420        match self {
421            Self::Owned(x) => x,
422            Self::Ref(x) => x,
423        }
424    }
425}
426
427impl<S: Slot> DerefMut for AddressBuffer<'_, S> {
428    fn deref_mut(&mut self) -> &mut Self::Target {
429        match self {
430            Self::Owned(x) => x,
431            Self::Ref(x) => x,
432        }
433    }
434}
435
436impl<VM: VMBinding, const KIND: EdgeKind> GCWork<VM> for ProcessIncs<VM, KIND> {
437    fn do_work(&mut self, worker: &mut GCWorker<VM>, mmtk: &'static MMTK<VM>) {
438        self.lxr = mmtk.get_plan().downcast_ref::<LXR<VM>>().unwrap();
439        self.pause = self.lxr.current_pause().unwrap();
440        self.in_cm = self.lxr.concurrent_work_in_progress();
441        if NO_EVAC.load(Ordering::Relaxed) {
442            self.no_evac = true;
443        } else {
444            let over_space = mmtk.get_plan().get_used_pages()
445                - mmtk.get_plan().get_collection_reserved_pages()
446                > mmtk.get_plan().get_total_pages();
447            if over_space {
448                self.no_evac = true;
449                NO_EVAC.store(true, Ordering::Relaxed);
450            }
451        }
452        // Process main buffer
453        let root_slots = if KIND == EDGE_KIND_ROOT
454            && (self.pause == Pause::FinalMark || self.pause == Pause::Full)
455        {
456            self.incs.clone()
457        } else {
458            vec![]
459        };
460        let roots = {
461            let incs = std::mem::take(&mut self.incs);
462            self.process_incs::<KIND>(worker, AddressBuffer::Owned(incs), self.depth, false)
463        };
464        if let Some(roots) = roots {
465            if self.lxr.cm_enabled()
466                && self.pause == Pause::InitialMark
467                && !self.root_kind.unwrap().should_skip_mark_and_decs()
468            {
469                if cfg!(any(feature = "sanity", debug_assertions)) {
470                    for r in &roots {
471                        assert!(
472                            r.to_raw_address().is_mapped(),
473                            "Invalid object {:?}: address is not mapped",
474                            r
475                        );
476                    }
477                }
478                worker.scheduler().work_buckets[WorkBucketStage::ConcurrentResumable]
479                    .add(LXRConcurrentTraceObjects::new(roots.clone(), mmtk));
480            }
481            if self.pause == Pause::FinalMark || self.pause == Pause::Full {
482                if !root_slots.is_empty() && self.root_kind != Some(RootKind::Weak) {
483                    if self.pause == Pause::FinalMark {
484                        let mut w = LXRStopTheWorldProcessEdges::<_, false>::new(
485                            root_slots,
486                            true,
487                            mmtk,
488                            WorkBucketStage::Closure,
489                        );
490                        w.root_kind = self.root_kind;
491                        worker.add_work(WorkBucketStage::Closure, w)
492                    } else {
493                        let mut w = LXRStopTheWorldProcessEdges::<_, true>::new(
494                            root_slots,
495                            true,
496                            mmtk,
497                            WorkBucketStage::Closure,
498                        );
499                        w.root_kind = self.root_kind;
500                        worker.add_work(WorkBucketStage::Closure, w)
501                    };
502                }
503            } else if !self.root_kind.unwrap().should_skip_decs() {
504                self.lxr.curr_roots.read().unwrap().push(roots);
505            }
506        }
507        // Process recursively generated buffer
508        let mut depth = self.depth;
509        let mut incs = vec![];
510        const ACTIVE_PACKET_SPLIT: bool = false;
511        while !self.new_incs.is_empty() {
512            self.new_incs_count = 0;
513            depth += 1;
514            incs.clear();
515            self.new_incs.swap(&mut incs);
516            if ACTIVE_PACKET_SPLIT && depth >= 16 && incs.len() > 1 {
517                let (a, b) = incs.split_at(incs.len() / 2);
518                let mut w = ProcessIncs::<VM, EDGE_KIND_NURSERY>::new(b.to_vec(), self.lxr);
519                w.depth = depth;
520                worker.add_work(WorkBucketStage::Unconstrained, w);
521                incs = a.to_vec();
522            }
523            if !incs.is_empty() {
524                self.process_incs::<EDGE_KIND_NURSERY>(
525                    worker,
526                    AddressBuffer::Ref(&mut incs),
527                    depth,
528                    false,
529                );
530            }
531        }
532        self.survival_ratio_predictor_local.sync();
533    }
534}
535
536pub struct ProcessDecs<VM: VMBinding> {
537    /// Decrements to process
538    decs: Option<Vec<ObjectReference>>,
539    decs_arc: Option<Arc<Vec<ObjectReference>>>,
540    /// Recursively generated new decrements
541    new_decs: VectorQueue<ObjectReference>,
542    counter: LazySweepingJobsCounter,
543    mark_objects: VectorQueue<ObjectReference>,
544    mark_dead_objects: bool,
545    mature_sweeping_in_progress: bool,
546    rc: RefCountHelper<VM>,
547}
548
549impl<VM: VMBinding> ProcessDecs<VM> {
550    pub fn new(decs: Vec<ObjectReference>, counter: LazySweepingJobsCounter) -> Self {
551        Self {
552            decs: Some(decs),
553            decs_arc: None,
554            new_decs: VectorQueue::default(),
555            counter,
556            mark_objects: VectorQueue::default(),
557            mark_dead_objects: false,
558            mature_sweeping_in_progress: false,
559            rc: RefCountHelper::NEW,
560        }
561    }
562
563    pub fn new_arc(decs: Arc<Vec<ObjectReference>>, counter: LazySweepingJobsCounter) -> Self {
564        Self {
565            decs: None,
566            decs_arc: Some(decs),
567            new_decs: VectorQueue::default(),
568            counter,
569            mark_objects: VectorQueue::default(),
570            mark_dead_objects: false,
571            mature_sweeping_in_progress: false,
572            rc: RefCountHelper::NEW,
573        }
574    }
575
576    fn recursive_dec(&mut self, worker: &mut GCWorker<VM>, o: ObjectReference) {
577        self.new_decs.push(o);
578        if self.new_decs.is_full() {
579            self.flush(worker)
580        }
581    }
582
583    fn new_work(&self, worker: &mut GCWorker<VM>, w: ProcessDecs<VM>) {
584        worker.add_work(WorkBucketStage::Unconstrained, w);
585    }
586
587    fn flush(&mut self, worker: &mut GCWorker<VM>) {
588        let mmtk = worker.mmtk;
589        if !self.new_decs.is_empty() {
590            let new_decs = self.new_decs.take();
591            self.new_work(
592                worker,
593                ProcessDecs::new(new_decs, self.counter.clone_with_decs()),
594            );
595        }
596        if !self.mark_objects.is_empty() {
597            let objects = self.mark_objects.take();
598            let w = LXRConcurrentTraceObjects::new(objects, mmtk);
599            if LAZY_DECREMENTS {
600                worker.add_work(WorkBucketStage::Unconstrained, w);
601            } else {
602                worker.scheduler().work_buckets[WorkBucketStage::ConcurrentResumable].add(w);
603            }
604        }
605    }
606
607    #[cold]
608    fn process_dead_object(
609        &mut self,
610        worker: &mut GCWorker<VM>,
611        o: ObjectReference,
612        lxr: &LXR<VM>,
613    ) -> bool {
614        if self.mark_dead_objects {
615            lxr.mark(o);
616        }
617        // Recursively decrease field ref counts
618        let tls = worker.tls.0;
619        o.iterate_fields::<VM, _>(tls, |slot| {
620            if let Some(x) = slot.load() {
621                // println!(" -- rec dec {:?}.{:?} -> {:?}", o, slot, x);
622                let rc = self.rc.count(x);
623                if rc != MAX_REF_COUNT && rc != 0 {
624                    self.recursive_dec(worker, x);
625                }
626                if self.mark_dead_objects && !lxr.is_marked(x) {
627                    if cfg!(any(feature = "sanity", debug_assertions)) {
628                        assert!(
629                            x.to_raw_address().is_mapped(),
630                            "Invalid object {:?}.{:?} -> {:?}: address is not mapped",
631                            o,
632                            slot,
633                            x
634                        );
635                    }
636                    self.mark_objects.push(x);
637                    if self.mark_objects.is_full() {
638                        self.flush(worker);
639                    }
640                }
641            }
642        });
643        let in_ix_space = lxr.immix_space.in_space(o);
644        if in_ix_space {
645            // Clear the VO bit if `o` is in the immix space.
646            // Note that if the object is in the LOS,
647            // the VO bit will be cleared in `LargeObjectSpace::release_object`.
648            #[cfg(feature = "vo_bit")]
649            crate::util::metadata::vo_bit::unset_vo_bit(o);
650
651            self.rc.unmark_straddle_object(o);
652        }
653        if RefCountHelper::<VM>::SANITY {
654            unsafe { o.to_raw_address().store(0xdeadusize) };
655        }
656        if in_ix_space {
657            let block = Block::containing(o);
658            lxr.add_to_possibly_dead_mature_blocks(block, false);
659            false
660        } else {
661            true
662        }
663    }
664
665    fn process_decs(&mut self, worker: &mut GCWorker<VM>, decs: &[ObjectReference], lxr: &LXR<VM>) {
666        for o in decs.iter() {
667            if self.rc.is_dead_or_stuck(*o)
668                || (self.mature_sweeping_in_progress && !lxr.is_marked(*o))
669            {
670                continue;
671            }
672            let o = if MATURE_EVACUATION && object_forwarding::is_forwarded::<VM>(*o) {
673                object_forwarding::read_forwarding_pointer::<VM>(*o)
674            } else {
675                *o
676            };
677            let mut dead = false;
678            let mut is_los = false;
679            let mut already_run = false;
680            let result = self.rc.clone().fetch_update(o, |c| {
681                if already_run {
682                    log::warn!("fetch_update is re-run! o: {o}");
683                } else {
684                    already_run = true;
685                }
686                if c == 1 && !dead {
687                    dead = true;
688                    is_los = self.process_dead_object(worker, o, lxr);
689                }
690                debug_assert!(c <= MAX_REF_COUNT);
691                if c == 0 || c == MAX_REF_COUNT {
692                    None /* sticky */
693                } else {
694                    Some(c - 1)
695                }
696            });
697            if result == Ok(1) && is_los {
698                lxr.los().rc_free(o);
699            }
700        }
701    }
702}
703
704impl<VM: VMBinding> GCWork<VM> for ProcessDecs<VM> {
705    fn do_work(&mut self, worker: &mut GCWorker<VM>, mmtk: &'static MMTK<VM>) {
706        let lxr = mmtk.get_plan().downcast_ref::<LXR<VM>>().unwrap();
707        self.mark_dead_objects = if LAZY_DECREMENTS {
708            lxr.concurrent_work_in_progress() && lxr.previous_pause() != Some(Pause::InitialMark)
709        } else {
710            lxr.concurrent_work_in_progress() && lxr.current_pause() != Some(Pause::InitialMark)
711        };
712        self.mature_sweeping_in_progress = if LAZY_DECREMENTS {
713            lxr.previous_pause() == Some(Pause::FinalMark)
714                || lxr.current_pause() == Some(Pause::Full)
715        } else {
716            lxr.current_pause() == Some(Pause::FinalMark)
717                || lxr.current_pause() == Some(Pause::Full)
718        };
719        if let Some(decs) = std::mem::take(&mut self.decs) {
720            self.process_decs(worker, &decs, lxr);
721        } else if let Some(decs) = std::mem::take(&mut self.decs_arc) {
722            self.process_decs(worker, &decs, lxr);
723        }
724        let mut decs = vec![];
725        while !self.new_decs.is_empty() {
726            decs.clear();
727            self.new_decs.swap(&mut decs);
728            self.process_decs(worker, &decs, lxr);
729        }
730        self.flush(worker);
731    }
732}
733
734pub struct CollectRoots<VM: VMBinding> {
735    base: ProcessEdgesBase<VM>,
736}
737
738impl<VM: VMBinding> CollectRoots<VM> {
739    pub fn new(
740        slots: Vec<VM::VMSlot>,
741        roots: bool,
742        mmtk: &'static MMTK<VM>,
743        bucket: WorkBucketStage,
744    ) -> Self {
745        debug_assert!(roots);
746        let base = ProcessEdgesBase::new(slots, roots, mmtk, bucket);
747        Self { base }
748    }
749}
750
751impl<VM: VMBinding> GCWork<VM> for CollectRoots<VM> {
752    fn do_work(&mut self, worker: &mut GCWorker<VM>, _mmtk: &'static MMTK<VM>) {
753        self.set_worker(worker);
754        if !self.slots.is_empty() {
755            let lxr = self.mmtk().get_plan().downcast_ref::<LXR<VM>>().unwrap();
756            let roots = std::mem::take(&mut self.slots);
757            let mut w = ProcessIncs::<_, EDGE_KIND_ROOT>::new(roots, lxr);
758            w.root_kind = self.root_kind;
759            GCWork::do_work(&mut w, self.worker(), self.mmtk());
760        }
761    }
762}
763
764impl<VM: VMBinding> Deref for CollectRoots<VM> {
765    type Target = ProcessEdgesBase<VM>;
766    fn deref(&self) -> &Self::Target {
767        &self.base
768    }
769}
770
771impl<VM: VMBinding> DerefMut for CollectRoots<VM> {
772    fn deref_mut(&mut self) -> &mut Self::Target {
773        &mut self.base
774    }
775}
mmtk/plan/lxr/gc_work/rc.rs

mmtk/plan/lxr/gc_work/
rc.rs
