mmtk/plan/lxr/gc_work/
tracing.rs

1use super::super::LXR;
2use super::ProcessEdgesBase;
3use crate::plan::concurrent::Pause;
4use crate::plan::VectorQueue;
5use crate::policy::immix::block::Block;
6use crate::policy::space::Space;
7use crate::scheduler::RootKind;
8use crate::util::copy::CopySemantics;
9use crate::util::linear_scan::UnstraddlableRegion;
10use crate::util::rc::RefCountHelper;
11use crate::util::{ObjectReference, VMThread};
12use crate::vm::slot::Slot;
13use crate::{
14    plan::ObjectQueue,
15    scheduler::{GCWork, GCWorker, WorkBucketStage},
16    vm::*,
17    MMTK,
18};
19use atomic::Ordering;
20use std::ops::{Deref, DerefMut};
21use std::sync::Arc;
22
23pub struct LXRConcurrentTraceObjects<VM: VMBinding> {
24    plan: &'static LXR<VM>,
25    // objects to mark and scan
26    objects: Option<Vec<ObjectReference>>,
27    objects_arc: Option<Arc<Vec<ObjectReference>>>,
28    // recursively generated objects
29    next_objects: VectorQueue<ObjectReference>,
30    rc: RefCountHelper<VM>,
31    worker: *mut GCWorker<VM>,
32}
33
34impl<VM: VMBinding> LXRConcurrentTraceObjects<VM> {
35    const SATB_BUFFER_SIZE: usize = 8192;
36
37    pub fn new(objects: Vec<ObjectReference>, mmtk: &'static MMTK<VM>) -> Self {
38        let plan = mmtk.get_plan().downcast_ref::<LXR<VM>>().unwrap();
39        super::super::NUM_CONCURRENT_TRACING_PACKETS.fetch_add(1, Ordering::SeqCst);
40        Self {
41            plan,
42            objects: Some(objects),
43            objects_arc: None,
44            next_objects: VectorQueue::default(),
45            rc: RefCountHelper::NEW,
46            worker: std::ptr::null_mut(),
47        }
48    }
49
50    pub fn new_arc(objects: Arc<Vec<ObjectReference>>, mmtk: &'static MMTK<VM>) -> Self {
51        let plan = mmtk.get_plan().downcast_ref::<LXR<VM>>().unwrap();
52        super::super::NUM_CONCURRENT_TRACING_PACKETS.fetch_add(1, Ordering::SeqCst);
53        Self {
54            plan,
55            objects: None,
56            objects_arc: Some(objects),
57            next_objects: VectorQueue::default(),
58            rc: RefCountHelper::NEW,
59            worker: std::ptr::null_mut(),
60        }
61    }
62
63    #[cold]
64    fn flush(&mut self) {
65        if !self.next_objects.is_empty() {
66            let objects = self.next_objects.take();
67            let worker = unsafe { &mut *self.worker };
68            debug_assert!(self.plan.cm_enabled());
69            let w = Self::new(objects, worker.mmtk);
70            worker.add_work(WorkBucketStage::ConcurrentResumable, w);
71        }
72    }
73
74    fn trace_object(&mut self, object: ObjectReference) -> ObjectReference {
75        if self.rc.count(object) == 0 {
76            return object;
77        }
78        if self.plan.immix_space.in_space(object) {
79            self.plan
80                .immix_space
81                .trace_object_without_moving_rc(self, object);
82        } else {
83            self.plan.los().trace_object(self, object);
84        }
85        object
86    }
87
88    fn trace_objects(&mut self, objects: &[ObjectReference]) {
89        for o in objects {
90            self.trace_object(*o);
91        }
92    }
93
94    fn scan_and_enqueue<const CHECK_REMSET: bool>(&mut self, object: ObjectReference) {
95        object.iterate_fields::<VM, _>(unsafe { (*self.worker).tls }.0, |s| {
96            let Some(t) = s.load() else {
97                return;
98            };
99            if super::super::MATURE_EVACUATION && CHECK_REMSET && self.plan.in_defrag(t) {
100                self.plan.mature_evac_remset.record(s, t, self.plan);
101            }
102            self.next_objects.push(t);
103            if self.next_objects.len() > Self::SATB_BUFFER_SIZE {
104                self.flush();
105            }
106        });
107    }
108}
109
110impl<VM: VMBinding> ObjectQueue for LXRConcurrentTraceObjects<VM> {
111    fn enqueue(&mut self, object: ObjectReference) {
112        if cfg!(feature = "sanity") {
113            assert!(
114                object.to_raw_address().is_mapped(),
115                "Invalid obj {:?}: address is not mapped",
116                object
117            );
118        }
119        let should_check_remset = !self.plan.in_defrag(object);
120        if should_check_remset {
121            self.scan_and_enqueue::<true>(object)
122        } else {
123            self.scan_and_enqueue::<false>(object)
124        }
125    }
126}
127
128unsafe impl<VM: VMBinding> Send for LXRConcurrentTraceObjects<VM> {}
129
130impl<VM: VMBinding> GCWork<VM> for LXRConcurrentTraceObjects<VM> {
131    fn do_work(&mut self, worker: &mut GCWorker<VM>, mmtk: &'static MMTK<VM>) {
132        self.worker = worker;
133        debug_assert!(!mmtk.scheduler.work_buckets[WorkBucketStage::RCProcessIncs].is_open());
134        // mark objects
135        if let Some(objects) = self.objects.take() {
136            self.trace_objects(&objects)
137        } else if let Some(objects) = self.objects_arc.take() {
138            self.trace_objects(&objects)
139        }
140        let pause_opt = self.plan.current_pause();
141        if pause_opt == Some(Pause::FinalMark) || pause_opt.is_none() {
142            let mut next_objects = vec![];
143            while !self.next_objects.is_empty() {
144                let pause_opt = self.plan.current_pause();
145                if !(pause_opt == Some(Pause::FinalMark) || pause_opt.is_none()) {
146                    break;
147                }
148                next_objects.clear();
149                self.next_objects.swap(&mut next_objects);
150                self.trace_objects(&next_objects);
151            }
152        }
153        self.flush();
154        // CM: Decrease counter
155        super::super::NUM_CONCURRENT_TRACING_PACKETS.fetch_sub(1, Ordering::SeqCst);
156        debug_assert!(!mmtk.scheduler.work_buckets[WorkBucketStage::RCProcessIncs].is_open());
157    }
158}
159
160pub struct ProcessModBufSATB {
161    nodes: Option<Vec<ObjectReference>>,
162    nodes_arc: Option<Arc<Vec<ObjectReference>>>,
163}
164
165impl ProcessModBufSATB {
166    pub fn new(nodes: Vec<ObjectReference>) -> Self {
167        // super::NUM_CONCURRENT_TRACING_PACKETS.fetch_add(1, Ordering::SeqCst);
168        Self {
169            nodes: Some(nodes),
170            nodes_arc: None,
171        }
172    }
173    pub fn new_arc(nodes: Arc<Vec<ObjectReference>>) -> Self {
174        // super::NUM_CONCURRENT_TRACING_PACKETS.fetch_add(1, Ordering::SeqCst);
175        Self {
176            nodes: None,
177            nodes_arc: Some(nodes),
178        }
179    }
180}
181
182impl<VM: VMBinding> GCWork<VM> for ProcessModBufSATB {
183    fn do_work(&mut self, worker: &mut GCWorker<VM>, mmtk: &'static MMTK<VM>) {
184        let mut w = if let Some(nodes) = self.nodes.take() {
185            if nodes.is_empty() {
186                return;
187            }
188            if cfg!(any(feature = "sanity", debug_assertions)) {
189                for o in &nodes {
190                    assert!(
191                        o.to_raw_address().is_mapped(),
192                        "Invalid object {:?}: address is not mapped",
193                        o
194                    );
195                }
196            }
197            LXRConcurrentTraceObjects::new(nodes, mmtk)
198        } else if let Some(nodes) = self.nodes_arc.take() {
199            if nodes.is_empty() {
200                return;
201            }
202            if cfg!(any(feature = "sanity", debug_assertions)) {
203                for o in &*nodes {
204                    assert!(
205                        o.to_raw_address().is_mapped(),
206                        "Invalid object {:?}: address is not mapped",
207                        o
208                    );
209                }
210            }
211            LXRConcurrentTraceObjects::new_arc(nodes, mmtk)
212        } else {
213            return;
214        };
215
216        let current_pause = mmtk
217            .get_plan()
218            .downcast_ref::<LXR<VM>>()
219            .unwrap()
220            .current_pause();
221        if current_pause != Some(Pause::FinalMark) {
222            worker.scheduler().work_buckets[WorkBucketStage::ConcurrentResumable].add(w);
223        } else {
224            GCWork::do_work(&mut w, worker, mmtk);
225        }
226    }
227}
228
229pub struct LXRStopTheWorldProcessEdges<VM: VMBinding, const FULL_GC: bool> {
230    lxr: &'static LXR<VM>,
231    pause: Pause,
232    base: ProcessEdgesBase<VM>,
233    forwarded_roots: Vec<ObjectReference>,
234    next_slots: VectorQueue<VM::VMSlot>,
235    next_slot_count: u32,
236    remset_recorded_slots: bool,
237    should_record_forwarded_roots: bool,
238}
239
240impl<VM: VMBinding, const FULL_GC: bool> LXRStopTheWorldProcessEdges<VM, FULL_GC> {
241    const OVERWRITE_REFERENCE: bool = super::super::MATURE_EVACUATION;
242
243    pub fn new_remset(slots: Vec<VM::VMSlot>, mmtk: &'static MMTK<VM>) -> Self {
244        let mut me = Self::new(slots, false, mmtk, WorkBucketStage::Closure);
245        me.remset_recorded_slots = true;
246        me
247    }
248
249    pub fn new(
250        slots: Vec<VM::VMSlot>,
251        roots: bool,
252        mmtk: &'static MMTK<VM>,
253        bucket: WorkBucketStage,
254    ) -> Self {
255        let base = ProcessEdgesBase::new(slots, roots, mmtk, bucket);
256        let lxr = base.plan().downcast_ref::<LXR<VM>>().unwrap();
257        Self {
258            lxr,
259            base,
260            pause: Pause::RefCount,
261            forwarded_roots: vec![],
262            next_slots: VectorQueue::new(),
263            next_slot_count: 0,
264            remset_recorded_slots: false,
265            should_record_forwarded_roots: false,
266        }
267    }
268
269    #[cold]
270    fn flush(&mut self) {
271        if !self.next_slots.is_empty() {
272            let slots = self.next_slots.take();
273            let w = Self::new(slots, false, self.mmtk(), self.bucket);
274            self.worker()
275                .add_boxed_work(WorkBucketStage::Unconstrained, Box::new(w));
276        }
277        assert!(self.nodes.is_empty());
278        self.next_slot_count = 0;
279    }
280
281    fn process_slots(&mut self) {
282        self.should_record_forwarded_roots = self.roots
283            && !self
284                .root_kind
285                .map(|r| r.should_skip_decs())
286                .unwrap_or_default();
287        self.pause = self.lxr.current_pause().unwrap();
288        if self.should_record_forwarded_roots {
289            self.forwarded_roots.reserve(self.slots.len());
290        }
291        let slots = std::mem::take(&mut self.slots);
292        if self.roots && self.root_kind == Some(RootKind::Weak) {
293            self.process_slots_impl::<true, false>(&slots);
294        } else if self.remset_recorded_slots {
295            self.process_slots_impl::<false, true>(&slots);
296        } else {
297            self.process_slots_impl::<false, false>(&slots);
298        }
299        self.roots = false;
300        self.remset_recorded_slots = false;
301        let should_record_forwarded_roots = self.should_record_forwarded_roots;
302        self.should_record_forwarded_roots = false;
303        let mut slots = vec![];
304        while !self.next_slots.is_empty() {
305            self.next_slot_count = 0;
306            slots.clear();
307            self.next_slots.swap(&mut slots);
308            self.process_slots_impl::<false, false>(&slots);
309        }
310        self.flush();
311        if should_record_forwarded_roots {
312            let roots = std::mem::take(&mut self.forwarded_roots);
313            self.lxr.curr_roots.read().unwrap().push(roots);
314        }
315    }
316}
317
318impl<VM: VMBinding, const FULL_GC: bool> GCWork<VM> for LXRStopTheWorldProcessEdges<VM, FULL_GC> {
319    fn do_work(&mut self, worker: &mut GCWorker<VM>, _mmtk: &'static MMTK<VM>) {
320        self.set_worker(worker);
321        self.process_slots();
322        if !self.nodes.is_empty() {
323            self.flush();
324        }
325    }
326}
327
328impl<VM: VMBinding, const FULL_GC: bool> LXRStopTheWorldProcessEdges<VM, FULL_GC> {
329    #[inline]
330    fn full_gc_trace_object<const WEAK_ROOT: bool>(
331        &mut self,
332        object: ObjectReference,
333    ) -> ObjectReference {
334        debug_assert!(FULL_GC);
335        debug_assert!(object.is_in_any_space());
336        debug_assert!(object.to_raw_address().is_aligned_to(8));
337        // debug_assert!(object.class_is_valid::<VM>());
338        if WEAK_ROOT && !Block::containing(object).is_defrag_source() {
339            return object;
340        }
341        let x = if self.lxr.immix_space.in_space(object) {
342            let pause = self.pause;
343            let worker = self.worker();
344            self.lxr.immix_space.rc_trace_object(
345                self,
346                object,
347                CopySemantics::DefaultCopy,
348                pause,
349                true,
350                worker,
351            )
352        } else {
353            self.lxr.los().trace_object(self, object)
354        };
355        if self.should_record_forwarded_roots {
356            self.forwarded_roots.push(x)
357        }
358        x
359    }
360
361    #[inline]
362    fn mature_evac_trace_object<const WEAK_ROOT: bool, const REMSET: bool>(
363        &mut self,
364        object: ObjectReference,
365    ) -> ObjectReference {
366        debug_assert!(!FULL_GC);
367        // The memory (lines) of these slots can be reused at any time during mature evacuation.
368        // Filter out invalid target objects.
369        if REMSET && (!object.is_in_any_space() || !object.to_raw_address().is_aligned_to(8)) {
370            return object;
371        }
372        if self.lxr.rc.count(object) == 0 {
373            return object;
374        }
375        if WEAK_ROOT && !Block::containing(object).is_defrag_source() {
376            return object;
377        }
378        debug_assert!(object.is_in_any_space(), "Invalid {:?}", object);
379        debug_assert!(
380            object.to_raw_address().is_aligned_to(8),
381            "Invalid {:?} remset={}",
382            object,
383            self.remset_recorded_slots
384        );
385        let object = object.get_forwarded_object().unwrap_or(object);
386        let new_object = if self.lxr.immix_space.in_space(object) {
387            if self.lxr.rc.object_is_in_straddle_line(object) {
388                return object;
389            }
390            let pause = self.pause;
391            let worker = self.worker();
392            self.lxr.immix_space.rc_trace_object(
393                self,
394                object,
395                CopySemantics::DefaultCopy,
396                pause,
397                true,
398                worker,
399            )
400        } else {
401            self.lxr.los().trace_object(self, object)
402        };
403        if self.should_record_forwarded_roots {
404            self.forwarded_roots.push(new_object)
405        }
406        new_object
407    }
408
409    #[inline]
410    fn __process_slot<const WEAK_ROOT: bool, const REMSET: bool>(&mut self, slot: VM::VMSlot) {
411        let Some(object) = slot.load() else {
412            return;
413        };
414        let new_object = if !FULL_GC {
415            self.mature_evac_trace_object::<WEAK_ROOT, REMSET>(object)
416        } else {
417            self.full_gc_trace_object::<WEAK_ROOT>(object)
418        };
419        if Self::OVERWRITE_REFERENCE && new_object != object {
420            slot.store(new_object);
421        }
422    }
423
424    fn process_slots_impl<const WEAK_ROOT: bool, const REMSET: bool>(
425        &mut self,
426        slots: &[VM::VMSlot],
427    ) {
428        for s in slots {
429            self.__process_slot::<WEAK_ROOT, REMSET>(*s);
430        }
431    }
432}
433
434impl<VM: VMBinding, const FULL_GC: bool> ObjectQueue for LXRStopTheWorldProcessEdges<VM, FULL_GC> {
435    fn enqueue(&mut self, object: ObjectReference) {
436        let limit: usize = if FULL_GC { 8192 } else { 1024 };
437        // TODO: Use actual TLS.
438        object.iterate_fields::<VM, _>(VMThread::UNINITIALIZED, |s| {
439            let Some(o) = s.load() else {
440                return;
441            };
442            if self.lxr.is_marked(o) && !self.lxr.in_defrag(o) {
443                return;
444            }
445            self.next_slots.push(s);
446            self.next_slot_count += 1;
447            if self.next_slot_count as usize >= limit {
448                self.flush();
449            }
450        });
451    }
452}
453
454impl<VM: VMBinding, const FULL_GC: bool> Deref for LXRStopTheWorldProcessEdges<VM, FULL_GC> {
455    type Target = ProcessEdgesBase<VM>;
456    fn deref(&self) -> &Self::Target {
457        &self.base
458    }
459}
460
461impl<VM: VMBinding, const FULL_GC: bool> DerefMut for LXRStopTheWorldProcessEdges<VM, FULL_GC> {
462    fn deref_mut(&mut self) -> &mut Self::Target {
463        &mut self.base
464    }
465}
mmtk/plan/lxr/gc_work/tracing.rs

mmtk/plan/lxr/gc_work/
tracing.rs
