mmtk/plan/concurrent/
barrier.rs

1use std::sync::atomic::Ordering;
2
3use super::{concurrent_marking_work::ProcessModBufSATB, Pause};
4use crate::plan::global::PlanTraceObject;
5use crate::policy::gc_work::TraceKind;
6use crate::util::VMMutatorThread;
7use crate::{
8    plan::{barriers::BarrierSemantics, concurrent::global::ConcurrentPlan, VectorQueue},
9    scheduler::{GCWork, WorkBucketStage},
10    util::ObjectReference,
11    vm::{
12        slot::{MemorySlice, Slot},
13        VMBinding,
14    },
15    MMTK,
16};
17
18pub struct SATBBarrierSemantics<
19    VM: VMBinding,
20    P: ConcurrentPlan<VM = VM> + PlanTraceObject<VM>,
21    const KIND: TraceKind,
22> {
23    mmtk: &'static MMTK<VM>,
24    tls: VMMutatorThread,
25    satb: VectorQueue<ObjectReference>,
26    refs: VectorQueue<ObjectReference>,
27    plan: &'static P,
28}
29
30impl<VM: VMBinding, P: ConcurrentPlan<VM = VM> + PlanTraceObject<VM>, const KIND: TraceKind>
31    SATBBarrierSemantics<VM, P, KIND>
32{
33    pub fn new(mmtk: &'static MMTK<VM>, tls: VMMutatorThread) -> Self {
34        Self {
35            mmtk,
36            tls,
37            satb: VectorQueue::default(),
38            refs: VectorQueue::default(),
39            plan: mmtk.get_plan().downcast_ref::<P>().unwrap(),
40        }
41    }
42
43    fn slow(&mut self, _src: Option<ObjectReference>, _slot: VM::VMSlot, old: ObjectReference) {
44        self.satb.push(old);
45        if self.satb.is_full() {
46            self.flush_satb();
47        }
48    }
49
50    fn enqueue_node(
51        &mut self,
52        src: Option<ObjectReference>,
53        slot: VM::VMSlot,
54        _new: Option<ObjectReference>,
55    ) -> bool {
56        if let Some(old) = slot.load() {
57            self.slow(src, slot, old);
58        }
59        true
60    }
61
62    /// Attempt to atomically log an object.
63    /// Returns true if the object is not logged previously.
64    fn log_object(&self, object: ObjectReference) -> bool {
65        Self::UNLOG_BIT_SPEC.store_atomic::<VM, u8>(object, 0, None, Ordering::SeqCst);
66        true
67    }
68
69    fn flush_satb(&mut self) {
70        if !self.satb.is_empty() {
71            if self.should_create_satb_packets() {
72                let satb = self.satb.take();
73                self.add_work(ProcessModBufSATB::<VM, P, KIND>::new(satb));
74            } else {
75                let _ = self.satb.take();
76            };
77        }
78    }
79
80    #[cold]
81    fn flush_weak_refs(&mut self) {
82        if !self.refs.is_empty() {
83            let nodes = self.refs.take();
84            self.add_work(ProcessModBufSATB::<VM, P, KIND>::new(nodes));
85        }
86    }
87
88    fn add_work(&self, work: impl GCWork<VM>) {
89        let bucket_stage = if self.plan.concurrent_work_in_progress() {
90            WorkBucketStage::Concurrent
91        } else {
92            debug_assert_ne!(self.plan.current_pause(), Some(Pause::InitialMark));
93            WorkBucketStage::Closure
94        };
95        let bucket = &self.mmtk.scheduler.work_buckets[bucket_stage];
96        // If the bucket is disabled, we still add the work, but we dont need to notify a worker.
97        if bucket.is_enabled() {
98            // If there is a race, and the bucket is disabled now, it is fine.
99            // We just additionally notify the worker, which is harmless.
100            bucket.add(work);
101        } else {
102            bucket.add_no_notify(work);
103        }
104    }
105
106    fn should_create_satb_packets(&self) -> bool {
107        self.plan.concurrent_work_in_progress()
108            || self.plan.current_pause() == Some(Pause::FinalMark)
109    }
110}
111
112impl<VM: VMBinding, P: ConcurrentPlan<VM = VM> + PlanTraceObject<VM>, const KIND: TraceKind>
113    BarrierSemantics for SATBBarrierSemantics<VM, P, KIND>
114{
115    type VM = VM;
116
117    #[cold]
118    fn flush(&mut self) {
119        self.flush_satb();
120        self.flush_weak_refs();
121    }
122
123    fn object_reference_write_slow(
124        &mut self,
125        src: ObjectReference,
126        _slot: <Self::VM as VMBinding>::VMSlot,
127        _target: Option<ObjectReference>,
128    ) {
129        self.object_probable_write_slow(src);
130        self.log_object(src);
131    }
132
133    fn memory_region_copy_slow(
134        &mut self,
135        _src: <Self::VM as VMBinding>::VMMemorySlice,
136        dst: <Self::VM as VMBinding>::VMMemorySlice,
137    ) {
138        for s in dst.iter_slots() {
139            self.enqueue_node(None, s, None);
140        }
141    }
142
143    /// Enqueue the referent during concurrent marking.
144    ///
145    /// Note: During concurrent marking, a collector based on snapshot-at-the-beginning (SATB) will
146    /// not reach objects that were weakly reachable at the time of `InitialMark`.  But if a mutator
147    /// loads from a weak reference field during concurrent marking, it will make the referent
148    /// strongly reachable, yet the referent is still not part of the SATB.  We must conservatively
149    /// enqueue the referent even though its reachability has not yet been established, otherwise it
150    /// (and its children) may be treated as garbage if it happened to be weakly reachable at the
151    /// time of `InitialMark`.
152    fn load_weak_reference(&mut self, o: ObjectReference) {
153        if !self.plan.concurrent_work_in_progress() {
154            return;
155        }
156        self.refs.push(o);
157        if self.refs.is_full() {
158            self.flush_weak_refs();
159        }
160    }
161
162    fn object_probable_write_slow(&mut self, obj: ObjectReference) {
163        obj.iterate_fields::<VM, _>(self.tls.0, |s| {
164            self.enqueue_node(Some(obj), s, None);
165        });
166    }
167}
mmtk/plan/concurrent/barrier.rs

mmtk/plan/concurrent/
barrier.rs
