mmtk/plan/lxr/
barrier.rs

1//! Read/Write barrier implementations.
2
3use std::sync::Arc;
4
5use atomic::Ordering;
6
7use super::LazySweepingJobsCounter;
8use super::LXR;
9use crate::plan::barriers::BarrierSemantics;
10use crate::plan::concurrent::global::ConcurrentPlan;
11use crate::plan::concurrent::Pause;
12use crate::plan::lxr::gc_work::rc::ProcessDecs;
13use crate::plan::lxr::gc_work::rc::ProcessIncs;
14use crate::plan::lxr::gc_work::rc::EDGE_KIND_MATURE;
15use crate::plan::lxr::gc_work::tracing::ProcessModBufSATB;
16use crate::plan::VectorQueue;
17use crate::scheduler::WorkBucketStage;
18use crate::util::metadata::log_bit::{LOGGED_VALUE, UNLOGGED_VALUE};
19use crate::util::metadata::side_metadata::address_to_meta_address;
20use crate::util::metadata::side_metadata::SideMetadataSpec;
21use crate::util::*;
22use crate::vm::slot::MemorySlice;
23use crate::vm::slot::Slot;
24use crate::vm::*;
25use crate::MMTK;
26
27pub struct LXRFieldBarrierSemantics<VM: VMBinding> {
28    mmtk: &'static MMTK<VM>,
29    tls: VMMutatorThread,
30    incs: VectorQueue<VM::VMSlot>,
31    decs: VectorQueue<ObjectReference>,
32    refs: VectorQueue<ObjectReference>,
33    lxr: &'static LXR<VM>,
34}
35
36impl<VM: VMBinding> LXRFieldBarrierSemantics<VM> {
37    const UNLOG_BITS: SideMetadataSpec = *VM::VMObjectModel::GLOBAL_FIELD_UNLOG_BIT_SPEC
38        .as_spec()
39        .extract_side_spec();
40
41    #[allow(unused)]
42    pub fn new(mmtk: &'static MMTK<VM>, tls: VMMutatorThread) -> Self {
43        Self {
44            mmtk,
45            tls,
46            incs: VectorQueue::default(),
47            decs: VectorQueue::default(),
48            refs: VectorQueue::default(),
49            lxr: mmtk.get_plan().downcast_ref::<LXR<VM>>().unwrap(),
50        }
51    }
52
53    fn get_slot_logging_state(&self, slot: VM::VMSlot) -> u8 {
54        unsafe { Self::UNLOG_BITS.load(slot.to_address()) }
55    }
56
57    fn attempt_to_log_field(&self, slot: VM::VMSlot) -> bool {
58        loop {
59            // Bailout if logged
60            if self.get_slot_logging_state(slot) == LOGGED_VALUE {
61                return false;
62            }
63            // Attempt to log the slots
64            match Self::UNLOG_BITS.compare_exchange_atomic(
65                slot.to_address(),
66                UNLOGGED_VALUE,
67                LOGGED_VALUE,
68                Ordering::SeqCst,
69                Ordering::SeqCst,
70            ) {
71                Ok(_) => return true,
72                Err(current) => {
73                    if current == LOGGED_VALUE {
74                        return false;
75                    }
76                }
77            }
78            // Failed to log the slot. Spin.
79            std::hint::spin_loop();
80        }
81    }
82
83    fn log_slot_and_get_old_target(&self, slot: VM::VMSlot) -> Result<Option<ObjectReference>, ()> {
84        if self.get_slot_logging_state(slot) == LOGGED_VALUE {
85            return Err(());
86        }
87        let old = slot.load();
88        if self.attempt_to_log_field(slot) {
89            Ok(old)
90        } else {
91            Err(())
92        }
93    }
94
95    fn slow(
96        &mut self,
97        _src: Option<ObjectReference>,
98        slot: VM::VMSlot,
99        old: Option<ObjectReference>,
100    ) {
101        // Reference counting
102        if let Some(old) = old {
103            self.decs.push(old);
104            if self.decs.is_full() {
105                self.flush_decs_and_satb();
106            }
107        }
108        self.incs.push(slot);
109        if self.incs.is_full() {
110            self.flush_incs();
111        }
112    }
113
114    fn enqueue_node(
115        &mut self,
116        src: Option<ObjectReference>,
117        slot: VM::VMSlot,
118        _new: Option<ObjectReference>,
119    ) -> bool {
120        if let Ok(old) = self.log_slot_and_get_old_target(slot) {
121            self.slow(src, slot, old);
122            true
123        } else {
124            false
125        }
126    }
127
128    fn should_create_satb_packets(&self) -> bool {
129        self.lxr.cm_enabled()
130            && (self.lxr.concurrent_work_in_progress()
131                || self.lxr.current_pause() == Some(Pause::FinalMark))
132    }
133
134    #[cold]
135    fn flush_incs(&mut self) {
136        if !self.incs.is_empty() {
137            let incs = self.incs.take();
138            self.lxr.rc.increase_inc_buffer_size(incs.len());
139            self.mmtk.scheduler.work_buckets[WorkBucketStage::RCProcessIncs].add(ProcessIncs::<
140                _,
141                EDGE_KIND_MATURE,
142            >::new(
143                incs, self.lxr
144            ));
145        }
146    }
147
148    #[cold]
149    fn flush_decs_and_satb(&mut self) {
150        if !self.decs.is_empty() {
151            let w = if self.should_create_satb_packets() {
152                let decs = Arc::new(self.decs.take());
153                self.mmtk.scheduler.work_buckets[WorkBucketStage::FinishConcurrentWork]
154                    .add(ProcessModBufSATB::new_arc(decs.clone()));
155                ProcessDecs::new_arc(decs, LazySweepingJobsCounter::new_decs())
156            } else {
157                let decs = self.decs.take();
158                ProcessDecs::new(decs, LazySweepingJobsCounter::new_decs())
159            };
160            if super::LAZY_DECREMENTS {
161                self.mmtk.scheduler.work_buckets[WorkBucketStage::Concurrent]
162                    .add_deferred(Box::new(w));
163            } else {
164                self.mmtk.scheduler.work_buckets[WorkBucketStage::STWRCDecsAndSweep].add(w);
165            }
166        }
167    }
168
169    #[cold]
170    fn flush_weak_refs(&mut self) {
171        if !self.refs.is_empty() {
172            debug_assert!(self.should_create_satb_packets());
173            let nodes = self.refs.take();
174            self.mmtk.scheduler.work_buckets[WorkBucketStage::FinishConcurrentWork]
175                .add(ProcessModBufSATB::new(nodes));
176        }
177    }
178}
179
180impl<VM: VMBinding> BarrierSemantics for LXRFieldBarrierSemantics<VM> {
181    type VM = VM;
182
183    #[cold]
184    fn flush(&mut self) {
185        self.flush_weak_refs();
186        self.flush_incs();
187        self.flush_decs_and_satb();
188    }
189
190    fn object_reference_write_slow(
191        &mut self,
192        src: ObjectReference,
193        slot: VM::VMSlot,
194        target: Option<ObjectReference>,
195    ) {
196        self.enqueue_node(Some(src), slot, target);
197    }
198
199    fn memory_region_copy_slow(&mut self, _src: VM::VMMemorySlice, dst: VM::VMMemorySlice) {
200        // Quickly check if all fields are logged. If yes, skip the barrier.
201        let unlog_bits_start = address_to_meta_address(&Self::UNLOG_BITS, dst.start());
202        let unlog_bits_start_aligned = unlog_bits_start.align_down(16);
203        let unlog_bits_end =
204            address_to_meta_address(&Self::UNLOG_BITS, dst.start() + dst.bytes() - 1);
205        let unlog_bits_end_aligned = unlog_bits_end.align_down(16);
206        let mut cursor = unlog_bits_start_aligned;
207        let mut all_logged = true;
208        while cursor <= unlog_bits_end_aligned {
209            if unsafe { cursor.load::<u128>() } != 0 {
210                all_logged = false;
211                break;
212            }
213            cursor += 16usize;
214        }
215        if all_logged {
216            return;
217        }
218
219        for s in dst.iter_slots() {
220            let _succ = self.enqueue_node(None, s, None);
221        }
222    }
223
224    fn load_weak_reference(&mut self, o: ObjectReference) {
225        if !self.lxr.concurrent_work_in_progress() || self.lxr.is_marked(o) {
226            return;
227        }
228        self.refs.push(o);
229        if self.refs.is_full() {
230            self.flush_weak_refs();
231        }
232    }
233
234    fn object_probable_write_slow(&mut self, obj: ObjectReference) {
235        obj.iterate_fields::<VM, _>(self.tls.0, |s| {
236            let _succ = self.enqueue_node(Some(obj), s, None);
237        });
238    }
239}
mmtk/plan/lxr/barrier.rs

mmtk/plan/lxr/
barrier.rs
