mmtk/plan/lxr/
mod.rs

1mod barrier;
2mod block_allocation;
3mod gc_work;
4pub(super) mod global;
5mod mature_evac;
6pub(super) mod mutator;
7
8use std::sync::atomic::{AtomicBool, AtomicUsize};
9use std::sync::Arc;
10
11pub use self::global::LXR;
12
13use atomic::Atomic;
14use atomic::Ordering;
15use spin::Lazy;
16type RwLock<T> = spin::rwlock::RwLock<T>;
17
18// --- LXR-specific global state ---
19
20static NUM_CONCURRENT_TRACING_PACKETS: AtomicUsize = AtomicUsize::new(0);
21static DISABLE_LASY_DEC_FOR_CURRENT_GC: AtomicBool = AtomicBool::new(false);
22static NO_EVAC: AtomicBool = AtomicBool::new(false);
23
24// --- LXR-specific global constants/flags ---
25
26/// Enable Lazy Decrements
27const LAZY_DECREMENTS: bool = !cfg!(feature = "lxr_no_lazy");
28
29/// Enable Nursery Evacuation
30const NURSERY_EVACUATION: bool = !cfg!(feature = "lxr_no_nursery_evac");
31
32/// Enable Mature Evacuation
33pub(crate) const MATURE_EVACUATION: bool = !cfg!(feature = "lxr_no_mature_evac");
34
35/// Stop triggering CM or RC pauses, and trigger Full GCs instead if the available heap after a RC pause is still small.
36const RC_STOP_PERCENT: usize = 15;
37
38/// Trigger a concurrent marking cycle when the predicted mature size is larger than this threshold.
39const TRACE_THRESHOLD: usize = 20;
40
41/// Start a concurrent marking cycle when the available pages in the previous pause is smaller than this threshold.
42const CYCLE_TRIGGER_THRESHOLD: usize = 1024;
43
44fn concurrent_marking_packets_drained() -> bool {
45    NUM_CONCURRENT_TRACING_PACKETS.load(Ordering::SeqCst) == 0
46}
47
48fn disable_lasy_dec_for_current_gc() -> bool {
49    DISABLE_LASY_DEC_FOR_CURRENT_GC.load(Ordering::SeqCst)
50}
51
52// --- Lazy sweeping job counters ---
53
54struct LazySweepingJobsCounter {
55    decs_counter: Option<Arc<AtomicUsize>>,
56    counter: Arc<AtomicUsize>,
57}
58impl LazySweepingJobsCounter {
59    pub fn new_decs() -> Self {
60        let lazy_sweeping_jobs = LAZY_SWEEPING_JOBS.read();
61        let decs_counter = lazy_sweeping_jobs.curr_decs_counter.as_ref().unwrap();
62        decs_counter.fetch_add(1, Ordering::SeqCst);
63        let counter = lazy_sweeping_jobs.curr_counter.as_ref().unwrap();
64        counter.fetch_add(1, Ordering::SeqCst);
65        Self {
66            decs_counter: Some(decs_counter.clone()),
67            counter: counter.clone(),
68        }
69    }
70
71    #[allow(clippy::should_implement_trait)]
72    pub fn clone(&self) -> Self {
73        self.counter.fetch_add(1, Ordering::SeqCst);
74        Self {
75            decs_counter: None,
76            counter: self.counter.clone(),
77        }
78    }
79
80    pub fn clone_with_decs(&self) -> Self {
81        self.decs_counter
82            .as_ref()
83            .unwrap()
84            .fetch_add(1, Ordering::SeqCst);
85        self.counter.fetch_add(1, Ordering::SeqCst);
86        Self {
87            decs_counter: self.decs_counter.clone(),
88            counter: self.counter.clone(),
89        }
90    }
91}
92
93impl Drop for LazySweepingJobsCounter {
94    fn drop(&mut self) {
95        let lazy_sweeping_jobs = LAZY_SWEEPING_JOBS.read();
96        if let Some(decs) = self.decs_counter.as_ref() {
97            if decs.fetch_sub(1, Ordering::SeqCst) == 1 {
98                let f = lazy_sweeping_jobs.end_of_decs.as_ref().unwrap();
99                f(self.clone())
100            }
101        }
102        if self.counter.fetch_sub(1, Ordering::SeqCst) == 1 {
103            if let Some(f) = lazy_sweeping_jobs.end_of_lazy.as_ref() {
104                f()
105            }
106        }
107    }
108}
109
110struct LazySweepingJobs {
111    prev_decs_counter: Option<Arc<AtomicUsize>>,
112    curr_decs_counter: Option<Arc<AtomicUsize>>,
113    prev_counter: Option<Arc<AtomicUsize>>,
114    curr_counter: Option<Arc<AtomicUsize>>,
115    pub end_of_decs: Option<Box<dyn Send + Sync + Fn(LazySweepingJobsCounter)>>,
116    pub end_of_lazy: Option<Box<dyn Send + Sync + Fn()>>,
117}
118
119impl LazySweepingJobs {
120    fn new() -> Self {
121        Self {
122            prev_decs_counter: None,
123            curr_decs_counter: None,
124            prev_counter: None,
125            curr_counter: None,
126            end_of_decs: None,
127            end_of_lazy: None,
128        }
129    }
130
131    pub fn all_finished() -> bool {
132        LAZY_SWEEPING_JOBS
133            .read()
134            .prev_counter
135            .as_ref()
136            .map(|c| c.load(Ordering::SeqCst))
137            .unwrap_or(0)
138            == 0
139    }
140
141    pub fn swap(&mut self) {
142        self.prev_decs_counter = self.curr_decs_counter.take();
143        self.curr_decs_counter = Some(Arc::new(AtomicUsize::new(0)));
144        self.prev_counter = self.curr_counter.take();
145        self.curr_counter = Some(Arc::new(AtomicUsize::new(0)));
146    }
147}
148
149static LAZY_SWEEPING_JOBS: Lazy<RwLock<LazySweepingJobs>> =
150    Lazy::new(|| RwLock::new(LazySweepingJobs::new()));
151
152static SURVIVAL_RATIO_PREDICTOR: SurvivalRatioPredictor = SurvivalRatioPredictor {
153    alloc_vol: AtomicUsize::new(0),
154    copy_promote_vol: AtomicUsize::new(0),
155    prev_copy_promote_ratio: Atomic::new(0.01),
156    promote_vol: AtomicUsize::new(0),
157    prev_promote_ratio: Atomic::new(0.01),
158};
159
160/// Predicts how much of the young allocation in the coming cycle will survive.
161struct SurvivalRatioPredictor {
162    /// Young allocation over the current cycle: the denominator of both ratios.
163    alloc_vol: AtomicUsize,
164    /// Volume promoted by copying during the current cycle.
165    copy_promote_vol: AtomicUsize,
166    /// Smoothed `copy_promote_vol / alloc_vol` over previous cycles.
167    prev_copy_promote_ratio: Atomic<f64>,
168    /// Volume promoted by any means during the current cycle.
169    promote_vol: AtomicUsize,
170    /// Smoothed `promote_vol / alloc_vol` over previous cycles.
171    prev_promote_ratio: Atomic<f64>,
172}
173
174impl SurvivalRatioPredictor {
175    pub fn set_alloc_size(&self, size: usize) {
176        assert_eq!(self.alloc_vol.load(Ordering::SeqCst), 0);
177        self.alloc_vol.store(size, Ordering::SeqCst);
178    }
179
180    /// Fraction of young allocation that survived *by being copied*.
181    pub fn copy_promote_ratio(&self) -> f64 {
182        self.prev_copy_promote_ratio.load(Ordering::Relaxed)
183    }
184
185    /// Fraction of young allocation that survived at all, copied or promoted in place.
186    pub fn promote_ratio(&self) -> f64 {
187        self.prev_promote_ratio.load(Ordering::Relaxed)
188    }
189
190    pub fn update_ratios(&self) {
191        let alloc_vol = self.alloc_vol.swap(0, Ordering::SeqCst);
192        let copy_promote_vol = self.copy_promote_vol.swap(0, Ordering::SeqCst);
193        let promote_vol = self.promote_vol.swap(0, Ordering::SeqCst);
194        if alloc_vol == 0 {
195            return;
196        }
197        let smooth = |prev_ratio: &Atomic<f64>, vol: usize| {
198            let curr = f64::min(vol as f64 / alloc_vol as f64, 1.0);
199            let prev = prev_ratio.load(Ordering::SeqCst);
200            prev_ratio.store(f64::min((curr * 3f64 + prev) / 4f64, 1.0), Ordering::SeqCst);
201        };
202        smooth(&self.prev_copy_promote_ratio, copy_promote_vol);
203        smooth(&self.prev_promote_ratio, promote_vol);
204    }
205}
206
207struct SurvivalRatioPredictorLocal {
208    copy_promote_vol: AtomicUsize,
209    promote_vol: AtomicUsize,
210}
211
212impl Default for SurvivalRatioPredictorLocal {
213    fn default() -> Self {
214        Self {
215            copy_promote_vol: AtomicUsize::new(0),
216            promote_vol: AtomicUsize::new(0),
217        }
218    }
219}
220
221impl SurvivalRatioPredictorLocal {
222    pub fn record_promotion(&self, size: usize, copied: bool) {
223        self.promote_vol.fetch_add(size, Ordering::Relaxed);
224        if copied {
225            self.copy_promote_vol.fetch_add(size, Ordering::Relaxed);
226        }
227    }
228
229    pub fn sync(&self) {
230        SURVIVAL_RATIO_PREDICTOR.copy_promote_vol.fetch_add(
231            self.copy_promote_vol.load(Ordering::Relaxed),
232            Ordering::Relaxed,
233        );
234        SURVIVAL_RATIO_PREDICTOR
235            .promote_vol
236            .fetch_add(self.promote_vol.load(Ordering::Relaxed), Ordering::Relaxed);
237    }
238}
239
240static MATURE_LIVE_PREDICTOR: MatureLivePredictor = MatureLivePredictor {
241    live_pages: Atomic::new(0f64),
242};
243
244struct MatureLivePredictor {
245    live_pages: Atomic<f64>,
246}
247
248impl MatureLivePredictor {
249    pub fn live_pages(&self) -> f64 {
250        self.live_pages.load(Ordering::Relaxed)
251    }
252
253    pub fn update(&self, live_pages: usize) -> f64 {
254        // println!("live_pages {}", live_pages);
255        let prev = self.live_pages.load(Ordering::Relaxed);
256        let curr = live_pages as f64;
257        let weight = 3f64;
258        let next = (weight * curr + prev) / (weight + 1f64);
259        // println!("predict {}", next);
260        // crate::add_mature_reclaim(live_pages, prev);
261        self.live_pages.store(next, Ordering::Relaxed);
262        next
263    }
264}
mmtk/plan/lxr/mod.rs

mmtk/plan/lxr/
mod.rs
