mmtk/plan/lxr/
mod.rs

1mod barrier;
2mod block_allocation;
3mod gc_work;
4pub(super) mod global;
5mod mature_evac;
6pub(super) mod mutator;
7
8use std::sync::atomic::{AtomicBool, AtomicUsize};
9use std::sync::Arc;
10
11pub use self::global::LXR;
12
13use atomic::Atomic;
14use atomic::Ordering;
15use spin::Lazy;
16type RwLock<T> = spin::rwlock::RwLock<T>;
17
18// --- LXR-specific global state ---
19
20static NUM_CONCURRENT_TRACING_PACKETS: AtomicUsize = AtomicUsize::new(0);
21static DISABLE_LASY_DEC_FOR_CURRENT_GC: AtomicBool = AtomicBool::new(false);
22static NO_EVAC: AtomicBool = AtomicBool::new(false);
23
24// --- LXR-specific global constants/flags ---
25
26/// Enable Lazy Decrements
27const LAZY_DECREMENTS: bool = !cfg!(feature = "lxr_no_lazy");
28
29/// Enable Nursery Evacuation
30const NURSERY_EVACUATION: bool = !cfg!(feature = "lxr_no_nursery_evac");
31
32/// Enable Mature Evacuation
33pub(crate) const MATURE_EVACUATION: bool = !cfg!(feature = "lxr_no_mature_evac");
34
35/// Stop triggering CM or RC pauses, and trigger Full GCs instead if the available heap after a RC pause is still small.
36const RC_STOP_PERCENT: usize = 15;
37
38/// Trigger an RC pause when the predicted max survival size is larger than this threshold.
39const MAX_SURVIVAL_MB: usize = 128;
40
41/// Trigger a concurrent marking cycle when the predicted mature size is larger than this threshold.
42const TRACE_THRESHOLD: usize = 20;
43
44/// Start a concurrent marking cycle when the available pages in the previous pause is smaller than this threshold.
45const CYCLE_TRIGGER_THRESHOLD: usize = 1024;
46
47fn concurrent_marking_packets_drained() -> bool {
48    NUM_CONCURRENT_TRACING_PACKETS.load(Ordering::SeqCst) == 0
49}
50
51fn disable_lasy_dec_for_current_gc() -> bool {
52    DISABLE_LASY_DEC_FOR_CURRENT_GC.load(Ordering::SeqCst)
53}
54
55// --- Lazy sweeping job counters ---
56
57struct LazySweepingJobsCounter {
58    decs_counter: Option<Arc<AtomicUsize>>,
59    counter: Arc<AtomicUsize>,
60}
61impl LazySweepingJobsCounter {
62    pub fn new_decs() -> Self {
63        let lazy_sweeping_jobs = LAZY_SWEEPING_JOBS.read();
64        let decs_counter = lazy_sweeping_jobs.curr_decs_counter.as_ref().unwrap();
65        decs_counter.fetch_add(1, Ordering::SeqCst);
66        let counter = lazy_sweeping_jobs.curr_counter.as_ref().unwrap();
67        counter.fetch_add(1, Ordering::SeqCst);
68        Self {
69            decs_counter: Some(decs_counter.clone()),
70            counter: counter.clone(),
71        }
72    }
73
74    #[allow(clippy::should_implement_trait)]
75    pub fn clone(&self) -> Self {
76        self.counter.fetch_add(1, Ordering::SeqCst);
77        Self {
78            decs_counter: None,
79            counter: self.counter.clone(),
80        }
81    }
82
83    pub fn clone_with_decs(&self) -> Self {
84        self.decs_counter
85            .as_ref()
86            .unwrap()
87            .fetch_add(1, Ordering::SeqCst);
88        self.counter.fetch_add(1, Ordering::SeqCst);
89        Self {
90            decs_counter: self.decs_counter.clone(),
91            counter: self.counter.clone(),
92        }
93    }
94}
95
96impl Drop for LazySweepingJobsCounter {
97    fn drop(&mut self) {
98        let lazy_sweeping_jobs = LAZY_SWEEPING_JOBS.read();
99        if let Some(decs) = self.decs_counter.as_ref() {
100            if decs.fetch_sub(1, Ordering::SeqCst) == 1 {
101                let f = lazy_sweeping_jobs.end_of_decs.as_ref().unwrap();
102                f(self.clone())
103            }
104        }
105        if self.counter.fetch_sub(1, Ordering::SeqCst) == 1 {
106            if let Some(f) = lazy_sweeping_jobs.end_of_lazy.as_ref() {
107                f()
108            }
109        }
110    }
111}
112
113struct LazySweepingJobs {
114    prev_decs_counter: Option<Arc<AtomicUsize>>,
115    curr_decs_counter: Option<Arc<AtomicUsize>>,
116    prev_counter: Option<Arc<AtomicUsize>>,
117    curr_counter: Option<Arc<AtomicUsize>>,
118    pub end_of_decs: Option<Box<dyn Send + Sync + Fn(LazySweepingJobsCounter)>>,
119    pub end_of_lazy: Option<Box<dyn Send + Sync + Fn()>>,
120}
121
122impl LazySweepingJobs {
123    fn new() -> Self {
124        Self {
125            prev_decs_counter: None,
126            curr_decs_counter: None,
127            prev_counter: None,
128            curr_counter: None,
129            end_of_decs: None,
130            end_of_lazy: None,
131        }
132    }
133
134    pub fn all_finished() -> bool {
135        LAZY_SWEEPING_JOBS
136            .read()
137            .prev_counter
138            .as_ref()
139            .map(|c| c.load(Ordering::SeqCst))
140            .unwrap_or(0)
141            == 0
142    }
143
144    pub fn swap(&mut self) {
145        self.prev_decs_counter = self.curr_decs_counter.take();
146        self.curr_decs_counter = Some(Arc::new(AtomicUsize::new(0)));
147        self.prev_counter = self.curr_counter.take();
148        self.curr_counter = Some(Arc::new(AtomicUsize::new(0)));
149    }
150}
151
152static LAZY_SWEEPING_JOBS: Lazy<RwLock<LazySweepingJobs>> =
153    Lazy::new(|| RwLock::new(LazySweepingJobs::new()));
154
155static SURVIVAL_RATIO_PREDICTOR: SurvivalRatioPredictor = SurvivalRatioPredictor {
156    prev_ratio: Atomic::new(0.01),
157    alloc_vol: AtomicUsize::new(0),
158    copy_promote_vol: AtomicUsize::new(0),
159};
160
161struct SurvivalRatioPredictor {
162    prev_ratio: Atomic<f64>,
163    alloc_vol: AtomicUsize,
164    copy_promote_vol: AtomicUsize,
165}
166
167impl SurvivalRatioPredictor {
168    pub fn set_alloc_size(&self, size: usize) {
169        assert_eq!(self.alloc_vol.load(Ordering::SeqCst), 0);
170        self.alloc_vol.store(size, Ordering::SeqCst);
171    }
172
173    pub fn ratio(&self) -> f64 {
174        self.prev_ratio.load(Ordering::Relaxed)
175    }
176
177    pub fn update_ratio(&self) -> f64 {
178        if self.alloc_vol.load(Ordering::SeqCst) == 0 {
179            self.copy_promote_vol.store(0, Ordering::SeqCst);
180            return self.ratio();
181        }
182        let prev = self.prev_ratio.load(Ordering::SeqCst);
183        let curr = self.copy_promote_vol.load(Ordering::SeqCst) as f64
184            / self.alloc_vol.load(Ordering::SeqCst) as f64;
185        let curr = f64::min(curr, 1.0);
186        let ratio = (curr * 3f64 + prev) / 4f64;
187        let ratio = f64::min(ratio, 1.0);
188        self.prev_ratio.store(ratio, Ordering::SeqCst);
189        self.alloc_vol.store(0, Ordering::SeqCst);
190        self.copy_promote_vol.store(0, Ordering::SeqCst);
191        ratio
192    }
193}
194
195struct SurvivalRatioPredictorLocal {
196    copy_promote_vol: AtomicUsize,
197}
198
199impl Default for SurvivalRatioPredictorLocal {
200    fn default() -> Self {
201        Self {
202            copy_promote_vol: AtomicUsize::new(0),
203        }
204    }
205}
206
207impl SurvivalRatioPredictorLocal {
208    pub fn record_copied_promotion(&self, size: usize) {
209        self.copy_promote_vol.store(
210            self.copy_promote_vol.load(Ordering::Relaxed) + size,
211            Ordering::Relaxed,
212        );
213    }
214
215    pub fn sync(&self) {
216        SURVIVAL_RATIO_PREDICTOR.copy_promote_vol.fetch_add(
217            self.copy_promote_vol.load(Ordering::Relaxed),
218            Ordering::Relaxed,
219        );
220    }
221}
222
223static MATURE_LIVE_PREDICTOR: MatureLivePredictor = MatureLivePredictor {
224    live_pages: Atomic::new(0f64),
225};
226
227struct MatureLivePredictor {
228    live_pages: Atomic<f64>,
229}
230
231impl MatureLivePredictor {
232    pub fn live_pages(&self) -> f64 {
233        self.live_pages.load(Ordering::Relaxed)
234    }
235
236    pub fn update(&self, live_pages: usize) -> f64 {
237        // println!("live_pages {}", live_pages);
238        let prev = self.live_pages.load(Ordering::Relaxed);
239        let curr = live_pages as f64;
240        let weight = 3f64;
241        let next = (weight * curr + prev) / (weight + 1f64);
242        // println!("predict {}", next);
243        // crate::add_mature_reclaim(live_pages, prev);
244        self.live_pages.store(next, Ordering::Relaxed);
245        next
246    }
247}
mmtk/plan/lxr/mod.rs

mmtk/plan/lxr/
mod.rs
