mmtk/util/heap/gc_trigger.rs
1use atomic::Ordering;
2
3use crate::global_state::{GcStatus, GlobalState};
4use crate::plan::Plan;
5use crate::policy::space::Space;
6use crate::scheduler::GCWorkScheduler;
7use crate::util::constants::BYTES_IN_PAGE;
8use crate::util::conversions;
9use crate::util::options::{GCTriggerSelector, Options, DEFAULT_MAX_NURSERY, DEFAULT_MIN_NURSERY};
10use crate::vm::VMBinding;
11use crate::MMTK;
12use std::mem::MaybeUninit;
13use std::sync::atomic::AtomicUsize;
14use std::sync::Arc;
15
16/// GCTrigger is responsible for triggering GCs based on the given policy.
17/// All the decisions about heap limit and GC triggering should be resolved here.
18/// Depending on the actual policy, we may either forward the calls either to the plan
19/// or to the binding/runtime.
20pub struct GCTrigger<VM: VMBinding> {
21 /// The current plan. This is uninitialized when we create it, and later initialized
22 /// once we have a fixed address for the plan.
23 plan: MaybeUninit<&'static dyn Plan<VM = VM>>,
24 /// The triggering policy.
25 pub policy: Box<dyn GCTriggerPolicy<VM>>,
26 scheduler: Arc<GCWorkScheduler<VM>>,
27 options: Arc<Options>,
28 state: Arc<GlobalState>,
29}
30
31impl<VM: VMBinding> GCTrigger<VM> {
32 pub fn new(
33 options: Arc<Options>,
34 scheduler: Arc<GCWorkScheduler<VM>>,
35 state: Arc<GlobalState>,
36 ) -> Self {
37 GCTrigger {
38 plan: MaybeUninit::uninit(),
39 policy: match *options.gc_trigger {
40 GCTriggerSelector::FixedHeapSize(size) => Box::new(FixedHeapSizeTrigger {
41 total_pages: conversions::bytes_to_pages_up(size),
42 }),
43 GCTriggerSelector::DynamicHeapSize(min, max) => 'dynamic_heap_size: {
44 let min_pages = conversions::bytes_to_pages_up(min);
45 let max_pages = conversions::bytes_to_pages_up(max);
46
47 if *options.plan == crate::util::options::PlanSelector::NoGC {
48 warn!("Cannot use dynamic heap size with NoGC. Using fixed heap size trigger instead.");
49 break 'dynamic_heap_size Box::new(FixedHeapSizeTrigger {
50 total_pages: max_pages,
51 });
52 }
53
54 Box::new(MemBalancerTrigger::new(min_pages, max_pages))
55 }
56 GCTriggerSelector::Delegated => {
57 <VM::VMCollection as crate::vm::Collection<VM>>::create_gc_trigger()
58 }
59 },
60 options,
61 scheduler,
62 state,
63 }
64 }
65
66 /// Set the plan. This is called in `create_plan()` after we created a boxed plan.
67 pub fn set_plan(&mut self, plan: &'static dyn Plan<VM = VM>) {
68 self.plan.write(plan);
69 }
70
71 fn plan(&self) -> &dyn Plan<VM = VM> {
72 unsafe { self.plan.assume_init() }
73 }
74
75 /// Request a GC. Called by mutators when polling (during allocation) and when handling user
76 /// GC requests (e.g. `System.gc();` in Java).
77 /// Atomically check that collection is enabled, and if so, request a GC. This makes the
78 /// enabled-check and the request atomic with respect to [`GCTrigger::disable_collection`]
79 /// and [`GCTrigger::enable_collection`], so a GC is never requested after collection has
80 /// been disabled.
81 /// Returns whether a GC was actually requested.
82 fn request(&self) -> bool {
83 // `GCWorkScheduler::request_schedule_collection` needs to hold a mutex to communicate
84 // with GC workers, which is expensive for functions like `poll`. `try_request_pause`
85 // only returns `Ok` to the thread that actually wins the race to transition the status,
86 // so only that thread calls it, instead of every thread that observes the old status.
87 match self.state.gc_status.try_request_pause() {
88 Ok(_) => {
89 probe!(mmtk, gc_requested);
90 self.scheduler.request_schedule_collection();
91 true
92 },
93 Err(GcStatus::Disabled(_)) => false,
94 // A GC is genuinely required (the heap policy has been exceeded), but MMTk has no GC
95 // worker threads to service it. Silently returning `false` here would let allocation
96 // grow the heap without bound instead of respecting the configured limit.
97 Err(GcStatus::Uninitialized) => panic!(
98 "GC is not allowed here: collection is not initialized (did you call initialize_collection()?)."
99 ),
100 Err(GcStatus::PauseRequested) => true,
101 _ => unreachable!(),
102 }
103 }
104
105 /// Disable collection. On success, returns `Ok(true)` if this call actually switched
106 /// collection from enabled to disabled, `Ok(false)` if it only increased the nesting depth of
107 /// an already-disabled status. If MMTk is unable to disable GC right now (possibly a GC is in
108 /// progress, or a GC has been requested), returns `Err` with the status that prevented it;
109 /// users should invoke runtime safepoints or other mechanisms to prepare for a GC pause, and
110 /// then call this function again.
111 ///
112 /// This call is nestable. Each call must be paired with a matching call to
113 /// [`GCTrigger::enable_collection`].
114 pub fn disable_collection(&self) -> Result<bool, GcStatus> {
115 self.state.gc_status.set_disabled()
116 }
117
118 /// Re-enable collection. If collection is not currently disabled (e.g. there was no prior
119 /// matching call to [`GCTrigger::disable_collection`]), this is a no-op.
120 /// Returns `true` if this call actually re-enabled collection (i.e. it was the outermost
121 /// matching call), `false` if it only decremented the nesting depth, or if collection was
122 /// already enabled.
123 pub fn enable_collection(&self) -> bool {
124 self.state.gc_status.set_enabled()
125 }
126
127 /// Return whether collection is currently enabled.
128 pub fn is_collection_enabled(&self) -> bool {
129 !self.state.gc_status.is_disabled()
130 }
131
132 /// This method is called periodically by the allocation subsystem
133 /// (by default, each time a page is consumed), and provides the
134 /// collector with an opportunity to collect.
135 ///
136 /// Arguments:
137 /// * `space_full`: Space request failed, must recover pages within 'space'.
138 /// * `space`: The space that triggered the poll. This could `None` if the poll is not triggered by a space.
139 pub fn poll(&self, space_full: bool, space: Option<&dyn Space<VM>>) -> bool {
140 if !self.is_collection_enabled() {
141 return false;
142 }
143
144 let plan = self.plan();
145 if self
146 .policy
147 .is_gc_required(space_full, space.map(|s| SpaceStats::new(s)), plan)
148 {
149 info!(
150 "[POLL] {}{} ({}/{} pages)",
151 if let Some(space) = space {
152 format!("{}: ", space.get_name())
153 } else {
154 "".to_string()
155 },
156 "Triggering collection",
157 plan.get_reserved_pages(),
158 plan.get_total_pages(),
159 );
160 return self.request();
161 }
162 false
163 }
164
165 /// This method is called when the user manually requests a collection, such as `System.gc()` in Java.
166 /// Returns true if a collection is actually requested.
167 ///
168 /// # Arguments
169 /// * `force`: If true, we force a collection regardless of the settings. If false, we only trigger a collection if the settings allow it.
170 /// * `exhaustive`: If true, we try to make the collection exhaustive (e.g. full heap collection). If false, the collection kind is determined internally.
171 pub fn handle_user_collection_request(&self, force: bool, exhaustive: bool) -> bool {
172 if !self.plan().constraints().collects_garbage {
173 warn!("User attempted a collection request, but the plan can not do GC. The request is ignored.");
174 return false;
175 }
176
177 if force || !*self.options.ignore_system_gc && self.is_collection_enabled() {
178 info!("User triggering collection");
179 // TODO: this may not work reliably. If a GC has been triggered, this will not force it to be a full heap GC.
180 if exhaustive {
181 if let Some(gen) = self.plan().generational() {
182 gen.force_full_heap_collection();
183 }
184 }
185
186 self.state
187 .user_triggered_collection
188 .store(true, Ordering::Relaxed);
189 return self.request();
190 }
191
192 false
193 }
194
195 /// MMTK has requested stop-the-world activity (e.g., stw within a concurrent gc).
196 // TODO: We should use this for concurrent GC. E.g. in concurrent Immix, when the initial mark is done, we
197 // can use this function to immediately trigger the final mark pause. The current implementation uses
198 // normal collection_required check, which may delay the final mark unnecessarily.
199 #[allow(unused)]
200 pub fn trigger_internal_collection_request(&self) {
201 self.state
202 .last_internal_triggered_collection
203 .store(true, Ordering::Relaxed);
204 self.state
205 .internal_triggered_collection
206 .store(true, Ordering::Relaxed);
207 // TODO: The current `request()` is probably incorrect for internally triggered GC.
208 // Consider removing functions related to "internal triggered collection".
209 self.request();
210 // TODO: Make sure this function works correctly for concurrent GC.
211 unimplemented!()
212 }
213
214 pub fn should_do_stress_gc(&self) -> bool {
215 Self::should_do_stress_gc_inner(&self.state, &self.options)
216 }
217
218 /// Check if we should do a stress GC now. If GC is initialized and the allocation bytes exceeds
219 /// the stress factor, we should do a stress GC.
220 pub(crate) fn should_do_stress_gc_inner(state: &GlobalState, options: &Options) -> bool {
221 state.is_initialized()
222 && (state.allocation_bytes.load(Ordering::SeqCst) > *options.stress_factor)
223 }
224
225 /// Check if the heap is full
226 pub fn is_heap_full(&self) -> bool {
227 self.policy.is_heap_full(self.plan())
228 }
229
230 /// Return upper bound of the nursery size (in number of bytes)
231 pub fn get_max_nursery_bytes(&self) -> usize {
232 use crate::util::options::NurserySize;
233 debug_assert!(self.plan().generational().is_some());
234 match *self.options.nursery {
235 NurserySize::Bounded { min: _, max } => max,
236 NurserySize::ProportionalBounded { min: _, max } => {
237 let heap_size_bytes =
238 conversions::pages_to_bytes(self.policy.get_current_heap_size_in_pages());
239 let max_bytes = heap_size_bytes as f64 * max;
240 let max_bytes = conversions::raw_align_up(max_bytes as usize, BYTES_IN_PAGE);
241 if max_bytes > DEFAULT_MAX_NURSERY {
242 warn!("Proportional nursery with max size {} ({}) is larger than DEFAULT_MAX_NURSERY ({}). Use DEFAULT_MAX_NURSERY instead.", max, max_bytes, DEFAULT_MAX_NURSERY);
243 DEFAULT_MAX_NURSERY
244 } else {
245 max_bytes
246 }
247 }
248 NurserySize::Fixed(sz) => sz,
249 }
250 }
251
252 /// Return lower bound of the nursery size (in number of bytes)
253 pub fn get_min_nursery_bytes(&self) -> usize {
254 use crate::util::options::NurserySize;
255 debug_assert!(self.plan().generational().is_some());
256 match *self.options.nursery {
257 NurserySize::Bounded { min, max: _ } => min,
258 NurserySize::ProportionalBounded { min, max: _ } => {
259 let min_bytes =
260 conversions::pages_to_bytes(self.policy.get_current_heap_size_in_pages())
261 as f64
262 * min;
263 let min_bytes = conversions::raw_align_up(min_bytes as usize, BYTES_IN_PAGE);
264 if min_bytes < DEFAULT_MIN_NURSERY {
265 warn!("Proportional nursery with min size {} ({}) is smaller than DEFAULT_MIN_NURSERY ({}). Use DEFAULT_MIN_NURSERY instead.", min, min_bytes, DEFAULT_MIN_NURSERY);
266 DEFAULT_MIN_NURSERY
267 } else {
268 min_bytes
269 }
270 }
271 NurserySize::Fixed(sz) => sz,
272 }
273 }
274
275 /// Return upper bound of the nursery size (in number of pages)
276 pub fn get_max_nursery_pages(&self) -> usize {
277 crate::util::conversions::bytes_to_pages_up(self.get_max_nursery_bytes())
278 }
279
280 /// Return lower bound of the nursery size (in number of pages)
281 pub fn get_min_nursery_pages(&self) -> usize {
282 crate::util::conversions::bytes_to_pages_up(self.get_min_nursery_bytes())
283 }
284
285 /// A check for the obvious out-of-memory case: if the requested size is larger than
286 /// the heap size, it is definitely an OOM. We would like to identify that, and
287 /// allows the binding to deal with OOM. Without this check, we will attempt
288 /// to allocate from the page resource. If the requested size is unrealistically large
289 /// (such as `usize::MAX`), it breaks the assumptions of our implementation of
290 /// page resource, vm map, etc. This check prevents that, and allows us to
291 /// handle the OOM case.
292 /// Each allocator that may request an arbitrary size should call this method before
293 /// acquring memory from the space. For example, bump pointer allocator and large object
294 /// allocator need to call this method. On the other hand, allocators that only allocate
295 /// memory in fixed size blocks do not need to call this method.
296 /// An allocator should call this method before doing any computation on the size to
297 /// avoid arithmatic overflow. If we have to do computation in the allocation fastpath and
298 /// overflow happens there, there is nothing we can do about it.
299 /// Return a boolean to indicate if we will be out of memory, determined by the check.
300 pub fn will_oom_on_alloc(&self, size: usize) -> bool {
301 let max_pages = self.policy.get_max_heap_size_in_pages();
302 let requested_pages = size >> crate::util::constants::LOG_BYTES_IN_PAGE;
303 requested_pages > max_pages
304 }
305}
306
307/// Provides statistics about the space. This is exposed to bindings, as it is used
308/// in both [`crate::plan::Plan`] and [`GCTriggerPolicy`].
309// This type exists so we do not need to expose the `Space` trait to the bindings.
310pub struct SpaceStats<'a, VM: VMBinding>(pub(crate) &'a dyn Space<VM>);
311
312impl<'a, VM: VMBinding> SpaceStats<'a, VM> {
313 /// Create new SpaceStats.
314 fn new(space: &'a dyn Space<VM>) -> Self {
315 Self(space)
316 }
317
318 /// Get the number of reserved pages for the space.
319 pub fn reserved_pages(&self) -> usize {
320 self.0.reserved_pages()
321 }
322
323 // We may expose more methods to bindings if they need more information for implementing GC triggers.
324 // But we should never expose `Space` itself.
325}
326
327/// This trait describes a GC trigger policy. A triggering policy have hooks to be informed about
328/// GC start/end so they can collect some statistics about GC and allocation. The policy needs to
329/// decide the (current) heap limit and decide whether a GC should be performed.
330pub trait GCTriggerPolicy<VM: VMBinding>: Sync + Send {
331 /// Inform the triggering policy that we have pending allocation.
332 /// Any GC trigger policy with dynamic heap size should take this into account when calculating a new heap size.
333 /// Failing to do so may result in unnecessay GCs, or result in an infinite loop if the new heap size
334 /// can never accomodate the pending allocation.
335 fn on_pending_allocation(&self, _pages: usize) {}
336 /// Inform the triggering policy that a GC cycle starts. A GC cycle consists of one or more
337 /// GC pauses (see [`Self::on_pause_start`]) plus any concurrent work in between the pauses.
338 /// For a stop-the-world GC, a GC cycle is just a single pause, and this is called at the same
339 /// time as [`Self::on_pause_start`]. For a concurrent GC that splits a cycle into multiple
340 /// pauses (e.g. an initial mark pause and a final mark pause with concurrent marking in
341 /// between), this is only called once per cycle, for the first pause in the cycle.
342 fn on_gc_start(&self, _mmtk: &'static MMTK<VM>) {}
343 /// Inform the triggering policy that a GC cycle ends. See [`Self::on_gc_start`] for what
344 /// a GC cycle is. This is only called once per GC cycle, for the last pause in the cycle.
345 fn on_gc_end(&self, _mmtk: &'static MMTK<VM>) {}
346 /// Inform the triggering policy that a pause starts. For a concurrent GC, this is called once
347 /// for every STW pause in a GC cycle, not just once per cycle. See
348 /// [`Self::on_gc_start`] for the hook that is only called once per GC cycle.
349 fn on_pause_start(&self, _mmtk: &'static MMTK<VM>) {}
350 /// Inform the triggering policy that a pause ends. For a concurrent GC, this is called once
351 /// for every STW pause in a GC cycle, not just once per cycle. See [`Self::on_gc_end`]
352 /// for the hook that is only called once per GC cycle.
353 fn on_pause_end(&self, _mmtk: &'static MMTK<VM>) {}
354 /// Inform the triggering policy that a GC is about to start the release work. This is called
355 /// in the global Release work packet. This means we assume a plan
356 /// do not schedule any work that reclaims memory before the global `Release` work. The current plans
357 /// satisfy this assumption: they schedule other release work in `plan.release()`.
358 fn on_gc_release(&self, _mmtk: &'static MMTK<VM>) {}
359 /// Is a GC required now? The GC trigger may implement its own heuristics to decide when
360 /// a GC should be performed. However, we recommend the implementation to do its own checks
361 /// first, and always call `plan.collection_required(space_full, space)` at the end as a fallback to see if the plan needs
362 /// to do a GC.
363 ///
364 /// Arguments:
365 /// * `space_full`: Is any space full?
366 /// * `space`: The space that is full. The GC trigger may access some stats of the space.
367 /// * `plan`: The reference to the plan in use.
368 fn is_gc_required(
369 &self,
370 space_full: bool,
371 space: Option<SpaceStats<VM>>,
372 plan: &dyn Plan<VM = VM>,
373 ) -> bool;
374 /// Is current heap full?
375 fn is_heap_full(&self, plan: &dyn Plan<VM = VM>) -> bool;
376 /// Return the current heap size (in pages)
377 fn get_current_heap_size_in_pages(&self) -> usize;
378 /// Return the upper bound of heap size
379 fn get_max_heap_size_in_pages(&self) -> usize;
380 /// Can the heap size grow?
381 fn can_heap_size_grow(&self) -> bool;
382}
383
384/// A simple GC trigger that uses a fixed heap size.
385pub struct FixedHeapSizeTrigger {
386 total_pages: usize,
387}
388impl<VM: VMBinding> GCTriggerPolicy<VM> for FixedHeapSizeTrigger {
389 fn is_gc_required(
390 &self,
391 space_full: bool,
392 space: Option<SpaceStats<VM>>,
393 plan: &dyn Plan<VM = VM>,
394 ) -> bool {
395 // Let the plan decide
396 plan.collection_required(space_full, space)
397 }
398
399 fn is_heap_full(&self, plan: &dyn Plan<VM = VM>) -> bool {
400 // If reserved pages is larger than the total pages, the heap is full.
401 plan.get_reserved_pages() > self.total_pages
402 }
403
404 fn get_current_heap_size_in_pages(&self) -> usize {
405 self.total_pages
406 }
407
408 fn get_max_heap_size_in_pages(&self) -> usize {
409 self.total_pages
410 }
411
412 fn can_heap_size_grow(&self) -> bool {
413 false
414 }
415}
416
417use atomic_refcell::AtomicRefCell;
418use std::time::Instant;
419
420/// An implementation of MemBalancer (Optimal heap limits for reducing browser memory use, <https://dl.acm.org/doi/10.1145/3563323>)
421/// We use MemBalancer to decide a heap limit between the min heap and the max heap.
422/// The current implementation is a simplified version of mem balancer and it does not take collection/allocation speed into account,
423/// and uses a fixed constant instead.
424// TODO: implement a complete mem balancer.
425pub struct MemBalancerTrigger {
426 /// The min heap size
427 min_heap_pages: usize,
428 /// The max heap size
429 max_heap_pages: usize,
430 /// The current heap size
431 current_heap_pages: AtomicUsize,
432 /// The number of pending allocation pages. The allocation requests for them have failed, and a GC is triggered.
433 /// We will need to take them into consideration so that the new heap size can accomodate those allocations.
434 pending_pages: AtomicUsize,
435 /// Statistics
436 stats: AtomicRefCell<MemBalancerStats>,
437}
438
439#[derive(Copy, Clone, Debug)]
440struct MemBalancerStats {
441 // Allocation/collection stats in the previous estimation. We keep this so we can use them to smooth the current value
442 /// Previous allocated memory in pages.
443 allocation_pages_prev: Option<f64>,
444 /// Previous allocation duration in secs
445 allocation_time_prev: Option<f64>,
446 /// Previous collected memory in pages
447 collection_pages_prev: Option<f64>,
448 /// Previous colleciton duration in secs
449 collection_time_prev: Option<f64>,
450
451 // Allocation/collection stats in this estimation.
452 /// Allocated memory in pages
453 allocation_pages: f64,
454 /// Allocation duration in secs
455 allocation_time: f64,
456 /// Collected memory in pages (memory traversed during collection)
457 collection_pages: f64,
458 /// Collection duration in secs
459 collection_time: f64,
460
461 /// The time when this GC starts
462 gc_start_time: Instant,
463 /// The time when this GC ends
464 gc_end_time: Instant,
465
466 /// The live pages before we release memory.
467 gc_release_live_pages: usize,
468 /// The live pages at the GC end
469 gc_end_live_pages: usize,
470}
471
472impl std::default::Default for MemBalancerStats {
473 fn default() -> Self {
474 let now = Instant::now();
475 Self {
476 allocation_pages_prev: None,
477 allocation_time_prev: None,
478 collection_pages_prev: None,
479 collection_time_prev: None,
480 allocation_pages: 0f64,
481 allocation_time: 0f64,
482 collection_pages: 0f64,
483 collection_time: 0f64,
484 gc_start_time: now,
485 gc_end_time: now,
486 gc_release_live_pages: 0,
487 gc_end_live_pages: 0,
488 }
489 }
490}
491
492use crate::plan::GenerationalPlan;
493
494impl MemBalancerStats {
495 // Collect mem stats for generational plans:
496 // * We ignore nursery GCs.
497 // * allocation = objects in mature space = promoted + pretentured = live pages in mature space before release - live pages at the end of last mature GC
498 // * collection = live pages in mature space at the end of GC - live pages in mature space before release
499
500 fn generational_mem_stats_on_gc_start<VM: VMBinding>(
501 &mut self,
502 _plan: &dyn GenerationalPlan<VM = VM>,
503 ) {
504 // We don't need to do anything
505 }
506 fn generational_mem_stats_on_gc_release<VM: VMBinding>(
507 &mut self,
508 plan: &dyn GenerationalPlan<VM = VM>,
509 ) {
510 if !plan.is_current_gc_nursery() {
511 self.gc_release_live_pages = plan.get_mature_reserved_pages();
512
513 // Calculate the promoted pages (including pre tentured objects)
514 let promoted = self
515 .gc_release_live_pages
516 .saturating_sub(self.gc_end_live_pages);
517 self.allocation_pages = promoted as f64;
518 trace!(
519 "promoted = mature live before release {} - mature live at prev gc end {} = {}",
520 self.gc_release_live_pages,
521 self.gc_end_live_pages,
522 promoted
523 );
524 trace!(
525 "allocated pages (accumulated to) = {}",
526 self.allocation_pages
527 );
528 }
529 }
530 /// Return true if we should compute a new heap limit. Only do so at the end of a mature GC
531 fn generational_mem_stats_on_gc_end<VM: VMBinding>(
532 &mut self,
533 plan: &dyn GenerationalPlan<VM = VM>,
534 ) -> bool {
535 if !plan.is_current_gc_nursery() {
536 self.gc_end_live_pages = plan.get_mature_reserved_pages();
537 // Use live pages as an estimate for pages traversed during GC
538 self.collection_pages = self.gc_end_live_pages as f64;
539 trace!(
540 "collected pages = mature live at gc end {} - mature live at gc release {} = {}",
541 self.gc_release_live_pages,
542 self.gc_end_live_pages,
543 self.collection_pages
544 );
545 true
546 } else {
547 false
548 }
549 }
550
551 // Collect mem stats for non generational plans
552 // * allocation = live pages at the start of GC - live pages at the end of last GC
553 // * collection = live pages at the end of GC - live pages before release
554
555 fn non_generational_mem_stats_on_gc_start<VM: VMBinding>(&mut self, mmtk: &'static MMTK<VM>) {
556 self.allocation_pages = mmtk
557 .get_plan()
558 .get_reserved_pages()
559 .saturating_sub(self.gc_end_live_pages) as f64;
560 trace!(
561 "allocated pages = used {} - live in last gc {} = {}",
562 mmtk.get_plan().get_reserved_pages(),
563 self.gc_end_live_pages,
564 self.allocation_pages
565 );
566 }
567 fn non_generational_mem_stats_on_gc_release<VM: VMBinding>(&mut self, mmtk: &'static MMTK<VM>) {
568 self.gc_release_live_pages = mmtk.get_plan().get_reserved_pages();
569 trace!("live before release = {}", self.gc_release_live_pages);
570 }
571 fn non_generational_mem_stats_on_gc_end<VM: VMBinding>(&mut self, mmtk: &'static MMTK<VM>) {
572 self.gc_end_live_pages = mmtk.get_plan().get_reserved_pages();
573 trace!("live pages = {}", self.gc_end_live_pages);
574 // Use live pages as an estimate for pages traversed during GC
575 self.collection_pages = self.gc_end_live_pages as f64;
576 trace!(
577 "collected pages = live at gc end {} - live at gc release {} = {}",
578 self.gc_release_live_pages,
579 self.gc_end_live_pages,
580 self.collection_pages
581 );
582 }
583}
584
585impl<VM: VMBinding> GCTriggerPolicy<VM> for MemBalancerTrigger {
586 fn is_gc_required(
587 &self,
588 space_full: bool,
589 space: Option<SpaceStats<VM>>,
590 plan: &dyn Plan<VM = VM>,
591 ) -> bool {
592 // Let the plan decide
593 plan.collection_required(space_full, space)
594 }
595
596 fn on_pending_allocation(&self, pages: usize) {
597 self.pending_pages.fetch_add(pages, Ordering::SeqCst);
598 }
599
600 fn on_gc_start(&self, mmtk: &'static MMTK<VM>) {
601 trace!("=== on_gc_start ===");
602 self.access_stats(|stats| {
603 stats.gc_start_time = Instant::now();
604 stats.allocation_time += (stats.gc_start_time - stats.gc_end_time).as_secs_f64();
605 trace!(
606 "gc_start = {:?}, allocation_time = {}",
607 stats.gc_start_time,
608 stats.allocation_time
609 );
610
611 if let Some(plan) = mmtk.get_plan().generational() {
612 stats.generational_mem_stats_on_gc_start(plan);
613 } else {
614 stats.non_generational_mem_stats_on_gc_start(mmtk);
615 }
616 });
617 }
618
619 fn on_gc_release(&self, mmtk: &'static MMTK<VM>) {
620 trace!("=== on_gc_release ===");
621 self.access_stats(|stats| {
622 if let Some(plan) = mmtk.get_plan().generational() {
623 stats.generational_mem_stats_on_gc_release(plan);
624 } else {
625 stats.non_generational_mem_stats_on_gc_release(mmtk);
626 }
627 });
628 }
629
630 fn on_gc_end(&self, mmtk: &'static MMTK<VM>) {
631 trace!("=== on_gc_end ===");
632 self.access_stats(|stats| {
633 stats.gc_end_time = Instant::now();
634 stats.collection_time += (stats.gc_end_time - stats.gc_start_time).as_secs_f64();
635 trace!(
636 "gc_end = {:?}, collection_time = {}",
637 stats.gc_end_time,
638 stats.collection_time
639 );
640
641 if let Some(plan) = mmtk.get_plan().generational() {
642 if stats.generational_mem_stats_on_gc_end(plan) {
643 self.compute_new_heap_limit(
644 mmtk.get_plan().get_reserved_pages(),
645 // We reserve an extra of min nursery. This ensures that we will not trigger
646 // a full heap GC in the next GC (if available pages is smaller than min nursery, we will force a full heap GC)
647 mmtk.get_plan().get_collection_reserved_pages()
648 + mmtk.gc_trigger.get_min_nursery_pages(),
649 stats,
650 );
651 }
652 } else {
653 stats.non_generational_mem_stats_on_gc_end(mmtk);
654 self.compute_new_heap_limit(
655 mmtk.get_plan().get_reserved_pages(),
656 mmtk.get_plan().get_collection_reserved_pages(),
657 stats,
658 );
659 }
660 });
661 // Clear pending allocation pages at the end of GC, no matter we used it or not.
662 self.pending_pages.store(0, Ordering::SeqCst);
663 }
664
665 fn is_heap_full(&self, plan: &dyn Plan<VM = VM>) -> bool {
666 // If reserved pages is larger than the current heap size, the heap is full.
667 plan.get_reserved_pages() > self.current_heap_pages.load(Ordering::Relaxed)
668 }
669
670 fn get_current_heap_size_in_pages(&self) -> usize {
671 self.current_heap_pages.load(Ordering::Relaxed)
672 }
673
674 fn get_max_heap_size_in_pages(&self) -> usize {
675 self.max_heap_pages
676 }
677
678 fn can_heap_size_grow(&self) -> bool {
679 self.current_heap_pages.load(Ordering::Relaxed) < self.max_heap_pages
680 }
681}
682impl MemBalancerTrigger {
683 fn new(min_heap_pages: usize, max_heap_pages: usize) -> Self {
684 Self {
685 min_heap_pages,
686 max_heap_pages,
687 pending_pages: AtomicUsize::new(0),
688 // start with min heap
689 current_heap_pages: AtomicUsize::new(min_heap_pages),
690 stats: AtomicRefCell::new(Default::default()),
691 }
692 }
693
694 fn access_stats<F>(&self, mut f: F)
695 where
696 F: FnMut(&mut MemBalancerStats),
697 {
698 let mut stats = self.stats.borrow_mut();
699 f(&mut stats);
700 }
701
702 fn compute_new_heap_limit(
703 &self,
704 live: usize,
705 extra_reserve: usize,
706 stats: &mut MemBalancerStats,
707 ) {
708 trace!("compute new heap limit: {:?}", stats);
709
710 // Constants from the original paper
711 const ALLOCATION_SMOOTH_FACTOR: f64 = 0.95;
712 const COLLECTION_SMOOTH_FACTOR: f64 = 0.5;
713 const TUNING_FACTOR: f64 = 0.2;
714
715 // Smooth memory/time for allocation/collection
716 let smooth = |prev: Option<f64>, cur, factor| {
717 prev.map(|p| p * factor + cur * (1.0f64 - factor))
718 .unwrap_or(cur)
719 };
720 let alloc_mem = smooth(
721 stats.allocation_pages_prev,
722 stats.allocation_pages,
723 ALLOCATION_SMOOTH_FACTOR,
724 );
725 let alloc_time = smooth(
726 stats.allocation_time_prev,
727 stats.allocation_time,
728 ALLOCATION_SMOOTH_FACTOR,
729 );
730 let gc_mem = smooth(
731 stats.collection_pages_prev,
732 stats.collection_pages,
733 COLLECTION_SMOOTH_FACTOR,
734 );
735 let gc_time = smooth(
736 stats.collection_time_prev,
737 stats.collection_time,
738 COLLECTION_SMOOTH_FACTOR,
739 );
740 trace!(
741 "after smoothing, alloc mem = {}, alloc_time = {}",
742 alloc_mem,
743 alloc_time
744 );
745 trace!(
746 "after smoothing, gc mem = {}, gc_time = {}",
747 gc_mem,
748 gc_time
749 );
750
751 // We got the smoothed stats. Now save the current stats as previous stats
752 stats.allocation_pages_prev = Some(stats.allocation_pages);
753 stats.allocation_pages = 0f64;
754 stats.allocation_time_prev = Some(stats.allocation_time);
755 stats.allocation_time = 0f64;
756 stats.collection_pages_prev = Some(stats.collection_pages);
757 stats.collection_pages = 0f64;
758 stats.collection_time_prev = Some(stats.collection_time);
759 stats.collection_time = 0f64;
760
761 // Calculate the square root
762 let e: f64 = if alloc_mem != 0f64 && gc_mem != 0f64 && alloc_time != 0f64 && gc_time != 0f64
763 {
764 let mut e = live as f64;
765 e *= alloc_mem / alloc_time;
766 e /= TUNING_FACTOR;
767 e /= gc_mem / gc_time;
768 e.sqrt()
769 } else {
770 // If any collected stat is abnormal, we use the fallback heuristics.
771 (live as f64 * 4096f64).sqrt()
772 };
773
774 // Get pending allocations
775 let pending_pages = self.pending_pages.load(Ordering::SeqCst);
776
777 // This is the optimal heap limit due to mem balancer. We will need to clamp the value to the defined min/max range.
778 let optimal_heap = live + e as usize + extra_reserve + pending_pages;
779 trace!(
780 "optimal = live {} + sqrt(live) {} + extra {}",
781 live,
782 e,
783 extra_reserve
784 );
785
786 // The new heap size must be within min/max.
787 let new_heap = optimal_heap.clamp(self.min_heap_pages, self.max_heap_pages);
788 debug!(
789 "MemBalander: new heap limit = {} pages (optimal = {}, clamped to [{}, {}])",
790 new_heap, optimal_heap, self.min_heap_pages, self.max_heap_pages
791 );
792 self.current_heap_pages.store(new_heap, Ordering::Relaxed);
793 }
794}