mmtk/
global_state.rs

1use atomic_refcell::AtomicRefCell;
2use std::collections::HashMap;
3use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
4use std::time::{Duration, Instant};
5
6/// This stores some global states for an MMTK instance.
7/// Some MMTK components like plans and allocators may keep an reference to the struct, and can access it.
8// This used to be a part of the `BasePlan`. In that case, any component that accesses
9// the states needs a reference to the plan. It makes it harder for us to reason about the access pattern
10// for the plan, as many components hold references to the plan. Besides, the states
11// actually are not related with a plan, they are just global states for MMTK. So we refactored
12// those fields to this separate struct. For components that access the state, they just need
13// a reference to the struct, and are no longer dependent on the plan.
14// We may consider further break down the fields into smaller structs.
15pub struct GlobalState {
16    /// The current GC status.
17    pub(crate) gc_status: GcStatusWord,
18    /// The time when a GC pause is requested. Used to calculate the time-to-yield metric.
19    pub(crate) pause_requested_time: AtomicRefCell<Option<Instant>>,
20    /// When did the current GC pause begin, i.e. when did all mutators finish stopping (see
21    /// `pause_requested_time`)? Consumed once the pause ends (mutators are about to resume) to
22    /// compute the "pause time" statistic: the duration mutators spend stopped for the pause.
23    pub(crate) pause_start_time: AtomicRefCell<Option<Instant>>,
24    /// Is the current GC an emergency collection? Emergency means we may run out of memory soon, and we should
25    /// attempt to collect as much as we can.
26    pub(crate) emergency_collection: AtomicBool,
27    /// Is the current GC triggered by the user?
28    pub(crate) user_triggered_collection: AtomicBool,
29    /// Is the current GC triggered internally by MMTK? This is unused for now. We may have internally triggered GC
30    /// for a concurrent plan.
31    pub(crate) internal_triggered_collection: AtomicBool,
32    /// Is the last GC internally triggered?
33    pub(crate) last_internal_triggered_collection: AtomicBool,
34    // Has an allocation succeeded since the emergency collection?
35    pub(crate) allocation_success: AtomicBool,
36    // Maximum number of failed attempts by a single thread
37    pub(crate) max_collection_attempts: AtomicUsize,
38    // Current collection attempt
39    pub(crate) cur_collection_attempts: AtomicUsize,
40    /// A counter for per-mutator stack scanning
41    pub(crate) scanned_stacks: AtomicUsize,
42    /// Have we scanned all the stacks?
43    pub(crate) stacks_prepared: AtomicBool,
44    /// A counter that keeps tracks of the number of bytes allocated since last stress test
45    pub(crate) allocation_bytes: AtomicUsize,
46    /// Are we inside the benchmark harness?
47    pub(crate) inside_harness: AtomicBool,
48    /// A counteer that keeps tracks of the number of bytes allocated by malloc
49    #[cfg(feature = "malloc_counted_size")]
50    pub(crate) malloc_bytes: AtomicUsize,
51    /// This stores the live bytes and the used bytes (by pages) for each space in last GC. This counter is only updated in the GC release phase.
52    pub(crate) live_bytes_in_last_gc: AtomicRefCell<HashMap<&'static str, LiveBytesStats>>,
53    /// The number of used pages at the end of the last GC. This can be used to estimate how many pages we have allocated since last GC.
54    pub(crate) used_pages_after_last_gc: AtomicUsize,
55}
56
57impl GlobalState {
58    /// Is MMTk initialized?
59    pub fn is_initialized(&self) -> bool {
60        self.gc_status.is_initialized()
61    }
62
63    /// Set the collection kind for the current GC. This is called before
64    /// scheduling collection to determin what kind of collection it will be.
65    pub fn set_collection_kind(
66        &self,
67        last_collection_was_exhaustive: bool,
68        heap_can_grow: bool,
69    ) -> bool {
70        self.cur_collection_attempts.store(
71            if self.user_triggered_collection.load(Ordering::Relaxed) {
72                1
73            } else {
74                self.determine_collection_attempts()
75            },
76            Ordering::Relaxed,
77        );
78
79        let emergency_collection = !self.is_internal_triggered_collection()
80            && last_collection_was_exhaustive
81            && self.cur_collection_attempts.load(Ordering::Relaxed) > 1
82            && !heap_can_grow;
83        self.emergency_collection
84            .store(emergency_collection, Ordering::Relaxed);
85
86        emergency_collection
87    }
88
89    fn determine_collection_attempts(&self) -> usize {
90        if !self.allocation_success.load(Ordering::Relaxed) {
91            self.max_collection_attempts.fetch_add(1, Ordering::Relaxed);
92        } else {
93            self.allocation_success.store(false, Ordering::Relaxed);
94            self.max_collection_attempts.store(1, Ordering::Relaxed);
95        }
96
97        self.max_collection_attempts.load(Ordering::Relaxed)
98    }
99
100    fn is_internal_triggered_collection(&self) -> bool {
101        let is_internal_triggered = self
102            .last_internal_triggered_collection
103            .load(Ordering::SeqCst);
104        // Remove this assertion when we have concurrent GC.
105        assert!(
106            !is_internal_triggered,
107            "We have no concurrent GC implemented. We should not have internally triggered GC"
108        );
109        is_internal_triggered
110    }
111
112    pub fn is_emergency_collection(&self) -> bool {
113        self.emergency_collection.load(Ordering::Relaxed)
114    }
115
116    /// Return true if this collection was triggered by application code.
117    pub fn is_user_triggered_collection(&self) -> bool {
118        self.user_triggered_collection.load(Ordering::Relaxed)
119    }
120
121    /// Reset collection state information.
122    pub fn reset_collection_trigger(&self) {
123        self.last_internal_triggered_collection.store(
124            self.internal_triggered_collection.load(Ordering::SeqCst),
125            Ordering::Relaxed,
126        );
127        self.internal_triggered_collection
128            .store(false, Ordering::SeqCst);
129        self.user_triggered_collection
130            .store(false, Ordering::Relaxed);
131    }
132
133    /// Are the stacks scanned?
134    pub fn stacks_prepared(&self) -> bool {
135        self.stacks_prepared.load(Ordering::SeqCst)
136    }
137
138    /// Prepare for stack scanning. This is usually used with `inform_stack_scanned()`.
139    /// This should be called before doing stack scanning.
140    pub fn prepare_for_stack_scanning(&self) {
141        self.scanned_stacks.store(0, Ordering::SeqCst);
142        self.stacks_prepared.store(false, Ordering::SeqCst);
143    }
144
145    /// Inform that 1 stack has been scanned. The argument `n_mutators` indicates the
146    /// total stacks we should scan. This method returns true if the number of scanned
147    /// stacks equals the total mutator count. Otherwise it returns false. This method
148    /// is thread safe and we guarantee only one thread will return true.
149    pub fn inform_stack_scanned(&self, n_mutators: usize) -> bool {
150        let old = self.scanned_stacks.fetch_add(1, Ordering::SeqCst);
151        debug_assert!(
152            old < n_mutators,
153            "The number of scanned stacks ({}) is more than the number of mutators ({})",
154            old,
155            n_mutators
156        );
157        let scanning_done = old + 1 == n_mutators;
158        if scanning_done {
159            self.stacks_prepared.store(true, Ordering::SeqCst);
160        }
161        scanning_done
162    }
163
164    /// Increase the allocation bytes and return the current allocation bytes after increasing
165    pub fn increase_allocation_bytes_by(&self, size: usize) -> usize {
166        let old_allocation_bytes = self.allocation_bytes.fetch_add(size, Ordering::SeqCst);
167        trace!(
168            "Stress GC: old_allocation_bytes = {}, size = {}, allocation_bytes = {}",
169            old_allocation_bytes,
170            size,
171            self.allocation_bytes.load(Ordering::Relaxed),
172        );
173        old_allocation_bytes + size
174    }
175
176    #[cfg(feature = "malloc_counted_size")]
177    pub fn get_malloc_bytes_in_pages(&self) -> usize {
178        crate::util::conversions::bytes_to_pages_up(self.malloc_bytes.load(Ordering::Relaxed))
179    }
180
181    #[cfg(feature = "malloc_counted_size")]
182    pub(crate) fn increase_malloc_bytes_by(&self, size: usize) {
183        self.malloc_bytes.fetch_add(size, Ordering::SeqCst);
184    }
185
186    #[cfg(feature = "malloc_counted_size")]
187    pub(crate) fn decrease_malloc_bytes_by(&self, size: usize) {
188        self.malloc_bytes.fetch_sub(size, Ordering::SeqCst);
189    }
190
191    pub(crate) fn set_used_pages_after_last_gc(&self, pages: usize) {
192        self.used_pages_after_last_gc
193            .store(pages, Ordering::Relaxed);
194    }
195
196    pub(crate) fn get_used_pages_after_last_gc(&self) -> usize {
197        self.used_pages_after_last_gc.load(Ordering::Relaxed)
198    }
199
200    /// Record that a GC pause has just been successfully requested. Called by `GCTrigger::request()`
201    /// on the thread that won the race to move the GC status to `PauseRequested`.
202    pub(crate) fn record_pause_requested_time(&self) {
203        let mut pause_requested_time = self.pause_requested_time.borrow_mut();
204        assert!(
205            pause_requested_time.is_none(),
206            "A pause was requested while a previous pause request time is still pending"
207        );
208        *pause_requested_time = Some(Instant::now());
209    }
210
211    /// Take the time-to-yield duration: the time elapsed since the pause was requested (i.e.
212    /// since `record_pause_requested_time` was last called). This should be called exactly once
213    /// all mutators have stopped for the pause.
214    pub(crate) fn take_time_to_yield(&self) -> Duration {
215        self.pause_requested_time
216            .borrow_mut()
217            .take()
218            .expect("Pause requested time was not recorded")
219            .elapsed()
220    }
221
222    /// Record that a GC pause has just begun, i.e. all mutators have just finished stopping.
223    pub(crate) fn record_pause_start_time(&self) {
224        let mut pause_start_time = self.pause_start_time.borrow_mut();
225        assert!(
226            pause_start_time.is_none(),
227            "A pause was started while a previous pause start time is still pending"
228        );
229        *pause_start_time = Some(Instant::now());
230    }
231
232    /// Take the pause time duration: the time elapsed since the pause began (i.e. since
233    /// `record_pause_start_time` was last called). This should be called exactly once the pause
234    /// is about to end, i.e. right before mutators are resumed.
235    pub(crate) fn take_pause_time(&self) -> Duration {
236        self.pause_start_time
237            .borrow_mut()
238            .take()
239            .expect("Pause start time was not recorded")
240            .elapsed()
241    }
242}
243
244impl Default for GlobalState {
245    fn default() -> Self {
246        Self {
247            gc_status: GcStatusWord::new(GcStatus::Uninitialized),
248            pause_requested_time: AtomicRefCell::new(None),
249            pause_start_time: AtomicRefCell::new(None),
250            stacks_prepared: AtomicBool::new(false),
251            emergency_collection: AtomicBool::new(false),
252            user_triggered_collection: AtomicBool::new(false),
253            internal_triggered_collection: AtomicBool::new(false),
254            last_internal_triggered_collection: AtomicBool::new(false),
255            allocation_success: AtomicBool::new(false),
256            max_collection_attempts: AtomicUsize::new(0),
257            cur_collection_attempts: AtomicUsize::new(0),
258            scanned_stacks: AtomicUsize::new(0),
259            allocation_bytes: AtomicUsize::new(0),
260            inside_harness: AtomicBool::new(false),
261            #[cfg(feature = "malloc_counted_size")]
262            malloc_bytes: AtomicUsize::new(0),
263            live_bytes_in_last_gc: AtomicRefCell::new(HashMap::new()),
264            used_pages_after_last_gc: AtomicUsize::new(0),
265        }
266    }
267}
268
269/// The status of MMTk's GC subsystem. This doubles as the "is MMTk initialized" flag (via
270/// [`GcStatus::Uninitialized`]) and as the state that tracks whether a GC is running and, if so,
271/// what phase it is in. See `GcStatusWord` (the internal atomic encoding of this type) for how
272/// this is stored atomically and which transitions between variants are legal.
273#[derive(PartialEq, Copy, Clone, Debug)]
274pub enum GcStatus {
275    /// MMTk has not been initialized yet, i.e. `initialize_collection()` has not been called, so
276    /// there are no GC worker threads available to run a collection.
277    Uninitialized,
278    /// MMTk is initialized, and no GC is running, pending, or requested.
279    NotInGC,
280    /// A concurrent GC's background work (e.g. concurrent marking) is running while mutators
281    /// continue to run normally. See `ConcurrentPlan::concurrent_work_in_progress`.
282    InConcurrentGC,
283    /// A stop-the-world pause is active: mutators are stopped and GC workers are doing pause
284    /// work (e.g. tracing).
285    InPause,
286    /// A GC pause has been requested (by `GcStatusWord::try_request_pause`) but mutators have
287    /// not all stopped yet.
288    PauseRequested,
289    /// Collection is currently disabled (e.g. by a mutator calling `disable_collection()`).
290    /// The usize payload is the non-zero nesting depth of disable calls:
291    /// each call to `disable_collection()` increments the depth, and each call to `enable_collection()`
292    /// decrements it. When the depth is about to reach zero, the status is transitioned to `NoInGC`.
293    Disabled(usize),
294}
295
296/// A lock-free, atomic encoding of [`GcStatus`]. This packs the variant tag into the low bits
297/// of a `usize` and, for `GcStatus::Disabled`, the nesting depth into the remaining high bits,
298/// so the whole status fits in a single machine word and can be updated with atomic operations
299/// instead of behind a `Mutex<GcStatus>`.
300///
301/// `GcStatus` is a state machine: only a handful of transitions between its variants are legal.
302/// Every legal transition is exposed here as its own method, each performing its own
303/// compare-and-swap retry loop and asserting that the transition is legal for the status it
304/// finds. Do not add a generic "set the status to X" method: doing so would make it possible to
305/// bypass the state machine's invariants.
306pub(crate) struct GcStatusWord(AtomicUsize);
307
308impl GcStatusWord {
309    /// Number of bits used to encode the variant tag. 3 bits is enough to distinguish the 6
310    /// variants, leaving the rest of the word for `Disabled`'s nesting depth.
311    const TAG_BITS: u32 = 3;
312    const TAG_MASK: usize = (1 << Self::TAG_BITS) - 1;
313
314    fn encode(status: GcStatus) -> usize {
315        match status {
316            GcStatus::Uninitialized => 0,
317            GcStatus::NotInGC => 1,
318            GcStatus::InConcurrentGC => 2,
319            GcStatus::InPause => 3,
320            GcStatus::PauseRequested => 4,
321            GcStatus::Disabled(depth) => {
322                debug_assert!(
323                    depth < (1 << (usize::BITS - Self::TAG_BITS)),
324                    "GC-disable nesting depth overflows the bits reserved for it"
325                );
326                5 | (depth << Self::TAG_BITS)
327            }
328        }
329    }
330
331    fn decode(bits: usize) -> GcStatus {
332        match bits & Self::TAG_MASK {
333            0 => GcStatus::Uninitialized,
334            1 => GcStatus::NotInGC,
335            2 => GcStatus::InConcurrentGC,
336            3 => GcStatus::InPause,
337            4 => GcStatus::PauseRequested,
338            5 => GcStatus::Disabled(bits >> Self::TAG_BITS),
339            _ => unreachable!("invalid encoded GcStatus tag"),
340        }
341    }
342
343    pub(crate) fn new(status: GcStatus) -> Self {
344        GcStatusWord(AtomicUsize::new(Self::encode(status)))
345    }
346
347    /// Read the current status.
348    pub(crate) fn load(&self) -> GcStatus {
349        Self::decode(self.0.load(Ordering::SeqCst))
350    }
351
352    /// Inner implementation of [`Self::transition`], handling encoding, decoding, and atomic RMW
353    /// operation.
354    fn transition_inner<F: FnMut(GcStatus) -> GcStatus>(&self, mut f: F) -> GcStatus {
355        // `update()` allows infallable atomic update, but is not stablized until 1.95.
356        // Migrate to `update()` after bumping MSRV.
357        #[allow(deprecated)]
358        let old_bits = self
359            .0
360            .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |bits| {
361                Some(Self::encode(f(Self::decode(bits))))
362            })
363            .unwrap(); // `f` always returns a status to move to, so this never returns `Err`.
364        Self::decode(old_bits)
365    }
366
367    /// Attempt to atomically transition the GC status using function `f`.  Return the status
368    /// atomically transitioned *from* (not the new status).  It will retry `f` if the status is
369    /// modified concurrently.
370    ///
371    /// Note: Returning the old status (rather than the new one) lets a caller tell whether it "won"
372    /// the race when multiple threads concurrently drive the same transition: only the thread whose
373    /// CAS actually moved the status away from a given old value can be sure it is the one
374    /// responsible for that transition, so it is the one that should perform any side effect that
375    /// must happen exactly once (e.g. notifying the scheduler). If `transition` returned the new
376    /// status instead, every racing thread would observe the same new status and none could tell
377    /// which of them caused it. `f` may be invoked more than once under contention.
378    fn transition<F: FnMut(GcStatus) -> GcStatus>(&self, mut f: F) -> GcStatus {
379        let mut maybe_new_state = None;
380        let old_state = self.transition_inner(|old_state| {
381            let new_state = f(old_state);
382            maybe_new_state = Some(new_state);
383            new_state
384        });
385        let new_state = maybe_new_state.unwrap();
386        log::trace!("GC status transitioned from {old_state:?} to {new_state:?}");
387        old_state
388    }
389
390    /// Inner implementation of [`Self::try_transition`], handling encoding, decoding, and atomic RMW
391    /// operation.
392    fn try_transition_inner<F: FnMut(GcStatus) -> Option<GcStatus>>(
393        &self,
394        mut f: F,
395    ) -> Result<GcStatus, GcStatus> {
396        // `try_update()` is not stablized until 1.95.  Migrate to `try_update()` after bumping MSRV.
397        #[allow(deprecated)]
398        self.0
399            .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |bits| {
400                f(Self::decode(bits)).map(Self::encode)
401            })
402            .map(Self::decode)
403            .map_err(Self::decode)
404    }
405
406    /// Attempt to atomically transition the GC status using function `f`.  Return `Ok(old_status)`
407    /// if `f` returns `Some(new_status)`, in which case it has atomically transitioned the state
408    /// from `old_state` to `new_state`.  Return `Err(old_status)` if `f` returns `None`, in which
409    /// case `old_status` is the status passed to the last invocation of `f`.  It will retry `f` if
410    /// `f` returns `Some` but the underlying status is modified concurrently.
411    fn try_transition<F: FnMut(GcStatus) -> Option<GcStatus>>(
412        &self,
413        mut f: F,
414    ) -> Result<GcStatus, GcStatus> {
415        let mut maybe_new_state = None;
416        let result = self.try_transition_inner(|old_state| {
417            maybe_new_state = f(old_state);
418            maybe_new_state
419        });
420
421        match result {
422            Ok(old_state) => {
423                let new_state = maybe_new_state.unwrap();
424                log::trace!("GC status transitioned from {old_state:?} to {new_state:?}")
425            }
426            Err(old_state) => {
427                log::trace!("GC status transition attempted, but remains {old_state:?}")
428            }
429        }
430
431        result
432    }
433
434    pub(crate) fn is_initialized(&self) -> bool {
435        self.load() != GcStatus::Uninitialized
436    }
437
438    pub(crate) fn is_disabled(&self) -> bool {
439        matches!(self.load(), GcStatus::Disabled(_))
440    }
441
442    /// `Uninitialized` -> `NotInGC`.
443    pub(crate) fn set_initialized(&self) {
444        self.transition(|status| {
445            assert!(
446                status == GcStatus::Uninitialized,
447                "Trying to set initialized GC status when it is not uninitialized"
448            );
449            GcStatus::NotInGC
450        });
451    }
452
453    /// Any status other than `Uninitialized` -> `Uninitialized`.
454    pub(crate) fn set_uninitialized(&self) {
455        self.transition(|status| {
456            assert!(
457                status != GcStatus::Uninitialized,
458                "Trying to set uninitialized GC status when it is already uninitialized"
459            );
460            GcStatus::Uninitialized
461        });
462    }
463
464    /// `PauseRequested` -> `InPause`.
465    pub(crate) fn set_in_pause(&self) {
466        self.transition(|status| {
467            assert!(
468                status == GcStatus::PauseRequested,
469                "Trying to set in-pause GC status in invalid status: {:?}",
470                status
471            );
472            GcStatus::InPause
473        });
474    }
475
476    /// `InPause` -> `InConcurrentGC`, e.g. once a GC pause has finished but concurrent work (such
477    /// as concurrent marking) was scheduled to continue after mutators resume.
478    pub(crate) fn set_in_concurrent_gc(&self) {
479        self.transition(|status| {
480            assert!(
481                status == GcStatus::InPause,
482                "Trying to set in-concurrent-gc GC status in invalid status: {:?}",
483                status
484            );
485            GcStatus::InConcurrentGC
486        });
487    }
488
489    /// `InConcurrentGC` -> `NotInGC`: the concurrent phase's work has drained and no pause
490    /// follows it.
491    pub(crate) fn set_concurrent_gc_finished(&self) -> bool {
492        self.try_transition(|status| match status {
493            GcStatus::InConcurrentGC => Some(GcStatus::NotInGC),
494            _ => None,
495        })
496        .is_ok()
497    }
498
499    /// `InPause` -> `NotInGC`, e.g. once a GC pause has finished and no concurrent work remains.
500    pub(crate) fn set_not_in_gc(&self) {
501        self.transition(|status| {
502            assert!(
503                status == GcStatus::InPause,
504                "Trying to set not-in-gc GC status in invalid status: {:?}",
505                status
506            );
507            GcStatus::NotInGC
508        });
509    }
510
511    /// `NotInGC`/`Disabled(depth)` -> `Disabled(depth + 1)`. Leaves the status unchanged if
512    /// collection cannot be disabled from the current status (e.g. a GC is in progress or has
513    /// been requested), and returns `Err` with the status that blocked the transition.
514    ///
515    /// On success, returns `Ok(true)` if this call actually switched collection from enabled to
516    /// disabled (i.e. it was the outermost `NotInGC` -> `Disabled(1)` transition), `Ok(false)` if
517    /// it only increased the nesting depth of an already-disabled status. Mirrors the meaning of
518    /// [`GcStatusWord::set_enabled`]'s return value.
519    pub(crate) fn set_disabled(&self) -> Result<bool, GcStatus> {
520        self.try_transition(|status| match status {
521            GcStatus::Disabled(depth) => Some(GcStatus::Disabled(depth + 1)),
522            GcStatus::NotInGC => Some(GcStatus::Disabled(1)),
523            _ => None,
524        })
525        .map(|old_status| old_status == GcStatus::NotInGC)
526    }
527
528    /// `Disabled(depth)` -> `Disabled(depth - 1)`, or `Disabled(1)` -> `NotInGC`. If collection is
529    /// not currently disabled, this is a no-op (the status is left unchanged). Returns `true` if
530    /// this call actually re-enabled collection (i.e. it was the outermost `Disabled(1)` ->
531    /// `NotInGC` transition), `false` if it only decremented the nesting depth, or if collection
532    /// was already enabled.
533    pub(crate) fn set_enabled(&self) -> bool {
534        let old = self.transition(|status| match status {
535            GcStatus::Disabled(1) => GcStatus::NotInGC,
536            GcStatus::Disabled(depth) => GcStatus::Disabled(depth - 1),
537            other => other,
538        });
539        old == GcStatus::Disabled(1)
540    }
541
542    /// `NotInGC`/`InConcurrentGC` -> `PauseRequested`, unless collection is disabled, MMTk is not
543    /// yet initialized, or a pause has already been requested, in which case `Err` is returned
544    /// with the status that prevented the transition (`Disabled(_)`, `Uninitialized`, or
545    /// `PauseRequested` respectively). On success, returns `Ok` with the status transitioned from.
546    pub(crate) fn try_request_pause(&self) -> Result<GcStatus, GcStatus> {
547        self.try_transition(|status| match status {
548            GcStatus::Disabled(_) | GcStatus::Uninitialized | GcStatus::PauseRequested => None,
549            GcStatus::NotInGC | GcStatus::InConcurrentGC => Some(GcStatus::PauseRequested),
550            _ => panic!("Trying to request a GC pause in invalid status: {status:?}"),
551        })
552    }
553}
554
555#[cfg(test)]
556mod gc_status_tests {
557    use super::{GcStatus, GcStatusWord};
558
559    #[test]
560    fn encode_decode_roundtrip() {
561        let statuses = [
562            GcStatus::Uninitialized,
563            GcStatus::NotInGC,
564            GcStatus::InConcurrentGC,
565            GcStatus::InPause,
566            GcStatus::PauseRequested,
567            GcStatus::Disabled(1),
568            GcStatus::Disabled(42),
569        ];
570        for status in statuses {
571            assert_eq!(GcStatusWord::decode(GcStatusWord::encode(status)), status);
572        }
573    }
574
575    #[test]
576    fn new_and_load_roundtrip() {
577        let statuses = [
578            GcStatus::Uninitialized,
579            GcStatus::NotInGC,
580            GcStatus::InConcurrentGC,
581            GcStatus::InPause,
582            GcStatus::PauseRequested,
583            GcStatus::Disabled(1),
584            GcStatus::Disabled(42),
585        ];
586        for status in statuses {
587            assert_eq!(GcStatusWord::new(status).load(), status);
588        }
589    }
590
591    #[test]
592    fn set_initialized_from_uninitialized() {
593        let word = GcStatusWord::new(GcStatus::Uninitialized);
594        assert!(!word.is_initialized());
595        word.set_initialized();
596        assert_eq!(word.load(), GcStatus::NotInGC);
597        assert!(word.is_initialized());
598    }
599
600    #[test]
601    #[should_panic(expected = "not uninitialized")]
602    fn set_initialized_panics_if_already_initialized() {
603        GcStatusWord::new(GcStatus::NotInGC).set_initialized();
604    }
605
606    #[test]
607    fn set_uninitialized_from_not_in_gc() {
608        let word = GcStatusWord::new(GcStatus::NotInGC);
609        word.set_uninitialized();
610        assert_eq!(word.load(), GcStatus::Uninitialized);
611    }
612
613    #[test]
614    #[should_panic(expected = "already uninitialized")]
615    fn set_uninitialized_panics_if_already_uninitialized() {
616        GcStatusWord::new(GcStatus::Uninitialized).set_uninitialized();
617    }
618
619    #[test]
620    fn try_request_pause_from_not_in_gc() {
621        let word = GcStatusWord::new(GcStatus::NotInGC);
622        assert_eq!(word.try_request_pause(), Ok(GcStatus::NotInGC));
623        assert_eq!(word.load(), GcStatus::PauseRequested);
624    }
625
626    #[test]
627    fn try_request_pause_from_in_concurrent_gc() {
628        let word = GcStatusWord::new(GcStatus::InConcurrentGC);
629        assert_eq!(word.try_request_pause(), Ok(GcStatus::InConcurrentGC));
630        assert_eq!(word.load(), GcStatus::PauseRequested);
631    }
632
633    #[test]
634    fn try_request_pause_when_already_requested() {
635        let word = GcStatusWord::new(GcStatus::PauseRequested);
636        assert_eq!(word.try_request_pause(), Err(GcStatus::PauseRequested));
637        // Idempotent: the status is unchanged, not "double requested".
638        assert_eq!(word.load(), GcStatus::PauseRequested);
639    }
640
641    /// A GC pause should never be requested while one is already underway: by the time the
642    /// status reaches `InPause`, all mutators must already be stopped, so no mutator should be
643    /// calling `try_request_pause` at all. Observing `InPause` here indicates a state-machine
644    /// violation elsewhere, so it must panic rather than being silently treated as a no-op.
645    #[test]
646    #[should_panic(expected = "invalid status")]
647    fn try_request_pause_panics_when_already_in_pause() {
648        let _ = GcStatusWord::new(GcStatus::InPause).try_request_pause();
649    }
650
651    #[test]
652    fn try_request_pause_when_disabled() {
653        let word = GcStatusWord::new(GcStatus::Disabled(1));
654        assert_eq!(word.try_request_pause(), Err(GcStatus::Disabled(1)));
655        // Unchanged: disabling is not overridden by a pause request.
656        assert_eq!(word.load(), GcStatus::Disabled(1));
657    }
658
659    /// Allocation can call `poll()` (and thus `try_request_pause`) before
660    /// `initialize_collection()` has been called, e.g. if the heap fills up before the VM
661    /// binding initializes MMTk's GC worker threads. This must not panic here: the caller (e.g.
662    /// `Space::not_acquiring`) is responsible for producing a clear "GC is not allowed here"
663    /// error once it knows allocation has genuinely failed.
664    #[test]
665    fn try_request_pause_when_uninitialized() {
666        let word = GcStatusWord::new(GcStatus::Uninitialized);
667        assert_eq!(word.try_request_pause(), Err(GcStatus::Uninitialized));
668        assert_eq!(word.load(), GcStatus::Uninitialized);
669    }
670
671    #[test]
672    fn set_in_pause_from_pause_requested() {
673        let word = GcStatusWord::new(GcStatus::PauseRequested);
674        word.set_in_pause();
675        assert_eq!(word.load(), GcStatus::InPause);
676    }
677
678    #[test]
679    #[should_panic(expected = "invalid status")]
680    fn set_in_pause_panics_if_not_requested() {
681        GcStatusWord::new(GcStatus::NotInGC).set_in_pause();
682    }
683
684    #[test]
685    fn set_disabled_from_not_in_gc() {
686        let word = GcStatusWord::new(GcStatus::NotInGC);
687        assert_eq!(word.set_disabled(), Ok(true));
688        assert_eq!(word.load(), GcStatus::Disabled(1));
689    }
690
691    #[test]
692    fn set_disabled_nests() {
693        let word = GcStatusWord::new(GcStatus::Disabled(1));
694        assert_eq!(word.set_disabled(), Ok(false));
695        assert_eq!(word.load(), GcStatus::Disabled(2));
696
697        assert_eq!(word.set_disabled(), Ok(false));
698        assert_eq!(word.load(), GcStatus::Disabled(3));
699    }
700
701    #[test]
702    fn set_disabled_fails_without_changing_status() {
703        for status in [
704            GcStatus::Uninitialized,
705            GcStatus::InConcurrentGC,
706            GcStatus::PauseRequested,
707            GcStatus::InPause,
708        ] {
709            let word = GcStatusWord::new(status);
710            assert_eq!(word.set_disabled(), Err(status));
711            assert_eq!(word.load(), status);
712        }
713    }
714
715    #[test]
716    fn set_enabled_decrements_nesting() {
717        let word = GcStatusWord::new(GcStatus::Disabled(3));
718        assert!(!word.set_enabled());
719        assert_eq!(word.load(), GcStatus::Disabled(2));
720    }
721
722    #[test]
723    fn set_enabled_to_not_in_gc_at_zero_depth() {
724        let word = GcStatusWord::new(GcStatus::Disabled(1));
725        assert!(word.set_enabled());
726        assert_eq!(word.load(), GcStatus::NotInGC);
727    }
728
729    #[test]
730    fn set_disabled_and_set_enabled_nest_round_trip() {
731        let word = GcStatusWord::new(GcStatus::NotInGC);
732        assert!(word.set_disabled().is_ok());
733        assert!(word.set_disabled().is_ok());
734        assert!(word.set_disabled().is_ok());
735        assert_eq!(word.load(), GcStatus::Disabled(3));
736
737        // Only the call that brings the nesting depth back to 0 (i.e. all the way back to
738        // `NotInGC`) should return `true`.
739        assert!(!word.set_enabled());
740        assert_eq!(word.load(), GcStatus::Disabled(2));
741        assert!(!word.set_enabled());
742        assert_eq!(word.load(), GcStatus::Disabled(1));
743        assert!(word.set_enabled());
744        assert_eq!(word.load(), GcStatus::NotInGC);
745    }
746
747    #[test]
748    fn set_enabled_is_noop_if_not_disabled() {
749        for status in [
750            GcStatus::Uninitialized,
751            GcStatus::NotInGC,
752            GcStatus::InConcurrentGC,
753            GcStatus::InPause,
754            GcStatus::PauseRequested,
755        ] {
756            let word = GcStatusWord::new(status);
757            assert!(!word.set_enabled());
758            assert_eq!(word.load(), status);
759        }
760    }
761
762    #[test]
763    fn set_in_concurrent_gc_from_in_pause() {
764        let word = GcStatusWord::new(GcStatus::InPause);
765        word.set_in_concurrent_gc();
766        assert_eq!(word.load(), GcStatus::InConcurrentGC);
767    }
768
769    #[test]
770    #[should_panic(expected = "invalid status")]
771    fn set_in_concurrent_gc_panics_if_not_in_pause() {
772        GcStatusWord::new(GcStatus::NotInGC).set_in_concurrent_gc();
773    }
774
775    #[test]
776    fn set_concurrent_gc_finished_from_in_concurrent_gc() {
777        let word = GcStatusWord::new(GcStatus::InConcurrentGC);
778        assert!(word.set_concurrent_gc_finished());
779        assert_eq!(word.load(), GcStatus::NotInGC);
780    }
781
782    #[test]
783    fn set_concurrent_gc_finished_does_not_overwrite_a_requested_pause() {
784        // A mutator won the race to request a pause while the concurrent work was draining. The
785        // request must survive, and the caller must be told it did not perform the transition so
786        // it does not report the GC as over.
787        let word = GcStatusWord::new(GcStatus::PauseRequested);
788        assert!(!word.set_concurrent_gc_finished());
789        assert_eq!(word.load(), GcStatus::PauseRequested);
790    }
791
792    #[test]
793    fn set_concurrent_gc_finished_is_a_no_op_outside_a_concurrent_gc() {
794        for status in [
795            GcStatus::NotInGC,
796            GcStatus::InPause,
797            GcStatus::Disabled(1),
798            GcStatus::Uninitialized,
799        ] {
800            let word = GcStatusWord::new(status);
801            assert!(!word.set_concurrent_gc_finished());
802            assert_eq!(word.load(), status);
803        }
804    }
805
806    #[test]
807    fn set_not_in_gc_from_in_pause() {
808        let word = GcStatusWord::new(GcStatus::InPause);
809        word.set_not_in_gc();
810        assert_eq!(word.load(), GcStatus::NotInGC);
811    }
812
813    #[test]
814    #[should_panic(expected = "invalid status")]
815    fn set_not_in_gc_panics_if_not_in_pause() {
816        GcStatusWord::new(GcStatus::InConcurrentGC).set_not_in_gc();
817    }
818
819    #[test]
820    fn is_disabled_reflects_status() {
821        assert!(GcStatusWord::new(GcStatus::Disabled(1)).is_disabled());
822        assert!(!GcStatusWord::new(GcStatus::NotInGC).is_disabled());
823    }
824}
825
826/// Statistics for the live bytes in the last GC. The statistics is per space.
827#[derive(Copy, Clone, Debug)]
828pub struct LiveBytesStats {
829    /// Total accumulated bytes of live objects in the space.
830    pub live_bytes: usize,
831    /// Total pages used by the space.
832    pub used_pages: usize,
833    /// Total bytes used by the space, computed from `used_pages`.
834    /// The ratio of live_bytes and used_bytes reflects the utilization of the memory in the space.
835    pub used_bytes: usize,
836}
mmtk/global_state.rs

mmtk/
global_state.rs
