mmtk/plan/lxr/
global.rs

1use super::block_allocation::BlockAllocation;
2use super::gc_work::nursery_sweeping::ReleaseLOSNursery;
3use super::gc_work::prepare::FastRCPrepare;
4use super::gc_work::rc::ProcessDecs;
5use super::gc_work::LXRGCWorkContext;
6use super::mature_evac::MatureEvacuationSet;
7use super::mutator::ALLOCATOR_MAPPING;
8use super::{LazySweepingJobsCounter, LAZY_SWEEPING_JOBS};
9use crate::plan::concurrent::global::ConcurrentPlan;
10use crate::plan::concurrent::Pause;
11use crate::plan::global::CommonPlan;
12use crate::plan::global::{BasePlan, CreateGeneralPlanArgs, CreateSpecificPlanArgs};
13use crate::plan::lxr::gc_work::mature_sweeping::{RCSweepMatureAfterSATBLOS, SweepDeadCycles};
14use crate::plan::lxr::gc_work::nursery_sweeping::SweepBlocksAfterDecs;
15use crate::plan::lxr::gc_work::prepare::{ConcurrentChunkMetadataZeroing, PrepareChunksForFullGC};
16use crate::plan::lxr::mature_evac::MatureEvecRemSet;
17use crate::plan::AllocationSemantics;
18use crate::plan::MutatorContext;
19use crate::plan::Plan;
20use crate::plan::PlanConstraints;
21use crate::policy::immix::block::Block;
22use crate::policy::immix::ImmixSpaceArgs;
23use crate::policy::largeobjectspace::LargeObjectSpace;
24use crate::policy::space::Space;
25use crate::scheduler::gc_work::*;
26use crate::util::alloc::allocators::AllocatorSelector;
27#[cfg(feature = "analysis")]
28use crate::util::analysis::GcHookWork;
29use crate::util::constants::*;
30use crate::util::copy::*;
31use crate::util::heap::{SpaceStats, VMRequest};
32use crate::util::metadata::side_metadata::SideMetadataContext;
33use crate::util::metadata::MetadataSpec;
34use crate::util::rc::{RefCountHelper, RC_TABLE};
35#[cfg(feature = "sanity")]
36use crate::util::sanity::sanity_checker::*;
37use crate::util::{metadata, Address, ObjectReference};
38use crate::vm::ActivePlan;
39use crate::vm::{Collection, ObjectModel, VMBinding};
40use crate::BarrierSelector;
41use crate::{policy::immix::ImmixSpace, util::opaque_pointer::VMWorkerThread};
42use crate::{scheduler::*, MMTK};
43use atomic::{Atomic, Ordering};
44use crossbeam::queue::SegQueue;
45use enum_map::EnumMap;
46use spin::Lazy;
47use std::sync::atomic::{AtomicBool, AtomicUsize};
48use std::sync::{Condvar, Mutex, RwLock};
49
50const LOG_CONSERVATIVE_SURVIVAL_RATIO_MULTIPLER: usize = 1;
51
52static HEAP_AFTER_GC: AtomicUsize = AtomicUsize::new(0);
53
54use mmtk_macros::{HasSpaces, PlanTraceObject};
55
56#[derive(HasSpaces, PlanTraceObject)]
57pub struct LXR<VM: VMBinding> {
58    #[post_scan]
59    #[space]
60    #[copy_semantics(CopySemantics::DefaultCopy)]
61    pub immix_space: ImmixSpace<VM>,
62    #[parent]
63    pub common: CommonPlan<VM>,
64    /// Always true for non-rc immix.
65    /// For RC immix, this is used for enable backup tracing.
66    perform_cycle_collection: AtomicBool,
67    current_pause: Atomic<Option<Pause>>,
68    previous_pause: Atomic<Option<Pause>>,
69    hint_cycle_gc: AtomicBool,
70    hint_emergency_gc: AtomicBool,
71    avail_pages_at_end_of_last_gc: AtomicUsize,
72    zeroing_packets_scheduled: AtomicBool,
73    decide_cycle_collection: (Mutex<bool>, Condvar),
74    in_concurrent_marking: AtomicBool,
75    pub prev_roots: RwLock<SegQueue<Vec<ObjectReference>>>,
76    pub curr_roots: RwLock<SegQueue<Vec<ObjectReference>>>,
77    pub rc: RefCountHelper<VM>,
78    block_allocation: BlockAllocation<VM>,
79    pub(super) evac_set: MatureEvacuationSet,
80    pub(super) mature_evac_remset: MatureEvecRemSet<VM>,
81    pub(super) num_clean_blocks_released_lazy: AtomicUsize,
82    pub(super) possibly_dead_mature_blocks: SegQueue<(Block, bool)>,
83}
84
85pub static LXR_CONSTRAINTS: Lazy<PlanConstraints> = Lazy::new(|| PlanConstraints {
86    moves_objects: super::NURSERY_EVACUATION || super::MATURE_EVACUATION,
87    // Max immix object size is half of a block.
88    max_non_los_default_alloc_bytes: crate::policy::immix::MAX_IMMIX_OBJECT_SIZE,
89    barrier: BarrierSelector::FieldBarrier,
90    needs_log_bit: true,
91    needs_field_log_bit: true,
92    rc_enabled: true,
93    needs_prepare_mutator: false,
94    ..PlanConstraints::default()
95});
96
97impl<VM: VMBinding> Plan for LXR<VM> {
98    fn current_gc_may_move_object(&self) -> bool {
99        true
100    }
101
102    fn collection_required(&self, space_full: bool, _space: Option<SpaceStats<Self::VM>>) -> bool {
103        // Spaces or heap full
104        if self.base().collection_required(self, space_full) {
105            return true;
106        }
107        // SATB is finished
108        if self.concurrent_work_in_progress() && super::concurrent_marking_packets_drained() {
109            return true;
110        }
111        // Bound the pause by bounding the work it has to do (default to usize::MAX - disabled)
112        if self.rc.inc_buffer_size() >= *self.base().options.lxr_inc_buffer_limit {
113            return true;
114        }
115        // Survival limits
116        let total_young_alloc_pages =
117            self.block_allocation.total_young_allocation_in_bytes() >> LOG_BYTES_IN_MBYTE;
118        // Use copy promotion ratio, or just total promotion ratio.
119        let ratio = if LXR_CONSTRAINTS.moves_objects {
120            super::SURVIVAL_RATIO_PREDICTOR.copy_promote_ratio()
121        } else {
122            super::SURVIVAL_RATIO_PREDICTOR.promote_ratio()
123        };
124        let predicted_survival_mb: usize = ((total_young_alloc_pages as f64 * ratio) as usize)
125            << LOG_CONSERVATIVE_SURVIVAL_RATIO_MULTIPLER;
126        if predicted_survival_mb >= *self.base().options.lxr_max_survival_mb {
127            return true;
128        }
129        if !self.immix_space.common().contiguous {
130            let available_to_space = self.get_total_pages() - self.get_used_pages();
131            if predicted_survival_mb >= available_to_space {
132                return true;
133            }
134        }
135        false
136    }
137
138    fn last_collection_was_exhaustive(&self) -> bool {
139        self.previous_pause.load(Ordering::SeqCst) == Some(Pause::Full)
140    }
141
142    fn constraints(&self) -> &'static PlanConstraints {
143        &LXR_CONSTRAINTS
144    }
145
146    fn create_copy_config(&'static self) -> CopyConfig<VM> {
147        use enum_map::enum_map;
148        CopyConfig {
149            copy_mapping: enum_map! {
150                CopySemantics::DefaultCopy => CopySelector::Immix(0),
151                _ => CopySelector::Unused,
152            },
153            space_mapping: vec![(CopySelector::Immix(0), &self.immix_space)],
154            constraints: &LXR_CONSTRAINTS,
155        }
156    }
157
158    fn schedule_collection(&'static self, scheduler: &GCWorkScheduler<VM>) {
159        if !super::LazySweepingJobs::all_finished() {
160            warn!("LXR Lazy Sweeping Not Finished");
161        }
162        let pause = self.select_collection_kind();
163        // Wait for concurrent packets
164        // Mark table zeroing
165        if pause == Pause::InitialMark || pause == Pause::Full {
166            self.schedule_mark_table_zeroing_tasks(Some(pause))
167        }
168        self.zeroing_packets_scheduled
169            .store(false, Ordering::SeqCst);
170        // Set current pause kind
171        self.current_pause.store(Some(pause), Ordering::SeqCst);
172        self.perform_cycle_collection
173            .store(pause != Pause::RefCount, Ordering::SeqCst);
174        // Schedule work
175        match pause {
176            Pause::Full => self.schedule_emergency_full_heap_collection(scheduler),
177            Pause::RefCount => self.schedule_rc_collection(scheduler),
178            Pause::InitialMark => self.schedule_concurrent_marking_initial_pause(scheduler),
179            Pause::FinalMark => self.schedule_concurrent_marking_final_pause(scheduler),
180        }
181        // Analysis routine that is ran. It is generally recommended to take advantage
182        // of the scheduling system we have in place for more performance
183        #[cfg(feature = "analysis")]
184        scheduler.work_buckets[WorkBucketStage::Unconstrained].add(GcHookWork);
185        // Resume mutators
186        if pause == Pause::Full || pause == Pause::FinalMark {
187            #[cfg(feature = "sanity")]
188            scheduler.work_buckets[WorkBucketStage::Final].add(ScheduleSanityGC::<Self>::new(self));
189        }
190    }
191
192    fn get_allocator_mapping(&self) -> &'static EnumMap<AllocationSemantics, AllocatorSelector> {
193        &ALLOCATOR_MAPPING
194    }
195
196    fn prepare(&mut self, _tls: VMWorkerThread) {
197        let pause = self.current_pause().unwrap();
198        if pause == Pause::FinalMark || pause == Pause::Full {
199            self.common.los.is_end_of_satb_or_full_gc = true;
200            // release nursery memory before mature evacuation, to reduce the chance of to-space overflow.
201            self.immix_space.scheduler().work_buckets[WorkBucketStage::Unconstrained]
202                .add(ReleaseLOSNursery);
203        }
204
205        let starts_mark_cycle = pause == Pause::Full || pause == Pause::InitialMark;
206        // Only do prepare if we start a new mark cycle. Otherwise, let thoe spaces have sticky mark bits.
207        if starts_mark_cycle {
208            // We have tested that the following spaces -- they are used by Julia
209            self.common.immortal.prepare();
210            #[cfg(feature = "vm_space")]
211            self.common.base.vm_space.prepare();
212            // TODO: We haven't tested these spaces. But ideally they should be handled in the same way here.
213            self.common.prepare_nonmoving_space(starts_mark_cycle);
214            #[cfg(feature = "code_space")]
215            self.common.base.code_space.prepare();
216            #[cfg(feature = "code_space")]
217            self.common.base.code_lo_space.prepare();
218            #[cfg(feature = "ro_space")]
219            self.common.base.ro_space.prepare();
220        }
221        // LOS is aware of LXR. Call its prepare unconditionally.
222        self.common.los.prepare(starts_mark_cycle);
223
224        if super::MATURE_EVACUATION && (pause == Pause::FinalMark || pause == Pause::Full) {
225            self.process_mature_evacuation_remset();
226        }
227        if super::MATURE_EVACUATION && (pause == Pause::InitialMark || pause == Pause::Full) {
228            // Select mature evacuation set
229            self.schedule_defrag_selection_packets();
230        }
231        self.num_clean_blocks_released_lazy
232            .store(0, Ordering::SeqCst);
233        self.immix_space.prepare_rc(pause);
234        self.block_allocation
235            .reset_block_mark_for_mutator_reused_blocks(pause);
236    }
237
238    fn release(&mut self, tls: VMWorkerThread) {
239        super::SURVIVAL_RATIO_PREDICTOR.update_ratios();
240        let pause = self.current_pause().unwrap();
241        if pause == Pause::FinalMark || pause == Pause::Full {
242            VM::VMCollection::update_weak_processor(false);
243        }
244        <VM as VMBinding>::VMCollection::vm_release();
245        self.common.los.is_end_of_satb_or_full_gc = false;
246        self.common
247            .release(tls, pause == Pause::Full || pause == Pause::FinalMark);
248        self.block_allocation
249            .sweep_nursery_blocks(self.immix_space.scheduler(), pause);
250        self.block_allocation.sweep_mutator_reused_blocks(pause);
251        // Check if we want to do all decs and sweeping in the pause
252        if super::disable_lasy_dec_for_current_gc() {
253            self.immix_space
254                .scheduler()
255                .process_concurrent_packets_in_pause();
256        } else {
257            debug_assert_ne!(pause, Pause::Full);
258        }
259        self.immix_space.release_rc();
260        self.schedule_mature_sweeping(pause);
261        // swap roots
262        let mut prev_roots = self.prev_roots.write().unwrap();
263        let mut curr_roots = self.curr_roots.write().unwrap();
264        std::mem::swap::<SegQueue<_>>(&mut prev_roots, &mut curr_roots);
265        debug_assert!(curr_roots.is_empty());
266    }
267
268    fn get_collection_reserved_pages(&self) -> usize {
269        let survival = {
270            let predicted_survival = (self.block_allocation.clean_nursery_mb() as f64
271                * super::SURVIVAL_RATIO_PREDICTOR.copy_promote_ratio())
272                as usize;
273            predicted_survival << LOG_CONSERVATIVE_SURVIVAL_RATIO_MULTIPLER
274        };
275        survival + self.immix_space.defrag_headroom_pages()
276    }
277
278    fn get_used_pages(&self) -> usize {
279        self.immix_space.reserved_pages() + self.common.get_used_pages()
280    }
281
282    fn base(&self) -> &BasePlan<VM> {
283        &self.common.base
284    }
285
286    fn base_mut(&mut self) -> &mut BasePlan<VM> {
287        &mut self.common.base
288    }
289
290    fn common(&self) -> &CommonPlan<VM> {
291        &self.common
292    }
293
294    /// Get a mutable reference to the common plan. See [`Self::common`].
295    fn common_mut(&mut self) -> &mut CommonPlan<Self::VM> {
296        &mut self.common
297    }
298
299    fn on_pause_start(&self, mmtk: &'static MMTK<Self::VM>) {
300        super::NO_EVAC.store(false, Ordering::SeqCst);
301        let pause = self.current_pause().unwrap();
302
303        // Individual RC pauses that don't overlap with concurrent tracing consist of a GC cycle.
304        // Concurrent tracing, including RC pauses in between, counts as one GC cycle.
305        // A Full GC counts as a GC cycle.
306        if pause == Pause::RefCount && !self.concurrent_work_in_progress()
307            || pause == Pause::InitialMark
308            || pause == Pause::Full
309        {
310            mmtk.gc_trigger.policy.on_gc_start(mmtk);
311        }
312
313        super::SURVIVAL_RATIO_PREDICTOR
314            .set_alloc_size(self.block_allocation.total_young_allocation_in_bytes());
315
316        if pause == Pause::Full || pause == Pause::InitialMark {
317            // Reset block mark and object mark table.
318            let work_packets = self.generate_full_trace_prepare_tasks();
319            self.immix_space.scheduler().work_buckets[WorkBucketStage::RCProcessIncs]
320                .bulk_add(work_packets);
321        }
322
323        for mutator in <VM as VMBinding>::VMActivePlan::mutators() {
324            mutator.flush();
325        }
326
327        if pause == Pause::FinalMark {
328            self.set_concurrent_marking_state(false);
329        }
330    }
331
332    fn on_pause_end(&mut self, mmtk: &'static MMTK<Self::VM>, tls: VMWorkerThread) {
333        super::DISABLE_LASY_DEC_FOR_CURRENT_GC.store(false, Ordering::SeqCst);
334        // self.immix_space.flush_page_resource();
335        let pause = self.current_pause().unwrap();
336        if pause == Pause::InitialMark {
337            self.set_concurrent_marking_state(true);
338        }
339        self.previous_pause.store(Some(pause), Ordering::SeqCst);
340        self.current_pause.store(None, Ordering::SeqCst);
341        LAZY_SWEEPING_JOBS.write().swap();
342        if super::LAZY_DECREMENTS {
343            let perform_cycle_collection =
344                self.get_available_pages() < super::CYCLE_TRIGGER_THRESHOLD;
345            self.hint_cycle_gc
346                .store(perform_cycle_collection, Ordering::SeqCst);
347            self.hint_emergency_gc.store(false, Ordering::SeqCst);
348            self.perform_cycle_collection.store(false, Ordering::SeqCst);
349        }
350        self.avail_pages_at_end_of_last_gc
351            .store(self.get_available_pages(), Ordering::SeqCst);
352        HEAP_AFTER_GC.store(self.get_reserved_pages(), Ordering::SeqCst);
353
354        self.common_mut().on_pause_end(tls);
355
356        // Individual RC pauses that don't overlap with concurrent tracing consist of a GC cycle.
357        // Concurrent tracing, including RC pauses in between, counts as one GC cycle.
358        // A Full GC counts as a GC cycle.
359        if pause == Pause::RefCount && !self.concurrent_work_in_progress()
360            || pause == Pause::FinalMark
361            || pause == Pause::Full
362        {
363            mmtk.gc_trigger.policy.on_gc_end(mmtk);
364        }
365    }
366
367    fn root_scanning_stage(&self) -> WorkBucketStage {
368        // With `non_moving_root`, roots reported as objects have to be counted before
369        // `RCProcessIncs` -- which may evacuate -- can move anything, so scanning happens one
370        // stage earlier.  That costs parallelism: it stops root scanning from overlapping with
371        // the increments the barrier already recorded, and root scanning is bounded by the
372        // mutator count.  Without the feature no root is reported as an object, so scan in
373        // `RCProcessIncs` itself and keep the overlap.
374        #[cfg(feature = "non_moving_root")]
375        return WorkBucketStage::RCProcessIncsNonMoving;
376        #[cfg(not(feature = "non_moving_root"))]
377        return WorkBucketStage::RCProcessIncs;
378    }
379
380    fn concurrent(&self) -> Option<&dyn ConcurrentPlan<VM = VM>> {
381        Some(self)
382    }
383}
384
385impl<VM: VMBinding> ConcurrentPlan for LXR<VM> {
386    fn current_pause(&self) -> Option<Pause> {
387        self.current_pause.load(Ordering::SeqCst)
388    }
389
390    fn concurrent_work_in_progress(&self) -> bool {
391        self.in_concurrent_marking.load(Ordering::Acquire)
392    }
393
394    fn on_concurrent_work_interrupted(&self) {
395        // Do nothing
396    }
397
398    fn on_concurrent_work_drained(&self) {
399        // Concurrent marking will end with a pause. We dont need to do anything here.
400        if self.in_concurrent_marking.load(Ordering::Acquire) {
401            return;
402        }
403        // Concurrent marking was not running, so the only concurrent work was lazy decrements
404        // and the lazy sweeping they chain into. `LazySweepingJobs::all_finished` covers both.
405        debug_assert!(
406            super::LazySweepingJobs::all_finished(),
407            "Deferred decrement or sweeping jobs outstanding with every GC worker parked and no work left"
408        );
409        // Leave `InConcurrentGC`.
410        if self
411            .base()
412            .global_state
413            .gc_status
414            .set_concurrent_gc_finished()
415        {
416            <VM as VMBinding>::VMCollection::concurrent_work_finished_no_pause();
417        }
418    }
419}
420
421impl<VM: VMBinding> LXR<VM> {
422    pub fn new(args: CreateGeneralPlanArgs<VM>) -> Box<Self> {
423        assert!(
424            VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC.is_in_header(),
425            "LXR does not support placing forwarding bits on the side."
426        );
427        let num_workers = args.scheduler.num_workers();
428        #[allow(unused_mut)]
429        let mut specs = vec![
430            MetadataSpec::OnSide(RC_TABLE),
431            MetadataSpec::OnSide(
432                *VM::VMObjectModel::GLOBAL_FIELD_UNLOG_BIT_SPEC
433                    .as_spec()
434                    .extract_side_spec(),
435            ),
436        ];
437        // The per-object log bit has to be registered too, not just the per-field one. LXR's
438        // own barrier only consults the field bits, this can be an issue for the probable write API (no field given).
439        // With `lxr_object_log`, the probable write API also logs the object bit.
440        // TODO: We should examine if we can steal a bit from the field log its as the 'logical' object log bit.
441        // We potentially could use the field log bit at the object start, or (object ref - lower bound) -- there should
442        // be no field at those addresses.
443        #[cfg(feature = "lxr_object_log")]
444        specs.push(*VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.as_spec());
445        let immix_specs = metadata::extract_side_metadata(&specs);
446        let global_side_metadata_specs = SideMetadataContext::new_global_specs(&immix_specs);
447        let mut plan_args = CreateSpecificPlanArgs {
448            global_args: args,
449            constraints: &LXR_CONSTRAINTS,
450            global_side_metadata_specs,
451        };
452        let immix_space = ImmixSpace::new(
453            plan_args.get_mature_space_args("immix", true, false, VMRequest::discontiguous()),
454            ImmixSpaceArgs {
455                never_move_objects: false,
456                mixed_age: false,
457            },
458        );
459        let mut lxr = Box::new(LXR {
460            immix_space,
461            common: CommonPlan::new(plan_args),
462            perform_cycle_collection: AtomicBool::new(false),
463            hint_cycle_gc: AtomicBool::new(false),
464            hint_emergency_gc: AtomicBool::new(false),
465            current_pause: Atomic::new(None),
466            previous_pause: Atomic::new(None),
467            avail_pages_at_end_of_last_gc: AtomicUsize::new(0),
468            zeroing_packets_scheduled: AtomicBool::new(false),
469            decide_cycle_collection: (Mutex::new(true), Condvar::new()),
470            in_concurrent_marking: AtomicBool::new(false),
471            prev_roots: Default::default(),
472            curr_roots: Default::default(),
473            rc: RefCountHelper::NEW,
474            block_allocation: BlockAllocation::new(),
475            evac_set: MatureEvacuationSet::default(),
476            mature_evac_remset: MatureEvecRemSet::new(num_workers),
477            possibly_dead_mature_blocks: Default::default(),
478            num_clean_blocks_released_lazy: Default::default(),
479        });
480
481        lxr.gc_init();
482
483        // Note: `verify_side_metadata_sanity` is invoked later by `MMTK::new`, after the dynamic
484        // side metadata base address has been initialized. It must not be called here during plan
485        // construction, as the side metadata layout is not yet registered at this point.
486
487        lxr
488    }
489
490    pub fn cm_enabled(&self) -> bool {
491        !cfg!(feature = "lxr_no_cm")
492    }
493
494    fn schedule_defrag_selection_packets(&self) {
495        self.evac_set
496            .schedule_defrag_selection_packets(&self.immix_space)
497    }
498
499    /// Generate chunk sweep work packets.
500    fn generate_dead_cycle_sweep_tasks(&self) -> Vec<Box<dyn GCWork<VM>>> {
501        self.immix_space.chunk_map.generate_tasks_batched(
502            self.immix_space.scheduler().num_workers(),
503            |chunks| {
504                Box::new(SweepDeadCycles::new(
505                    chunks,
506                    LazySweepingJobsCounter::new_decs(),
507                ))
508            },
509        )
510    }
511
512    fn schedule_mature_sweeping(&self, pause: Pause) {
513        if pause == Pause::Full || pause == Pause::FinalMark {
514            self.evac_set
515                .sweep_mature_evac_candidates(&self.immix_space);
516            let disable_lasy_dec_for_current_gc =
517                crate::plan::lxr::disable_lasy_dec_for_current_gc();
518            let dead_cycle_sweep_packets = self.generate_dead_cycle_sweep_tasks();
519            let sweep_los = RCSweepMatureAfterSATBLOS::new(LazySweepingJobsCounter::new_decs());
520            if super::LAZY_DECREMENTS && !disable_lasy_dec_for_current_gc {
521                debug_assert_ne!(pause, Pause::Full);
522                let concurrent_bucket =
523                    &self.immix_space.scheduler().work_buckets[WorkBucketStage::Concurrent];
524                concurrent_bucket.bulk_add_deferred(dead_cycle_sweep_packets);
525                concurrent_bucket.add_deferred(Box::new(sweep_los));
526            } else {
527                self.immix_space.scheduler().work_buckets[WorkBucketStage::STWRCDecsAndSweep]
528                    .bulk_add(dead_cycle_sweep_packets);
529                self.immix_space.scheduler().work_buckets[WorkBucketStage::STWRCDecsAndSweep]
530                    .add(sweep_los);
531            }
532        }
533    }
534
535    /// Generate chunk sweep work packets.
536    fn generate_full_trace_prepare_tasks(&self) -> Vec<Box<dyn GCWork<VM>>> {
537        self.immix_space
538            .chunk_map
539            .generate_tasks_batched(self.immix_space.scheduler().num_workers(), |chunks| {
540                Box::new(PrepareChunksForFullGC { chunks })
541            })
542    }
543
544    fn schedule_rc_block_sweeping_tasks(&self, counter: LazySweepingJobsCounter) {
545        // while let Some(x) = self.last_mutator_recycled_blocks.pop() {
546        //     x.set_state(BlockState::Marked);
547        // }
548        // This may happen either within a pause, or in concurrent.
549        let size = self.possibly_dead_mature_blocks.len();
550        let num_bins = self.immix_space.scheduler().num_workers();
551        let bin_cap = size / num_bins + if size % num_bins == 0 { 0 } else { 1 };
552        let mut bins = (0..num_bins)
553            .map(|_| Vec::with_capacity(bin_cap))
554            .collect::<Vec<Vec<(Block, bool)>>>();
555        'out: for bin in bins.iter_mut() {
556            for _ in 0..bin_cap {
557                if let Some(block) = self.possibly_dead_mature_blocks.pop() {
558                    bin.push(block);
559                } else {
560                    break 'out;
561                }
562            }
563        }
564        let packets = bins
565            .into_iter()
566            .map::<Box<dyn GCWork<VM>>, _>(|blocks| {
567                Box::new(SweepBlocksAfterDecs::new(blocks, counter.clone()))
568            })
569            .collect();
570        self.immix_space.scheduler().work_buckets[WorkBucketStage::Unconstrained].bulk_add(packets);
571    }
572
573    pub(super) fn process_mature_evacuation_remset(&self) {
574        self.mature_evac_remset.flush_all();
575        let packets = self.mature_evac_remset.take_global_packets();
576        self.immix_space.scheduler().work_buckets[WorkBucketStage::RCEvacuateMature]
577            .bulk_add(packets);
578    }
579
580    pub(super) fn add_to_possibly_dead_mature_blocks(&self, block: Block, is_defrag_source: bool) {
581        if block.log() {
582            self.possibly_dead_mature_blocks
583                .push((block, is_defrag_source));
584        }
585    }
586
587    fn next_gc_is_emergency_gc(
588        &self,
589        total_pages: usize,
590        mature_space_pages: usize,
591        emergency_threshold: usize,
592    ) -> bool {
593        let min_avail_pages = usize::min(total_pages * emergency_threshold / 100, 1 << 30 >> 12);
594        total_pages < min_avail_pages + mature_space_pages
595    }
596
597    fn next_gc_is_cycle_gc(&self, mature_space_pages: usize, pause: Pause) -> bool {
598        if pause == Pause::FinalMark || pause == Pause::Full {
599            super::MATURE_LIVE_PREDICTOR.update(mature_space_pages);
600        }
601        let live_mature_pages = super::MATURE_LIVE_PREDICTOR.live_pages() as usize;
602        let garbage = mature_space_pages.saturating_sub(live_mature_pages);
603        let total_pages = self.get_total_pages();
604        !self.concurrent_work_in_progress()
605            && (self.cm_enabled() && garbage * 100 >= super::TRACE_THRESHOLD * total_pages)
606    }
607
608    fn decide_next_gc_may_perform_cycle_collection(&self, pause: Pause) {
609        let (lock, cvar) = &self.decide_cycle_collection;
610        let notify = || {
611            let mut decide_cycle_collection = lock.lock().unwrap();
612            *decide_cycle_collection = true;
613            cvar.notify_one();
614        };
615        // Reset states
616        self.hint_cycle_gc.store(false, Ordering::SeqCst);
617        self.hint_emergency_gc.store(false, Ordering::SeqCst);
618        let emergency_threshold = super::RC_STOP_PERCENT;
619        // Calculate mature space size
620        let total_pages = self.get_total_pages();
621        let mature_space_pages = {
622            let released_los_pages = self.los().num_pages_released_lazy.load(Ordering::SeqCst);
623            HEAP_AFTER_GC
624                .load(Ordering::SeqCst)
625                .saturating_sub(
626                    self.num_clean_blocks_released_lazy.load(Ordering::SeqCst) << Block::LOG_PAGES,
627                )
628                .saturating_sub(released_los_pages)
629        };
630        // Decide next GC kind
631        let hint_cycle_gc = self.next_gc_is_cycle_gc(mature_space_pages, pause);
632        let hint_emergency_gc =
633            self.next_gc_is_emergency_gc(total_pages, mature_space_pages, emergency_threshold);
634        // Update states
635        self.hint_cycle_gc.store(hint_cycle_gc, Ordering::SeqCst);
636        self.hint_emergency_gc
637            .store(hint_emergency_gc, Ordering::SeqCst);
638        // Eager mark-table zeroing
639        if !cfg!(feature = "sanity") && hint_cycle_gc {
640            self.schedule_mark_table_zeroing_tasks(None);
641        }
642        notify();
643    }
644
645    fn schedule_mark_table_zeroing_tasks(&self, pause: Option<Pause>) {
646        if let Some(pause) = pause {
647            assert!(pause == Pause::InitialMark || pause == Pause::Full);
648            if self.zeroing_packets_scheduled.load(Ordering::SeqCst) {
649                return;
650            }
651        }
652        let work_packets = self
653            .immix_space
654            .chunk_map
655            .generate_tasks_batched(self.immix_space.scheduler().num_workers(), |chunks| {
656                Box::new(ConcurrentChunkMetadataZeroing { chunks })
657            });
658        self.immix_space.scheduler().work_buckets[WorkBucketStage::Unconstrained]
659            .bulk_add(work_packets);
660        self.zeroing_packets_scheduled.store(true, Ordering::SeqCst);
661    }
662
663    fn wait_for_decide_cycle_collection(&self) {
664        let (lock, cvar) = &self.decide_cycle_collection;
665        let mut decide_cycle_collection = lock.lock().unwrap();
666        while !*decide_cycle_collection {
667            decide_cycle_collection = cvar.wait(decide_cycle_collection).unwrap();
668        }
669        *decide_cycle_collection = false;
670    }
671
672    fn select_collection_kind(&self) -> Pause {
673        self.wait_for_decide_cycle_collection();
674
675        let emergency = self.base().global_state.is_emergency_collection();
676        let user_triggered = self.base().global_state.is_user_triggered_collection();
677        let cm_in_progress = self.concurrent_work_in_progress();
678        let cm_packets_drained = super::concurrent_marking_packets_drained();
679        let hint_cycle_gc = self.hint_cycle_gc.load(Ordering::SeqCst);
680        let hint_emergency_gc = self.hint_emergency_gc.load(Ordering::SeqCst);
681        // If CM is finished, do a final mark pause
682        if cm_in_progress && cm_packets_drained {
683            return Pause::FinalMark;
684        }
685
686        // Either final mark pause or full pause for emergency GC
687        if emergency || user_triggered || hint_emergency_gc {
688            return if cm_in_progress {
689                Pause::FinalMark
690            } else {
691                Pause::Full
692            };
693        }
694
695        // Should trigger CM?
696        if hint_cycle_gc && !cm_in_progress {
697            if self.cm_enabled() {
698                Pause::InitialMark
699            } else {
700                Pause::Full
701            }
702        } else {
703            Pause::RefCount
704        }
705    }
706
707    fn disable_unnecessary_buckets(&'static self, scheduler: &GCWorkScheduler<VM>, pause: Pause) {
708        // Set conditional buckets
709        #[cfg(feature = "non_moving_root")]
710        scheduler.work_buckets[WorkBucketStage::RCProcessIncsNonMoving].set_enabled(true);
711        scheduler.work_buckets[WorkBucketStage::RCProcessIncs].set_enabled(true);
712        scheduler.work_buckets[WorkBucketStage::Prepare].set_enabled(pause != Pause::RefCount);
713        let final_mark_or_full = pause == Pause::FinalMark || pause == Pause::Full;
714        // Marks roots reported as objects, before `Closure` can evacuate anything.
715        #[cfg(feature = "non_moving_root")]
716        scheduler.work_buckets[WorkBucketStage::PinningRootsTrace].set_enabled(final_mark_or_full);
717        scheduler.work_buckets[WorkBucketStage::Closure].set_enabled(final_mark_or_full);
718        scheduler.work_buckets[WorkBucketStage::WeakRefClosure].set_enabled(final_mark_or_full);
719        scheduler.work_buckets[WorkBucketStage::FinalRefClosure].set_enabled(final_mark_or_full);
720        scheduler.work_buckets[WorkBucketStage::PhantomRefClosure].set_enabled(final_mark_or_full);
721        scheduler.work_buckets[WorkBucketStage::STWRCDecsAndSweep]
722            .set_enabled(!(super::LAZY_DECREMENTS && pause != Pause::Full));
723        // Always enabled
724        scheduler.work_buckets[WorkBucketStage::Concurrent].set_enabled(true);
725        scheduler.work_buckets[WorkBucketStage::ConcurrentResumable].set_enabled(true);
726        // Always disabled
727        // LXR never routes work here: it has no transitively pinning closure. Transitive
728        // pinning roots, where accepted at all, take the ordinary node-root path instead.
729        #[cfg(feature = "non_moving_root")]
730        scheduler.work_buckets[WorkBucketStage::TPinningClosure].set_enabled(false);
731        scheduler.work_buckets[WorkBucketStage::VMRefClosure].set_enabled(false);
732        scheduler.work_buckets[WorkBucketStage::VMRefForwarding].set_enabled(false);
733        scheduler.work_buckets[WorkBucketStage::SoftRefClosure].set_enabled(false);
734        scheduler.work_buckets[WorkBucketStage::CalculateForwarding].set_enabled(false);
735        scheduler.work_buckets[WorkBucketStage::SecondRoots].set_enabled(false);
736        scheduler.work_buckets[WorkBucketStage::RefForwarding].set_enabled(false);
737        scheduler.work_buckets[WorkBucketStage::FinalizableForwarding].set_enabled(false);
738        scheduler.work_buckets[WorkBucketStage::Compact].set_enabled(false);
739    }
740
741    fn schedule_rc_collection(&'static self, scheduler: &GCWorkScheduler<VM>) {
742        log::info!("Scheduling RC collection...");
743        self.disable_unnecessary_buckets(scheduler, Pause::RefCount);
744        // Before start yielding, wrap all the roots from the previous GC with work-packets.
745        self.process_prev_roots(scheduler);
746        // Stop & scan mutators (mutator scanning can happen before STW)
747        scheduler.work_buckets[WorkBucketStage::Unconstrained]
748            .add(StopMutators::<LXRGCWorkContext<VM>>::new_with_flush());
749        // Prepare global/collectors/mutators
750        scheduler.work_buckets[WorkBucketStage::RCProcessIncs].add(FastRCPrepare);
751        // Release global/collectors/mutators
752        scheduler.work_buckets[WorkBucketStage::Release]
753            .add(Release::<LXRGCWorkContext<VM>>::new(self));
754    }
755
756    fn schedule_concurrent_marking_initial_pause(&'static self, scheduler: &GCWorkScheduler<VM>) {
757        log::info!("Scheduling concurrent marking initial pause...");
758        self.disable_unnecessary_buckets(scheduler, Pause::InitialMark);
759        self.process_prev_roots(scheduler);
760        scheduler.work_buckets[WorkBucketStage::Unconstrained]
761            .add(StopMutators::<LXRGCWorkContext<VM>>::new_with_flush());
762        scheduler.work_buckets[WorkBucketStage::Prepare]
763            .add(Prepare::<LXRGCWorkContext<VM>>::new(self));
764        scheduler.work_buckets[WorkBucketStage::Release]
765            .add(Release::<LXRGCWorkContext<VM>>::new(self));
766    }
767
768    fn schedule_concurrent_marking_final_pause(&'static self, scheduler: &GCWorkScheduler<VM>) {
769        log::info!("Scheduling concurrent marking final pause...");
770        self.disable_unnecessary_buckets(scheduler, Pause::FinalMark);
771        self.process_prev_roots(scheduler);
772        scheduler.work_buckets[WorkBucketStage::Unconstrained]
773            .add(StopMutators::<LXRGCWorkContext<VM>>::new_with_flush());
774
775        scheduler.work_buckets[WorkBucketStage::Prepare]
776            .add(Prepare::<LXRGCWorkContext<VM>>::new(self));
777        scheduler.work_buckets[WorkBucketStage::Release]
778            .add(Release::<LXRGCWorkContext<VM>>::new(self));
779    }
780
781    fn schedule_emergency_full_heap_collection(&'static self, scheduler: &GCWorkScheduler<VM>) {
782        log::info!("Scheduling emergency full-heap collection...");
783        super::DISABLE_LASY_DEC_FOR_CURRENT_GC.store(true, Ordering::SeqCst);
784        self.disable_unnecessary_buckets(scheduler, Pause::Full);
785        // Before start yielding, wrap all the roots from the previous GC with work-packets.
786        self.process_prev_roots(scheduler);
787        // Stop & scan mutators (mutator scanning can happen before STW)
788        scheduler.work_buckets[WorkBucketStage::Unconstrained]
789            .add(StopMutators::<LXRGCWorkContext<VM>>::new_with_flush());
790        // Prepare global/collectors/mutators
791        scheduler.work_buckets[WorkBucketStage::Prepare]
792            .add(Prepare::<LXRGCWorkContext<VM>>::new(self));
793        // Release global/collectors/mutators
794        scheduler.work_buckets[WorkBucketStage::Release]
795            .add(Release::<LXRGCWorkContext<VM>>::new(self));
796    }
797
798    fn process_prev_roots(&self, scheduler: &GCWorkScheduler<VM>) {
799        let prev_roots = self.prev_roots.read().unwrap();
800        let mut work_packets: Vec<Box<dyn GCWork<VM>>> = Vec::with_capacity(prev_roots.len());
801        while let Some(decs) = prev_roots.pop() {
802            work_packets.push(Box::new(ProcessDecs::new(
803                decs,
804                LazySweepingJobsCounter::new_decs(),
805            )))
806        }
807        if work_packets.is_empty() {
808            work_packets.push(Box::new(ProcessDecs::new(
809                vec![],
810                LazySweepingJobsCounter::new_decs(),
811            )));
812        }
813        if super::LAZY_DECREMENTS {
814            scheduler.work_buckets[WorkBucketStage::Concurrent].bulk_add_deferred(work_packets);
815        } else {
816            scheduler.work_buckets[WorkBucketStage::STWRCDecsAndSweep].bulk_add(work_packets);
817        }
818    }
819
820    pub fn current_pause(&self) -> Option<Pause> {
821        self.current_pause.load(Ordering::SeqCst)
822    }
823
824    pub fn previous_pause(&self) -> Option<Pause> {
825        self.previous_pause.load(Ordering::SeqCst)
826    }
827
828    pub fn in_defrag(&self, o: ObjectReference) -> bool {
829        self.immix_space.in_space(o) && Block::in_defrag_block(o)
830    }
831
832    pub fn address_in_defrag(&self, a: Address) -> bool {
833        self.immix_space.address_in_space(a) && Block::address_in_defrag_block(a)
834    }
835
836    pub fn mark(&self, o: ObjectReference) -> bool {
837        if self.immix_space.in_space(o) {
838            self.immix_space.attempt_mark(o)
839        } else if self.common.los.in_space(o) {
840            self.common.los.attempt_mark(o)
841        } else {
842            // TODO: We need to properly handle this case.
843            // This is a temporary solution for Julia -- the only other spaces it uses are immortal space and vm space, where objects won't die.
844            debug_assert!(o.is_live());
845            false
846        }
847    }
848
849    pub fn is_marked(&self, o: ObjectReference) -> bool {
850        if self.immix_space.in_space(o) {
851            self.immix_space.is_marked(o)
852        } else if self.common.los.in_space(o) {
853            self.common.los.is_marked(o)
854        } else {
855            // TODO: We need to properly handle this case.
856            // This is a temporary solution for Julia -- the only other spaces it uses are immortal space and vm space, where objects won't die.
857            debug_assert!(o.is_live());
858            true
859        }
860    }
861
862    pub const fn los(&self) -> &LargeObjectSpace<VM> {
863        &self.common.los
864    }
865
866    /// Whether `o` lives in a space LXR reference-counts (immix space or LOS). Objects
867    /// elsewhere (e.g. Julia's sysimage in the immortal/VM space) carry no reference count.
868    pub fn is_rc_object(&self, o: ObjectReference) -> bool {
869        self.immix_space.in_space(o) || self.common.los.in_space(o)
870    }
871
872    fn on_lazy_decs_finished(&self, c: LazySweepingJobsCounter) {
873        self.schedule_rc_block_sweeping_tasks(c);
874    }
875
876    fn on_lazy_sweeping_finished(&self) {
877        self.immix_space.flush_page_resource();
878        // Update counters
879        if !super::LAZY_DECREMENTS {
880            HEAP_AFTER_GC.store(self.get_used_pages(), Ordering::SeqCst);
881        }
882        let pause = match self.current_pause() {
883            Some(p) => p,
884            None => self.previous_pause().unwrap(),
885        };
886        self.decide_next_gc_may_perform_cycle_collection(pause);
887    }
888
889    fn gc_init(&mut self) {
890        self.immix_space.rc_enabled = true;
891        self.common.los.rc_enabled = true;
892        unsafe {
893            let me: &'static Self = &*(self as *const Self);
894            me.block_allocation.init(&me.immix_space, me);
895            me.immix_space.install_hooks(&me.block_allocation);
896        }
897        let mut lazy_sweeping_jobs = LAZY_SWEEPING_JOBS.write();
898        lazy_sweeping_jobs.swap();
899        let lxr_ptr = self as *const Self as usize;
900        lazy_sweeping_jobs.end_of_decs = Some(Box::new(move |c| {
901            let lxr = unsafe { &*(lxr_ptr as *const Self) };
902            lxr.on_lazy_decs_finished(c);
903        }));
904        lazy_sweeping_jobs.end_of_lazy = Some(Box::new(move || {
905            let lxr = unsafe { &*(lxr_ptr as *const Self) };
906            lxr.on_lazy_sweeping_finished();
907        }));
908    }
909
910    fn set_concurrent_marking_state(&self, active: bool) {
911        self.in_concurrent_marking.store(active, Ordering::SeqCst);
912        self.common
913            .los
914            .bump_page_reuse_count
915            .store(active, Ordering::SeqCst);
916    }
917}
mmtk/plan/lxr/global.rs

mmtk/plan/lxr/
global.rs
