mmtk/plan/lxr/
block_allocation.rs

1use super::gc_work::nursery_sweeping::{RCLazySweepNurseryBlocks, RCSTWSweepNurseryBlocks};
2use super::LXR;
3use crate::plan::concurrent::global::ConcurrentPlan;
4use crate::plan::concurrent::Pause;
5use crate::policy::immix::block::{Block, BlockState};
6use crate::policy::immix::{ImmixHooks, ImmixSpace};
7use crate::scheduler::{GCWork, GCWorkScheduler, WorkBucketStage};
8use crate::util::constants::LOG_BYTES_IN_PAGE;
9use crate::util::linear_scan::Region;
10use crate::vm::VMBinding;
11use atomic::{Atomic, Ordering};
12use std::cell::UnsafeCell;
13use std::sync::atomic::AtomicUsize;
14use std::sync::RwLock;
15
16struct BlockCache {
17    cursor: AtomicUsize,
18    buffer: RwLock<Vec<Atomic<Block>>>,
19}
20
21impl BlockCache {
22    fn new() -> Self {
23        Self {
24            cursor: AtomicUsize::new(0),
25            buffer: RwLock::new((0..32768).map(|_| Atomic::new(Block::ZERO)).collect()),
26        }
27    }
28
29    fn len(&self) -> usize {
30        self.cursor.load(Ordering::SeqCst)
31    }
32
33    fn push(&self, block: Block) {
34        let i = self.cursor.fetch_add(1, Ordering::SeqCst);
35        let buffer = self.buffer.read().unwrap();
36        if i < buffer.len() {
37            buffer[i].store(block, Ordering::SeqCst);
38        } else {
39            std::mem::drop(buffer);
40            let mut buffer = self.buffer.write().unwrap();
41            if i >= buffer.len() {
42                buffer.resize_with(i << 1, || Atomic::new(Block::ZERO));
43            }
44            buffer[i].store(block, Ordering::Relaxed);
45        }
46    }
47
48    fn visit_slice(&self, f: impl Fn(&[Atomic<Block>])) {
49        let count = self.cursor.load(Ordering::SeqCst);
50        let blocks = self.buffer.read().unwrap();
51        f(&blocks[0..count])
52    }
53
54    fn reset(&self) {
55        self.cursor.store(0, Ordering::SeqCst);
56    }
57}
58
59pub struct BlockAllocation<VM: VMBinding> {
60    space: UnsafeCell<*const ImmixSpace<VM>>,
61    lxr: UnsafeCell<*const LXR<VM>>,
62    nursery_blocks: BlockCache,
63    reused_blocks: BlockCache,
64}
65
66unsafe impl<VM: VMBinding> Sync for BlockAllocation<VM> {}
67unsafe impl<VM: VMBinding> Send for BlockAllocation<VM> {}
68
69impl<VM: VMBinding> BlockAllocation<VM> {
70    pub fn new() -> Self {
71        Self {
72            space: UnsafeCell::new(std::ptr::null()),
73            lxr: UnsafeCell::new(std::ptr::null()),
74            nursery_blocks: BlockCache::new(),
75            reused_blocks: BlockCache::new(),
76        }
77    }
78
79    pub fn init(&self, space: &ImmixSpace<VM>, lxr: &'static LXR<VM>) {
80        unsafe {
81            *self.space.get() = space as *const ImmixSpace<VM>;
82            *self.lxr.get() = lxr as *const LXR<VM>;
83        }
84    }
85
86    fn space(&self) -> &'static ImmixSpace<VM> {
87        unsafe { &**self.space.get() }
88    }
89
90    fn lxr(&self) -> &'static LXR<VM> {
91        unsafe { &**self.lxr.get() }
92    }
93
94    pub fn clean_nursery_mb(&self) -> usize {
95        self.nursery_blocks.len() << Block::LOG_BYTES >> 20
96    }
97
98    pub fn total_young_allocation_in_bytes(&self) -> usize {
99        (self.nursery_blocks.len() << Block::LOG_BYTES)
100            + (self.space().get_mutator_recycled_lines_in_pages() << LOG_BYTES_IN_PAGE)
101    }
102
103    pub fn reset_block_mark_for_mutator_reused_blocks(&self, _pause: Pause) {
104        // SATB sweep has problem scanning mutator recycled blocks.
105        // Remaing the block state as "reusing" and reset them here.
106        self.reused_blocks.visit_slice(|blocks| {
107            for b in blocks {
108                let b = b.load(Ordering::Relaxed);
109                b.set_state(BlockState::Marked);
110            }
111        });
112    }
113
114    pub fn sweep_mutator_reused_blocks(&self, pause: Pause) {
115        if pause == Pause::Full || pause == Pause::FinalMark {
116            self.reused_blocks.reset();
117            return;
118        }
119        self.reused_blocks.visit_slice(|blocks| {
120            for b in blocks {
121                let block = b.load(Ordering::Relaxed);
122                self.lxr().add_to_possibly_dead_mature_blocks(block, false);
123            }
124        });
125        self.reused_blocks.reset();
126    }
127
128    /// Reset allocated_block_buffer and free nursery blocks.
129    pub fn sweep_nursery_blocks(&self, scheduler: &GCWorkScheduler<VM>, pause: Pause) {
130        const PARALLEL_STW_SWEEPING: bool = false;
131        let max_stw_sweep_blocks: usize = usize::MAX;
132        let space = self.space();
133        self.nursery_blocks.visit_slice(|blocks| {
134            if PARALLEL_STW_SWEEPING {
135                return self.parallel_sweep_all_nursery_blocks(scheduler, blocks);
136            }
137            let total_nursery_blocks = blocks.len();
138            let stw_limit = if pause == Pause::Full {
139                total_nursery_blocks
140            } else {
141                usize::min(total_nursery_blocks, max_stw_sweep_blocks)
142            };
143            for b in &blocks[0..stw_limit] {
144                let block = b.load(Ordering::Relaxed);
145                debug_assert_ne!(block.get_state(), BlockState::Unallocated);
146                block.rc_sweep_nursery(space);
147            }
148            if total_nursery_blocks > stw_limit {
149                let packets = blocks[stw_limit..total_nursery_blocks]
150                    .chunks(1024)
151                    .map(|c| {
152                        let blocks: Vec<Block> =
153                            c.iter().map(|x| x.load(Ordering::Relaxed)).collect();
154                        Box::new(RCLazySweepNurseryBlocks::new(blocks)) as Box<dyn GCWork<VM>>
155                    })
156                    .collect();
157                scheduler.work_buckets[WorkBucketStage::Concurrent].bulk_add_deferred(packets);
158            }
159        });
160        self.nursery_blocks.reset();
161    }
162
163    fn parallel_sweep_all_nursery_blocks(
164        &self,
165        scheduler: &GCWorkScheduler<VM>,
166        blocks: &[Atomic<Block>],
167    ) {
168        let total_nursery_blocks = blocks.len();
169        let packets = blocks[..total_nursery_blocks]
170            .chunks(1024)
171            .map(|c| {
172                let blocks: Vec<Block> = c.iter().map(|x| x.load(Ordering::Relaxed)).collect();
173                Box::new(RCSTWSweepNurseryBlocks::new(blocks)) as Box<dyn GCWork<VM>>
174            })
175            .collect();
176        scheduler.work_buckets[WorkBucketStage::Unconstrained].bulk_add(packets);
177    }
178}
179
180impl<VM: VMBinding> ImmixHooks<VM> for BlockAllocation<VM> {
181    fn on_clean_block_acquired(&self, block: Block, copy: bool) {
182        if !copy {
183            self.nursery_blocks.push(block);
184        }
185        if copy {
186            block.initialize_field_unlog_table_as_unlogged::<VM>();
187        }
188        if self.cm_in_progress_or_final_mark() {
189            block.initialize_mark_table_as_marked::<VM>();
190        } else {
191            block.clear_mark_table::<VM>();
192        }
193    }
194
195    fn on_reusable_block_acquired(&self, block: Block, copy: bool) {
196        if !copy {
197            self.reused_blocks.push(block);
198        }
199    }
200
201    fn cm_in_progress_or_final_mark(&self) -> bool {
202        let lxr = self.lxr();
203        lxr.concurrent_work_in_progress() || lxr.current_pause() == Some(Pause::FinalMark)
204    }
205}
mmtk/plan/lxr/block_allocation.rs

mmtk/plan/lxr/
block_allocation.rs
