mmtk/plan/lxr/
block_allocation.rs

1use super::gc_work::nursery_sweeping::{RCLazySweepNurseryBlocks, RCSTWSweepNurseryBlocks};
2use super::LXR;
3use crate::plan::concurrent::global::ConcurrentPlan;
4use crate::plan::concurrent::Pause;
5use crate::plan::global::Plan;
6use crate::policy::immix::block::{Block, BlockState};
7use crate::policy::immix::{ImmixHooks, ImmixSpace};
8use crate::scheduler::{GCWork, GCWorkScheduler, WorkBucketStage};
9use crate::util::constants::LOG_BYTES_IN_PAGE;
10use crate::util::linear_scan::Region;
11use crate::vm::VMBinding;
12use atomic::{Atomic, Ordering};
13use std::cell::UnsafeCell;
14use std::sync::atomic::AtomicUsize;
15use std::sync::RwLock;
16
17struct BlockCache {
18    cursor: AtomicUsize,
19    buffer: RwLock<Vec<Atomic<Block>>>,
20}
21
22impl BlockCache {
23    fn new() -> Self {
24        Self {
25            cursor: AtomicUsize::new(0),
26            buffer: RwLock::new((0..32768).map(|_| Atomic::new(Block::ZERO)).collect()),
27        }
28    }
29
30    fn len(&self) -> usize {
31        self.cursor.load(Ordering::SeqCst)
32    }
33
34    fn push(&self, block: Block) {
35        let i = self.cursor.fetch_add(1, Ordering::SeqCst);
36        let buffer = self.buffer.read().unwrap();
37        if i < buffer.len() {
38            buffer[i].store(block, Ordering::SeqCst);
39        } else {
40            std::mem::drop(buffer);
41            let mut buffer = self.buffer.write().unwrap();
42            if i >= buffer.len() {
43                buffer.resize_with(i << 1, || Atomic::new(Block::ZERO));
44            }
45            buffer[i].store(block, Ordering::Relaxed);
46        }
47    }
48
49    fn visit_slice(&self, f: impl Fn(&[Atomic<Block>])) {
50        let count = self.cursor.load(Ordering::SeqCst);
51        let blocks = self.buffer.read().unwrap();
52        f(&blocks[0..count])
53    }
54
55    fn reset(&self) {
56        self.cursor.store(0, Ordering::SeqCst);
57    }
58}
59
60pub struct BlockAllocation<VM: VMBinding> {
61    space: UnsafeCell<*const ImmixSpace<VM>>,
62    lxr: UnsafeCell<*const LXR<VM>>,
63    nursery_blocks: BlockCache,
64    reused_blocks: BlockCache,
65}
66
67unsafe impl<VM: VMBinding> Sync for BlockAllocation<VM> {}
68unsafe impl<VM: VMBinding> Send for BlockAllocation<VM> {}
69
70impl<VM: VMBinding> BlockAllocation<VM> {
71    pub fn new() -> Self {
72        Self {
73            space: UnsafeCell::new(std::ptr::null()),
74            lxr: UnsafeCell::new(std::ptr::null()),
75            nursery_blocks: BlockCache::new(),
76            reused_blocks: BlockCache::new(),
77        }
78    }
79
80    pub fn init(&self, space: &ImmixSpace<VM>, lxr: &'static LXR<VM>) {
81        unsafe {
82            *self.space.get() = space as *const ImmixSpace<VM>;
83            *self.lxr.get() = lxr as *const LXR<VM>;
84        }
85    }
86
87    fn space(&self) -> &'static ImmixSpace<VM> {
88        unsafe { &**self.space.get() }
89    }
90
91    fn lxr(&self) -> &'static LXR<VM> {
92        unsafe { &**self.lxr.get() }
93    }
94
95    pub fn clean_nursery_mb(&self) -> usize {
96        self.nursery_blocks.len() << Block::LOG_BYTES >> 20
97    }
98
99    pub fn total_young_allocation_in_bytes(&self) -> usize {
100        (self.nursery_blocks.len() << Block::LOG_BYTES)
101            + (self.space().get_mutator_recycled_lines_in_pages() << LOG_BYTES_IN_PAGE)
102    }
103
104    pub fn reset_block_mark_for_mutator_reused_blocks(&self, _pause: Pause) {
105        // SATB sweep has problem scanning mutator recycled blocks.
106        // Remaing the block state as "reusing" and reset them here.
107        self.reused_blocks.visit_slice(|blocks| {
108            for b in blocks {
109                let b = b.load(Ordering::Relaxed);
110                b.set_state(BlockState::Marked);
111            }
112        });
113    }
114
115    pub fn sweep_mutator_reused_blocks(&self, pause: Pause) {
116        if pause == Pause::Full || pause == Pause::FinalMark {
117            self.reused_blocks.reset();
118            return;
119        }
120        self.reused_blocks.visit_slice(|blocks| {
121            for b in blocks {
122                let block = b.load(Ordering::Relaxed);
123                self.lxr().add_to_possibly_dead_mature_blocks(block, false);
124            }
125        });
126        self.reused_blocks.reset();
127    }
128
129    /// Reset allocated_block_buffer and free nursery blocks.
130    pub fn sweep_nursery_blocks(&self, scheduler: &GCWorkScheduler<VM>, pause: Pause) {
131        const PARALLEL_STW_SWEEPING: bool = false;
132        let max_stw_sweep_blocks: usize =
133            *self.lxr().base().options.lxr_max_stw_sweep_nursery_blocks;
134        let space = self.space();
135        self.nursery_blocks.visit_slice(|blocks| {
136            if PARALLEL_STW_SWEEPING {
137                return self.parallel_sweep_all_nursery_blocks(scheduler, blocks);
138            }
139            let total_nursery_blocks = blocks.len();
140            let stw_limit = if pause == Pause::Full {
141                total_nursery_blocks
142            } else {
143                usize::min(total_nursery_blocks, max_stw_sweep_blocks)
144            };
145            for b in &blocks[0..stw_limit] {
146                let block = b.load(Ordering::Relaxed);
147                debug_assert_ne!(block.get_state(), BlockState::Unallocated);
148                block.rc_sweep_nursery(space);
149            }
150            if total_nursery_blocks > stw_limit {
151                let packets = blocks[stw_limit..total_nursery_blocks]
152                    .chunks(1024)
153                    .map(|c| {
154                        let blocks: Vec<Block> =
155                            c.iter().map(|x| x.load(Ordering::Relaxed)).collect();
156                        Box::new(RCLazySweepNurseryBlocks::new(blocks)) as Box<dyn GCWork<VM>>
157                    })
158                    .collect();
159                scheduler.work_buckets[WorkBucketStage::Concurrent].bulk_add_deferred(packets);
160            }
161        });
162        self.nursery_blocks.reset();
163    }
164
165    fn parallel_sweep_all_nursery_blocks(
166        &self,
167        scheduler: &GCWorkScheduler<VM>,
168        blocks: &[Atomic<Block>],
169    ) {
170        let total_nursery_blocks = blocks.len();
171        let packets = blocks[..total_nursery_blocks]
172            .chunks(1024)
173            .map(|c| {
174                let blocks: Vec<Block> = c.iter().map(|x| x.load(Ordering::Relaxed)).collect();
175                Box::new(RCSTWSweepNurseryBlocks::new(blocks)) as Box<dyn GCWork<VM>>
176            })
177            .collect();
178        scheduler.work_buckets[WorkBucketStage::Unconstrained].bulk_add(packets);
179    }
180}
181
182impl<VM: VMBinding> ImmixHooks<VM> for BlockAllocation<VM> {
183    fn on_clean_block_acquired(&self, block: Block, copy: bool) {
184        if !copy {
185            self.nursery_blocks.push(block);
186        }
187        if copy {
188            block.initialize_field_unlog_table_as_unlogged::<VM>();
189        }
190        if self.cm_in_progress_or_final_mark() {
191            block.initialize_mark_table_as_marked::<VM>();
192        } else {
193            block.clear_mark_table::<VM>();
194        }
195    }
196
197    fn on_reusable_block_acquired(&self, block: Block, copy: bool) {
198        if !copy {
199            self.reused_blocks.push(block);
200        }
201    }
202
203    fn cm_in_progress_or_final_mark(&self) -> bool {
204        let lxr = self.lxr();
205        lxr.concurrent_work_in_progress() || lxr.current_pause() == Some(Pause::FinalMark)
206    }
207}
mmtk/plan/lxr/block_allocation.rs

mmtk/plan/lxr/
block_allocation.rs
