mmtk/plan/lxr/
mature_evac.rs

1use std::sync::Mutex;
2use std::{cell::UnsafeCell, marker::PhantomData};
3
4use crate::plan::concurrent::Pause;
5use crate::plan::global::Plan;
6use crate::plan::lxr::gc_work::mature_evac::SelectDefragBlocks;
7use crate::plan::lxr::gc_work::mature_evac::SELECT_DEFRAG_BLOCK_JOB_COUNTER;
8use crate::policy::immix::block::{Block, BlockState};
9use crate::policy::immix::line::Line;
10use crate::policy::immix::ImmixSpace;
11use crate::policy::space::Space;
12use crate::scheduler::WorkBucketStage;
13use crate::util::linear_scan::Region;
14use crate::util::metadata::side_metadata::spec_defs::{IX_LINE_REUSE_COUNT, LOS_PAGE_REUSE_COUNT};
15use crate::util::ObjectReference;
16use crate::{
17    plan::lxr::LXR,
18    scheduler::GCWork,
19    vm::{slot::Slot, VMBinding},
20};
21
22use super::gc_work::mature_evac::EvacuateMatureObjects;
23use crate::util::constants::LOG_BYTES_IN_PAGE;
24use atomic::Ordering;
25use crossbeam::queue::SegQueue;
26use std::sync::atomic::AtomicUsize;
27
28#[repr(C)]
29pub struct RemSetEntry<VM: VMBinding>(VM::VMSlot, u8);
30
31impl<VM: VMBinding> RemSetEntry<VM> {
32    fn encode(slot: VM::VMSlot, ix: bool) -> Self {
33        let reuse = if ix {
34            IX_LINE_REUSE_COUNT.load_atomic::<u8>(slot.to_address(), Ordering::SeqCst)
35        } else {
36            LOS_PAGE_REUSE_COUNT.load_atomic::<u8>(slot.to_address(), Ordering::SeqCst)
37        };
38        Self(slot, reuse)
39    }
40
41    pub fn decode(&self) -> (VM::VMSlot, u8) {
42        (self.0, self.1)
43    }
44}
45
46pub struct MatureEvecRemSet<VM: VMBinding> {
47    pub gc_buffers: Vec<UnsafeCell<Vec<RemSetEntry<VM>>>>,
48    pub global_packets: Mutex<Vec<Box<dyn GCWork<VM>>>>,
49    local_packets: Vec<UnsafeCell<Vec<Box<dyn GCWork<VM>>>>>,
50    _p: PhantomData<VM>,
51    size: AtomicUsize,
52}
53
54unsafe impl<VM: VMBinding> Send for MatureEvecRemSet<VM> {}
55unsafe impl<VM: VMBinding> Sync for MatureEvecRemSet<VM> {}
56
57impl<VM: VMBinding> MatureEvecRemSet<VM> {
58    pub fn new(workers: usize) -> Self {
59        let mut rs = Self {
60            gc_buffers: vec![],
61            global_packets: Mutex::new(vec![]),
62            local_packets: vec![],
63            _p: PhantomData,
64            size: AtomicUsize::new(0),
65        };
66        rs.gc_buffers
67            .resize_with(workers, || UnsafeCell::new(vec![]));
68        rs.local_packets
69            .resize_with(workers, || UnsafeCell::new(vec![]));
70        rs
71    }
72
73    #[allow(clippy::mut_from_ref)]
74    fn gc_buffer(&self, id: usize) -> &mut Vec<RemSetEntry<VM>> {
75        unsafe { &mut *self.gc_buffers[id].get() }
76    }
77
78    pub fn flush_all(&self) {
79        let mut mature_evac_remsets = self.global_packets.lock().unwrap();
80        self.size.store(0, Ordering::SeqCst);
81        for id in 0..self.gc_buffers.len() {
82            if !self.gc_buffer(id).is_empty() {
83                let remset = std::mem::take(self.gc_buffer(id));
84                mature_evac_remsets.push(Box::new(EvacuateMatureObjects::new(remset)));
85            }
86        }
87        for id in 0..self.local_packets.len() {
88            let buf = unsafe { &mut *self.local_packets[id].get() };
89            if !buf.is_empty() {
90                let packets = std::mem::take(buf);
91                for p in packets {
92                    mature_evac_remsets.push(p);
93                }
94            }
95        }
96    }
97
98    pub fn take_global_packets(&self) -> Vec<Box<dyn GCWork<VM>>> {
99        let mut mature_evac_remsets = self.global_packets.lock().unwrap();
100        std::mem::take(&mut *mature_evac_remsets)
101    }
102
103    #[cold]
104    fn flush(&self, id: usize) {
105        if !self.gc_buffer(id).is_empty() {
106            let remset = std::mem::take(self.gc_buffer(id));
107            self.size.fetch_add(remset.len(), Ordering::SeqCst);
108            let w = EvacuateMatureObjects::new(remset);
109            let packet_buffer = unsafe { &mut *self.local_packets[id].get() };
110            packet_buffer.push(Box::new(w));
111        }
112    }
113
114    pub fn record(&self, s: VM::VMSlot, _o: ObjectReference, lxr: &LXR<VM>) {
115        let id = crate::scheduler::current_worker_ordinal().unwrap();
116        let ix = lxr.immix_space.address_in_space(s.to_address());
117        self.gc_buffer(id).push(RemSetEntry::<VM>::encode(s, ix));
118        if self.gc_buffer(id).len() >= EvacuateMatureObjects::<VM>::CAPACITY {
119            self.flush(id)
120        }
121    }
122}
123
124#[derive(Default)]
125pub struct MatureEvacuationSet {
126    pub fragmented_blocks: SegQueue<Vec<(Block, usize)>>,
127    pub fragmented_blocks_size: AtomicUsize,
128    pub blocks_in_fragmented_chunks: SegQueue<Vec<(Block, usize)>>,
129    pub blocks_in_fragmented_chunks_size: AtomicUsize,
130    pub defrag_blocks: Mutex<Vec<Block>>,
131    pub num_defrag_blocks: AtomicUsize,
132}
133
134impl MatureEvacuationSet {
135    /// Release all the mature defrag source blocks
136    pub fn sweep_mature_evac_candidates<VM: VMBinding>(&self, space: &ImmixSpace<VM>) {
137        let mut defrag_blocks: Vec<Block> =
138            std::mem::take(&mut *self.defrag_blocks.lock().unwrap());
139        if defrag_blocks.is_empty() {
140            return;
141        }
142        while let Some(block) = defrag_blocks.pop() {
143            if !block.is_defrag_source() || block.get_state() == BlockState::Unallocated {
144                // This block has been eagerly released (probably be reused again). Skip it.
145                continue;
146            }
147            block.clear_rc_table();
148            block.clear_striddle_table();
149            block.rc_sweep_mature::<VM>(space, true);
150            assert!(!block.is_defrag_source());
151        }
152    }
153
154    pub fn schedule_defrag_selection_packets<VM: VMBinding>(&self, space: &ImmixSpace<VM>) {
155        let tasks =
156            space
157                .chunk_map
158                .generate_tasks_batched(space.scheduler().num_workers(), |chunks| {
159                    Box::new(SelectDefragBlocks {
160                        chunks,
161                        defrag_threshold: 1,
162                    })
163                });
164        self.fragmented_blocks_size.store(0, Ordering::SeqCst);
165        SELECT_DEFRAG_BLOCK_JOB_COUNTER.store(tasks.len(), Ordering::SeqCst);
166        space.scheduler().work_buckets[WorkBucketStage::Unconstrained].bulk_add(tasks);
167    }
168
169    pub fn skip_block(b: Block) -> bool {
170        let s = b.get_state();
171        b.is_defrag_source() || s == BlockState::Unallocated || s == BlockState::Nursery
172    }
173
174    fn select_fragmented_blocks(
175        &self,
176        selected_blocks: &mut Vec<Block>,
177        copy_bytes: &mut usize,
178        max_copy_bytes: usize,
179    ) {
180        let mut blocks = Vec::with_capacity(self.fragmented_blocks_size.load(Ordering::SeqCst));
181        while let Some(mut x) = self.fragmented_blocks.pop() {
182            blocks.append(&mut x);
183        }
184        blocks.sort_by_key(|x| x.1);
185        while let Some((block, _dead_bytes)) = blocks.pop() {
186            if Self::skip_block(block) {
187                continue;
188            }
189            block.set_as_defrag_source(true);
190            selected_blocks.push(block);
191            *copy_bytes += (Block::BYTES - (block.calc_dead_lines() << Line::LOG_BYTES)) >> 1;
192            if *copy_bytes >= max_copy_bytes {
193                break;
194            }
195        }
196    }
197
198    #[allow(clippy::assertions_on_constants)]
199    pub fn select_mature_evacuation_candidates<VM: VMBinding>(&self, lxr: &LXR<VM>) {
200        debug_assert!(crate::plan::lxr::MATURE_EVACUATION);
201        if lxr.current_pause().unwrap() == Pause::Full {
202            // Make sure LOS sweeping finishes before evac selection begin
203            // FIXME: This can be done in parallel with SelectDefragBlocksInChunk packets
204            let los = lxr.common().get_los();
205            los.release_rc_nursery_objects();
206        }
207        // Select mature defrag blocks
208        let available_clean_pages_for_defrag = if lxr.current_pause().unwrap() == Pause::Full {
209            lxr.get_total_pages()
210                .saturating_sub(lxr.get_used_pages())
211                .max(lxr.immix_space.defrag_headroom_pages())
212        } else {
213            lxr.immix_space.defrag_headroom_pages()
214        };
215        let max_copy_bytes = available_clean_pages_for_defrag << LOG_BYTES_IN_PAGE;
216        let mut copy_bytes = 0usize;
217        let mut selected_blocks = vec![];
218        self.select_fragmented_blocks(&mut selected_blocks, &mut copy_bytes, max_copy_bytes);
219        self.num_defrag_blocks
220            .store(selected_blocks.len(), Ordering::SeqCst);
221        let mut defrag_blocks = self.defrag_blocks.lock().unwrap();
222        *defrag_blocks = selected_blocks;
223        // cleanup
224        assert!(self.fragmented_blocks.is_empty());
225        assert!(self.blocks_in_fragmented_chunks.is_empty());
226        self.fragmented_blocks_size.store(0, Ordering::SeqCst);
227        self.blocks_in_fragmented_chunks_size
228            .store(0, Ordering::SeqCst);
229    }
230}
mmtk/plan/lxr/mature_evac.rs

mmtk/plan/lxr/
mature_evac.rs
