mmtk/policy/
vmspace.rs

1use crate::mmtk::SFT_MAP;
2use crate::plan::tracing::OptionObjectQueue;
3use crate::plan::ObjectQueue;
4use crate::policy::sft::GCWorkerMutRef;
5use crate::policy::sft::SFT;
6use crate::policy::space::{CommonSpace, Space};
7use crate::util::address::Address;
8use crate::util::alloc::allocator::AllocationOptions;
9use crate::util::constants::BYTES_IN_PAGE;
10use crate::util::heap::externalpageresource::{ExternalPageResource, ExternalPages};
11use crate::util::heap::layout::vm_layout::BYTES_IN_CHUNK;
12use crate::util::heap::PageResource;
13use crate::util::metadata::mark_bit::MarkState;
14#[cfg(feature = "set_unlog_bits_vm_space")]
15use crate::util::metadata::MetadataSpec;
16use crate::util::object_enum::ObjectEnumerator;
17use crate::util::opaque_pointer::*;
18use crate::util::ObjectReference;
19use crate::vm::{ObjectModel, VMBinding};
20
21use std::sync::atomic::Ordering;
22
23/// A special space for VM/Runtime managed memory. The implementation is similar to [`crate::policy::immortalspace::ImmortalSpace`],
24/// except that VM space does not allocate. Instead, the runtime can add regions that are externally managed
25/// and mmapped to the space, and allow objects in those regions to be traced in the same way
26/// as other MMTk objects allocated by MMTk.
27pub struct VMSpace<VM: VMBinding> {
28    mark_state: MarkState,
29    common: CommonSpace<VM>,
30    pr: ExternalPageResource<VM>,
31}
32
33impl<VM: VMBinding> SFT for VMSpace<VM> {
34    fn name(&self) -> &'static str {
35        self.common.name
36    }
37    fn is_live(&self, _object: ObjectReference) -> bool {
38        true
39    }
40    fn is_reachable(&self, object: ObjectReference) -> bool {
41        self.mark_state.is_marked::<VM>(object)
42    }
43    #[cfg(feature = "object_pinning")]
44    fn pin_object(&self, _object: ObjectReference) -> bool {
45        false
46    }
47    #[cfg(feature = "object_pinning")]
48    fn unpin_object(&self, _object: ObjectReference) -> bool {
49        false
50    }
51    #[cfg(feature = "object_pinning")]
52    fn is_object_pinned(&self, _object: ObjectReference) -> bool {
53        true
54    }
55    fn is_movable(&self) -> bool {
56        false
57    }
58    #[cfg(feature = "sanity")]
59    fn is_sane(&self) -> bool {
60        true
61    }
62    fn initialize_object_metadata(&self, object: ObjectReference, _bytes: usize) {
63        self.mark_state
64            .on_object_metadata_initialization::<VM>(object);
65        if self.common.unlog_allocated_object {
66            VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.mark_as_unlogged::<VM>(object, Ordering::SeqCst);
67        }
68        #[cfg(feature = "vo_bit")]
69        crate::util::metadata::vo_bit::set_vo_bit(object);
70    }
71    #[cfg(feature = "vo_bit")]
72    fn is_mmtk_object(&self, addr: Address) -> Option<ObjectReference> {
73        crate::util::metadata::vo_bit::is_vo_bit_set_for_addr(addr)
74    }
75    #[cfg(feature = "vo_bit")]
76    fn find_object_from_internal_pointer(
77        &self,
78        ptr: Address,
79        max_search_bytes: usize,
80    ) -> Option<ObjectReference> {
81        crate::util::metadata::vo_bit::find_object_from_internal_pointer::<VM>(
82            ptr,
83            max_search_bytes,
84        )
85    }
86    fn sft_trace_object(
87        &self,
88        queue: &mut OptionObjectQueue,
89        object: ObjectReference,
90        _worker: GCWorkerMutRef,
91    ) -> ObjectReference {
92        self.trace_object(queue, object)
93    }
94}
95
96impl<VM: VMBinding> Space<VM> for VMSpace<VM> {
97    fn as_space(&self) -> &dyn Space<VM> {
98        self
99    }
100    fn as_sft(&self) -> &(dyn SFT + Sync + 'static) {
101        self
102    }
103    fn get_page_resource(&self) -> &dyn PageResource<VM> {
104        &self.pr
105    }
106    fn maybe_get_page_resource_mut(&mut self) -> Option<&mut dyn PageResource<VM>> {
107        Some(&mut self.pr)
108    }
109    fn common(&self) -> &CommonSpace<VM> {
110        &self.common
111    }
112
113    fn initialize_sft(&self, sft_map: &mut dyn crate::policy::sft_map::SFTMap) {
114        // Initialize sft for current external pages. This method is called at the end of plan creation.
115        // So we only set SFT for VM regions that are set by options (we skipped sft initialization for them earlier).
116        let vm_regions = self.pr.get_external_pages();
117        // We should have at most one region at this point (set by the option). If we allow setting multiple VM spaces through options,
118        // we can remove this assertion.
119        assert!(vm_regions.len() <= 1);
120        for external_pages in vm_regions.iter() {
121            // Chunk align things.
122            let start = external_pages.start.align_down(BYTES_IN_CHUNK);
123            let size = external_pages.end.align_up(BYTES_IN_CHUNK) - start;
124            // The region should be empty in SFT map -- if they were set before this point, there could be invalid SFT pointers.
125            debug_assert_eq!(
126                sft_map.get_checked(start).name(),
127                crate::policy::sft::EMPTY_SFT_NAME
128            );
129            self.set_sft(start, size);
130        }
131    }
132
133    fn initialize_side_metadata(&self) {
134        let vm_regions = self.pr.get_external_pages();
135        for external_pages in vm_regions.iter() {
136            // Chunk align things.
137            let chunk_start = external_pages.start.align_down(BYTES_IN_CHUNK);
138            let chunk_size = external_pages.end.align_up(BYTES_IN_CHUNK) - chunk_start;
139            let raw_start = external_pages.start;
140            let raw_size = external_pages.end - external_pages.start;
141            self.set_side_metadata(chunk_start, chunk_size, raw_start, raw_size);
142        }
143    }
144
145    fn release_multiple_pages(&mut self, _start: Address) {
146        unreachable!()
147    }
148
149    fn acquire(&self, _tls: VMThread, _pages: usize, _alloc_options: AllocationOptions) -> Address {
150        unreachable!()
151    }
152
153    fn address_in_space(&self, start: Address) -> bool {
154        // The default implementation checks with vm map. But vm map has some assumptions about
155        // the address range for spaces and the VM space may break those assumptions (as the space is
156        // mmapped by the runtime rather than us). So we we use SFT here.
157        SFT_MAP.get_checked(start).name() == self.name()
158    }
159
160    fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) {
161        let external_pages = self.pr.get_external_pages();
162        for ep in external_pages.iter() {
163            enumerator.visit_address_range(ep.start, ep.end);
164        }
165    }
166
167    fn clear_side_log_bits(&self) {
168        let log_bit = VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.extract_side_spec();
169        let external_pages = self.pr.get_external_pages();
170        for ep in external_pages.iter() {
171            log_bit.bzero_metadata(ep.start, ep.end - ep.start);
172        }
173    }
174
175    fn set_side_log_bits(&self) {
176        let log_bit = VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.extract_side_spec();
177        let external_pages = self.pr.get_external_pages();
178        for ep in external_pages.iter() {
179            log_bit.bset_metadata(ep.start, ep.end - ep.start);
180        }
181    }
182}
183
184use crate::scheduler::GCWorker;
185use crate::util::copy::CopySemantics;
186
187impl<VM: VMBinding> crate::policy::gc_work::PolicyTraceObject<VM> for VMSpace<VM> {
188    fn trace_object<Q: ObjectQueue, const KIND: crate::policy::gc_work::TraceKind>(
189        &self,
190        queue: &mut Q,
191        object: ObjectReference,
192        _copy: Option<CopySemantics>,
193        _worker: &mut GCWorker<VM>,
194    ) -> ObjectReference {
195        self.trace_object(queue, object)
196    }
197    fn may_move_objects<const KIND: crate::policy::gc_work::TraceKind>() -> bool {
198        false
199    }
200}
201
202impl<VM: VMBinding> VMSpace<VM> {
203    pub fn new(args: crate::policy::space::PlanCreateSpaceArgs<VM>) -> Self {
204        let (vm_space_start, vm_space_size) =
205            (*args.options.vm_space_start, *args.options.vm_space_size);
206        let space = Self {
207            mark_state: MarkState::new(),
208            pr: ExternalPageResource::new(args.vm_map),
209            common: CommonSpace::new(args.into_policy_args(
210                false,
211                true,
212                crate::util::metadata::extract_side_metadata(&[
213                    *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC,
214                ]),
215            )),
216        };
217
218        if !vm_space_start.is_zero() {
219            // Do not set sft here, as the space may be moved. We do so for those regions in `initialize_sft`.
220            space.set_vm_region_inner(vm_space_start, vm_space_size, true);
221        }
222
223        space
224    }
225
226    pub fn set_vm_region(&mut self, start: Address, size: usize) {
227        self.set_vm_region_inner(start, size, false);
228    }
229
230    fn set_vm_region_inner(&self, start: Address, size: usize, init: bool) {
231        assert!(size > 0);
232        assert!(!start.is_zero());
233
234        let end = start + size;
235
236        let chunk_start = start.align_down(BYTES_IN_CHUNK);
237        let chunk_end = end.align_up(BYTES_IN_CHUNK);
238        let chunk_size = chunk_end - chunk_start;
239
240        // For simplicity, VMSpace has to be outside our available heap range.
241        // TODO: Allow VMSpace in our available heap range.
242        assert!(Address::range_intersection(
243            &(chunk_start..chunk_end),
244            &crate::util::heap::layout::available_range()
245        )
246        .is_empty());
247
248        debug!(
249            "Align VM space ({}, {}) to chunk ({}, {})",
250            start, end, chunk_start, chunk_end
251        );
252
253        // Mark as mapped in mmapper
254        self.common.mmapper.mark_as_mapped(chunk_start, chunk_size);
255        // Map side metadata -- we can't access side metadata during initialization. We do it in initialize_side_metadata instead.
256        if !init {
257            self.set_side_metadata(chunk_start, chunk_size, start, size);
258        }
259        // Insert to vm map: it would be good if we can make VM map aware of the region. However, the region may be outside what we can map in our VM map implementation.
260        // self.common.vm_map.insert(chunk_start, chunk_size, self.common.descriptor);
261        // Set SFT if we should
262        if !init {
263            self.set_sft(chunk_start, chunk_size);
264        }
265
266        self.pr.add_new_external_pages(ExternalPages {
267            start: start.align_down(BYTES_IN_PAGE),
268            end: end.align_up(BYTES_IN_PAGE),
269        });
270    }
271
272    fn set_sft(&self, chunk_start: Address, chunk_size: usize) {
273        assert!(SFT_MAP.has_sft_entry(chunk_start), "The VM space start (aligned to {}) does not have a valid SFT entry. Possibly the address range is not in the address range we use.", chunk_start);
274        unsafe {
275            SFT_MAP.update(self.as_sft(), chunk_start, chunk_size);
276        }
277    }
278
279    fn set_side_metadata(
280        &self,
281        chunk_start: Address,
282        chunk_size: usize,
283        _raw_start: Address,
284        _raw_size: usize,
285    ) {
286        self.common
287            .metadata
288            .try_map_metadata_space(chunk_start, chunk_size, self.get_name())
289            .unwrap();
290        #[cfg(feature = "set_unlog_bits_vm_space")]
291        if self.common.needs_log_bit {
292            // Bulk set unlog bits for all addresses in the VM space. This ensures that any
293            // modification to the bootimage is logged
294            if let MetadataSpec::OnSide(side) = *VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC {
295                side.bset_metadata(_raw_start, _raw_size);
296            }
297        }
298    }
299
300    pub fn prepare(&mut self) {
301        self.mark_state.on_global_prepare::<VM>();
302        for external_pages in self.pr.get_external_pages().iter() {
303            self.mark_state.on_block_reset::<VM>(
304                external_pages.start,
305                external_pages.end - external_pages.start,
306            );
307        }
308    }
309
310    pub fn release(&mut self) {
311        self.mark_state.on_global_release::<VM>();
312    }
313
314    pub fn trace_object<Q: ObjectQueue>(
315        &self,
316        queue: &mut Q,
317        object: ObjectReference,
318    ) -> ObjectReference {
319        #[cfg(feature = "vo_bit")]
320        debug_assert!(
321            crate::util::metadata::vo_bit::is_vo_bit_set(object),
322            "{:x}: VO bit not set",
323            object
324        );
325        debug_assert!(self.in_space(object));
326        if self.mark_state.test_and_mark::<VM>(object) {
327            // Flip the per-object unlogged bits to "unlogged" state for objects inside the
328            // bootimage
329            #[cfg(feature = "set_unlog_bits_vm_space")]
330            if self.common.unlog_traced_object {
331                VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.store_atomic::<VM, u8>(
332                    object,
333                    1,
334                    None,
335                    Ordering::SeqCst,
336                );
337            }
338            queue.enqueue(object);
339        }
340        object
341    }
342}
mmtk/policy/vmspace.rs

mmtk/policy/
vmspace.rs
