mmtk/policy/
vmspace.rs

1use crate::mmtk::SFT_MAP;
2use crate::plan::tracing::OptionObjectQueue;
3use crate::plan::ObjectQueue;
4use crate::policy::sft::GCWorkerMutRef;
5use crate::policy::sft::SFT;
6use crate::policy::space::{CommonSpace, Space};
7use crate::util::address::Address;
8use crate::util::alloc::allocator::AllocationOptions;
9use crate::util::constants::BYTES_IN_PAGE;
10use crate::util::heap::externalpageresource::{ExternalPageResource, ExternalPages};
11use crate::util::heap::layout::vm_layout::BYTES_IN_CHUNK;
12use crate::util::heap::PageResource;
13use crate::util::metadata::mark_bit::MarkState;
14#[cfg(feature = "set_unlog_bits_vm_space")]
15use crate::util::metadata::MetadataSpec;
16use crate::util::object_enum::ObjectEnumerator;
17use crate::util::opaque_pointer::*;
18use crate::util::ObjectReference;
19use crate::vm::{ObjectModel, VMBinding};
20
21use std::sync::atomic::Ordering;
22
23/// A special space for VM/Runtime managed memory. The implementation is similar to [`crate::policy::immortalspace::ImmortalSpace`],
24/// except that VM space does not allocate. Instead, the runtime can add regions that are externally managed
25/// and mmapped to the space, and allow objects in those regions to be traced in the same way
26/// as other MMTk objects allocated by MMTk.
27pub struct VMSpace<VM: VMBinding> {
28    mark_state: MarkState,
29    common: CommonSpace<VM>,
30    pr: ExternalPageResource<VM>,
31}
32
33impl<VM: VMBinding> SFT for VMSpace<VM> {
34    fn name(&self) -> &'static str {
35        self.common.name
36    }
37    fn is_live(&self, _object: ObjectReference) -> bool {
38        true
39    }
40    fn is_reachable(&self, object: ObjectReference) -> bool {
41        self.mark_state.is_marked::<VM>(object)
42    }
43    #[cfg(feature = "object_pinning")]
44    fn pin_object(&self, _object: ObjectReference) -> bool {
45        false
46    }
47    #[cfg(feature = "object_pinning")]
48    fn unpin_object(&self, _object: ObjectReference) -> bool {
49        false
50    }
51    #[cfg(feature = "object_pinning")]
52    fn is_object_pinned(&self, _object: ObjectReference) -> bool {
53        true
54    }
55    fn is_movable(&self) -> bool {
56        false
57    }
58    #[cfg(feature = "sanity")]
59    fn is_sane(&self) -> bool {
60        true
61    }
62    fn initialize_object_metadata(&self, object: ObjectReference, _bytes: usize) {
63        self.mark_state
64            .on_object_metadata_initialization::<VM>(object);
65        if self.common.unlog_allocated_object {
66            VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.mark_as_unlogged::<VM>(object, Ordering::SeqCst);
67        }
68        #[cfg(feature = "vo_bit")]
69        crate::util::metadata::vo_bit::set_vo_bit(object);
70    }
71    #[cfg(feature = "vo_bit")]
72    fn is_mmtk_object(&self, addr: Address) -> Option<ObjectReference> {
73        crate::util::metadata::vo_bit::is_vo_bit_set_for_addr(addr)
74    }
75    #[cfg(feature = "vo_bit")]
76    fn find_object_from_internal_pointer(
77        &self,
78        ptr: Address,
79        max_search_bytes: usize,
80    ) -> Option<ObjectReference> {
81        crate::util::metadata::vo_bit::find_object_from_internal_pointer::<VM>(
82            ptr,
83            max_search_bytes,
84        )
85    }
86    fn sft_trace_object(
87        &self,
88        queue: &mut OptionObjectQueue,
89        object: ObjectReference,
90        _worker: GCWorkerMutRef,
91    ) -> ObjectReference {
92        self.trace_object(queue, object)
93    }
94}
95
96impl<VM: VMBinding> Space<VM> for VMSpace<VM> {
97    fn as_space(&self) -> &dyn Space<VM> {
98        self
99    }
100    fn as_sft(&self) -> &(dyn SFT + Sync + 'static) {
101        self
102    }
103    fn get_page_resource(&self) -> &dyn PageResource<VM> {
104        &self.pr
105    }
106    fn maybe_get_page_resource_mut(&mut self) -> Option<&mut dyn PageResource<VM>> {
107        Some(&mut self.pr)
108    }
109    fn common(&self) -> &CommonSpace<VM> {
110        &self.common
111    }
112
113    fn initialize_sft(&self, sft_map: &mut dyn crate::policy::sft_map::SFTMap) {
114        // Initialize sft for current external pages. This method is called at the end of plan creation.
115        // So we only set SFT for VM regions that are set by options (we skipped sft initialization for them earlier).
116        let vm_regions = self.pr.get_external_pages();
117        // We should have at most one region at this point (set by the option). If we allow setting multiple VM spaces through options,
118        // we can remove this assertion.
119        assert!(vm_regions.len() <= 1);
120        for external_pages in vm_regions.iter() {
121            // Chunk align things.
122            let start = external_pages.start.align_down(BYTES_IN_CHUNK);
123            let size = external_pages.end.align_up(BYTES_IN_CHUNK) - start;
124            // The region should be empty in SFT map -- if they were set before this point, there could be invalid SFT pointers.
125            debug_assert_eq!(
126                sft_map.get_checked(start).name(),
127                crate::policy::sft::EMPTY_SFT_NAME
128            );
129            // Set SFT
130            assert!(sft_map.has_sft_entry(start), "The VM space start (aligned to {}) does not have a valid SFT entry. Possibly the address range is not in the address range we use.", start);
131            unsafe {
132                sft_map.eager_initialize(self.as_sft(), start, size);
133            }
134        }
135    }
136
137    fn release_multiple_pages(&mut self, _start: Address) {
138        unreachable!()
139    }
140
141    fn acquire(&self, _tls: VMThread, _pages: usize, _alloc_options: AllocationOptions) -> Address {
142        unreachable!()
143    }
144
145    fn address_in_space(&self, start: Address) -> bool {
146        // The default implementation checks with vm map. But vm map has some assumptions about
147        // the address range for spaces and the VM space may break those assumptions (as the space is
148        // mmapped by the runtime rather than us). So we we use SFT here.
149        SFT_MAP.get_checked(start).name() == self.name()
150    }
151
152    fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) {
153        let external_pages = self.pr.get_external_pages();
154        for ep in external_pages.iter() {
155            enumerator.visit_address_range(ep.start, ep.end);
156        }
157    }
158
159    fn clear_side_log_bits(&self) {
160        let log_bit = VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.extract_side_spec();
161        let external_pages = self.pr.get_external_pages();
162        for ep in external_pages.iter() {
163            log_bit.bzero_metadata(ep.start, ep.end - ep.start);
164        }
165    }
166
167    fn set_side_log_bits(&self) {
168        let log_bit = VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.extract_side_spec();
169        let external_pages = self.pr.get_external_pages();
170        for ep in external_pages.iter() {
171            log_bit.bset_metadata(ep.start, ep.end - ep.start);
172        }
173    }
174}
175
176use crate::scheduler::GCWorker;
177use crate::util::copy::CopySemantics;
178
179impl<VM: VMBinding> crate::policy::gc_work::PolicyTraceObject<VM> for VMSpace<VM> {
180    fn trace_object<Q: ObjectQueue, const KIND: crate::policy::gc_work::TraceKind>(
181        &self,
182        queue: &mut Q,
183        object: ObjectReference,
184        _copy: Option<CopySemantics>,
185        _worker: &mut GCWorker<VM>,
186    ) -> ObjectReference {
187        self.trace_object(queue, object)
188    }
189    fn may_move_objects<const KIND: crate::policy::gc_work::TraceKind>() -> bool {
190        false
191    }
192}
193
194impl<VM: VMBinding> VMSpace<VM> {
195    pub fn new(args: crate::policy::space::PlanCreateSpaceArgs<VM>) -> Self {
196        let (vm_space_start, vm_space_size) =
197            (*args.options.vm_space_start, *args.options.vm_space_size);
198        let space = Self {
199            mark_state: MarkState::new(),
200            pr: ExternalPageResource::new(args.vm_map),
201            common: CommonSpace::new(args.into_policy_args(
202                false,
203                true,
204                crate::util::metadata::extract_side_metadata(&[
205                    *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC,
206                ]),
207            )),
208        };
209
210        if !vm_space_start.is_zero() {
211            // Do not set sft here, as the space may be moved. We do so for those regions in `initialize_sft`.
212            space.set_vm_region_inner(vm_space_start, vm_space_size, false);
213        }
214
215        space
216    }
217
218    pub fn set_vm_region(&mut self, start: Address, size: usize) {
219        self.set_vm_region_inner(start, size, true);
220    }
221
222    fn set_vm_region_inner(&self, start: Address, size: usize, set_sft: bool) {
223        assert!(size > 0);
224        assert!(!start.is_zero());
225
226        let end = start + size;
227
228        let chunk_start = start.align_down(BYTES_IN_CHUNK);
229        let chunk_end = end.align_up(BYTES_IN_CHUNK);
230        let chunk_size = chunk_end - chunk_start;
231
232        // For simplicity, VMSpace has to be outside our available heap range.
233        // TODO: Allow VMSpace in our available heap range.
234        assert!(Address::range_intersection(
235            &(chunk_start..chunk_end),
236            &crate::util::heap::layout::available_range()
237        )
238        .is_empty());
239
240        debug!(
241            "Align VM space ({}, {}) to chunk ({}, {})",
242            start, end, chunk_start, chunk_end
243        );
244
245        // Mark as mapped in mmapper
246        self.common.mmapper.mark_as_mapped(chunk_start, chunk_size);
247        // Map side metadata
248        self.common
249            .metadata
250            .try_map_metadata_space(chunk_start, chunk_size, self.get_name())
251            .unwrap();
252        // Insert to vm map: it would be good if we can make VM map aware of the region. However, the region may be outside what we can map in our VM map implementation.
253        // self.common.vm_map.insert(chunk_start, chunk_size, self.common.descriptor);
254        // Set SFT if we should
255        if set_sft {
256            assert!(SFT_MAP.has_sft_entry(chunk_start), "The VM space start (aligned to {}) does not have a valid SFT entry. Possibly the address range is not in the address range we use.", chunk_start);
257            unsafe {
258                SFT_MAP.update(self.as_sft(), chunk_start, chunk_size);
259            }
260        }
261
262        self.pr.add_new_external_pages(ExternalPages {
263            start: start.align_down(BYTES_IN_PAGE),
264            end: end.align_up(BYTES_IN_PAGE),
265        });
266
267        #[cfg(feature = "set_unlog_bits_vm_space")]
268        if self.common.needs_log_bit {
269            // Bulk set unlog bits for all addresses in the VM space. This ensures that any
270            // modification to the bootimage is logged
271            if let MetadataSpec::OnSide(side) = *VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC {
272                side.bset_metadata(start, size);
273            }
274        }
275    }
276
277    pub fn prepare(&mut self) {
278        self.mark_state.on_global_prepare::<VM>();
279        for external_pages in self.pr.get_external_pages().iter() {
280            self.mark_state.on_block_reset::<VM>(
281                external_pages.start,
282                external_pages.end - external_pages.start,
283            );
284        }
285    }
286
287    pub fn release(&mut self) {
288        self.mark_state.on_global_release::<VM>();
289    }
290
291    pub fn trace_object<Q: ObjectQueue>(
292        &self,
293        queue: &mut Q,
294        object: ObjectReference,
295    ) -> ObjectReference {
296        #[cfg(feature = "vo_bit")]
297        debug_assert!(
298            crate::util::metadata::vo_bit::is_vo_bit_set(object),
299            "{:x}: VO bit not set",
300            object
301        );
302        debug_assert!(self.in_space(object));
303        if self.mark_state.test_and_mark::<VM>(object) {
304            // Flip the per-object unlogged bits to "unlogged" state for objects inside the
305            // bootimage
306            #[cfg(feature = "set_unlog_bits_vm_space")]
307            if self.common.unlog_traced_object {
308                VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.store_atomic::<VM, u8>(
309                    object,
310                    1,
311                    None,
312                    Ordering::SeqCst,
313                );
314            }
315            queue.enqueue(object);
316        }
317        object
318    }
319}
mmtk/policy/vmspace.rs

mmtk/policy/
vmspace.rs
