mmtk/util/heap/
monotonepageresource.rs

1use super::layout::vm_layout::{BYTES_IN_CHUNK, PAGES_IN_CHUNK};
2use crate::policy::space::required_chunks;
3use crate::util::address::Address;
4use crate::util::constants::BYTES_IN_PAGE;
5use crate::util::conversions::*;
6use std::ops::Range;
7use std::sync::{Mutex, MutexGuard};
8
9use crate::util::alloc::embedded_meta_data::*;
10use crate::util::heap::layout::vm_layout::LOG_BYTES_IN_CHUNK;
11use crate::util::heap::pageresource::CommonPageResource;
12use crate::util::opaque_pointer::*;
13
14use super::layout::VMMap;
15use super::pageresource::{PRAllocFail, PRAllocResult};
16use super::PageResource;
17use crate::util::heap::space_descriptor::SpaceDescriptor;
18use crate::vm::VMBinding;
19use std::marker::PhantomData;
20
21pub struct MonotonePageResource<VM: VMBinding> {
22    common: CommonPageResource,
23    sync: Mutex<MonotonePageResourceSync>,
24    _p: PhantomData<VM>,
25}
26
27struct MonotonePageResourceSync {
28    /** Pointer to the next block to be allocated. */
29    cursor: Address,
30    /** The limit of the currently allocated address space. */
31    sentinel: Address,
32    /** Base address of the current chunk of addresses */
33    current_chunk: Address,
34    conditional: MonotonePageResourceConditional,
35}
36
37pub enum MonotonePageResourceConditional {
38    Contiguous {
39        start: Address,
40        /** Current frontier of zeroing, in a separate zeroing thread */
41        zeroing_cursor: Address,
42        /** Current limit of zeroing.  If zeroingCursor < zeroingSentinel, zeroing is still happening. */
43        zeroing_sentinel: Address,
44    },
45    Discontiguous,
46}
47impl<VM: VMBinding> PageResource<VM> for MonotonePageResource<VM> {
48    fn common(&self) -> &CommonPageResource {
49        &self.common
50    }
51    fn common_mut(&mut self) -> &mut CommonPageResource {
52        &mut self.common
53    }
54
55    fn reserve_pages(&self, pages: usize) -> usize {
56        self.common().accounting.reserve(pages);
57        pages
58    }
59
60    fn get_available_physical_pages(&self) -> usize {
61        let sync = self.sync.lock().unwrap();
62        let mut rtn = bytes_to_pages_up(sync.sentinel - sync.cursor);
63        if !self.common.contiguous {
64            rtn += self.common.vm_map.get_available_discontiguous_chunks() * PAGES_IN_CHUNK;
65        }
66        rtn
67    }
68
69    fn alloc_pages(
70        &self,
71        space_descriptor: SpaceDescriptor,
72        reserved_pages: usize,
73        required_pages: usize,
74        tls: VMThread,
75    ) -> Result<PRAllocResult, PRAllocFail> {
76        debug!(
77            "In MonotonePageResource, reserved_pages = {}, required_pages = {}",
78            reserved_pages, required_pages
79        );
80        let mut new_chunk = false;
81        let mut sync = self.sync.lock().unwrap();
82        let mut rtn = sync.cursor;
83        debug!(
84            "cursor = {}, sentinel = {}, current_chunk = {}",
85            sync.cursor, sync.sentinel, sync.current_chunk
86        );
87
88        if cfg!(debug_assertions) {
89            /*
90             * Cursor should always be zero, or somewhere in the current chunk.  If we have just
91             * allocated exactly enough pages to exhaust the current chunk, then cursor can point
92             * to the next chunk.
93             */
94            if sync.current_chunk > sync.cursor
95                || (chunk_align_down(sync.cursor) != sync.current_chunk
96                    && chunk_align_down(sync.cursor) != sync.current_chunk + BYTES_IN_CHUNK)
97            {
98                self.log_chunk_fields(space_descriptor, "MonotonePageResource.alloc_pages:fail");
99            }
100            assert!(sync.current_chunk <= sync.cursor);
101            assert!(
102                sync.cursor.is_zero()
103                    || chunk_align_down(sync.cursor) == sync.current_chunk
104                    || chunk_align_down(sync.cursor) == (sync.current_chunk + BYTES_IN_CHUNK)
105            );
106        }
107
108        let bytes = pages_to_bytes(required_pages);
109        debug!("bytes={}", bytes);
110        let mut tmp = sync.cursor + bytes;
111        debug!("tmp={:?}", tmp);
112
113        if !self.common().contiguous && tmp > sync.sentinel {
114            /* we're out of virtual memory within our discontiguous region, so ask for more */
115            let required_chunks = required_chunks(required_pages);
116            sync.current_chunk =
117                self.common
118                    .grow_discontiguous_space(space_descriptor, required_chunks, None); // Returns zero on failure
119            sync.cursor = sync.current_chunk;
120            sync.sentinel = sync.cursor
121                + if sync.current_chunk.is_zero() {
122                    0
123                } else {
124                    required_chunks << LOG_BYTES_IN_CHUNK
125                };
126            //println!("{} {}->{}", self.common.space.unwrap().get_name(), sync.cursor, sync.sentinel);
127            rtn = sync.cursor;
128            tmp = sync.cursor + bytes;
129            new_chunk = true;
130        }
131
132        debug_assert!(rtn >= sync.cursor && rtn < sync.cursor + bytes);
133        if tmp > sync.sentinel {
134            //debug!("tmp={:?} > sync.sentinel={:?}", tmp, sync.sentinel);
135            Result::Err(PRAllocFail)
136        } else {
137            //debug!("tmp={:?} <= sync.sentinel={:?}", tmp, sync.sentinel);
138            sync.cursor = tmp;
139            debug!("update cursor = {}", tmp);
140
141            /* In a contiguous space we can bump along into the next chunk, so preserve the currentChunk invariant */
142            if self.common().contiguous && chunk_align_down(sync.cursor) != sync.current_chunk {
143                debug_assert!(
144                    chunk_align_down(sync.cursor) > sync.current_chunk,
145                    "Not monotonic.  chunk_align_down(sync.cursor): {}, sync.current_chunk: {}",
146                    chunk_align_down(sync.cursor),
147                    sync.current_chunk,
148                );
149                sync.current_chunk = chunk_align_down(sync.cursor);
150            }
151            self.commit_pages(reserved_pages, required_pages, tls);
152
153            Result::Ok(PRAllocResult {
154                start: rtn,
155                pages: required_pages,
156                new_chunk,
157            })
158        }
159    }
160}
161
162impl<VM: VMBinding> MonotonePageResource<VM> {
163    pub fn new_contiguous(start: Address, bytes: usize, vm_map: &'static dyn VMMap) -> Self {
164        let sentinel = start + bytes;
165
166        MonotonePageResource {
167            common: CommonPageResource::new(true, cfg!(target_pointer_width = "64"), vm_map),
168            sync: Mutex::new(MonotonePageResourceSync {
169                cursor: start,
170                current_chunk: chunk_align_down(start),
171                sentinel,
172                conditional: MonotonePageResourceConditional::Contiguous {
173                    start,
174                    zeroing_cursor: sentinel,
175                    zeroing_sentinel: start,
176                },
177            }),
178            _p: PhantomData,
179        }
180    }
181
182    pub fn new_discontiguous(vm_map: &'static dyn VMMap) -> Self {
183        MonotonePageResource {
184            common: CommonPageResource::new(false, true, vm_map),
185            sync: Mutex::new(MonotonePageResourceSync {
186                cursor: unsafe { Address::zero() },
187                current_chunk: unsafe { Address::zero() },
188                sentinel: unsafe { Address::zero() },
189                conditional: MonotonePageResourceConditional::Discontiguous,
190            }),
191            _p: PhantomData,
192        }
193    }
194
195    /// Get highwater mark of current monotone space.
196    pub fn cursor(&self) -> Address {
197        self.sync.lock().unwrap().cursor
198    }
199
200    fn log_chunk_fields(&self, space_descriptor: SpaceDescriptor, site: &str) {
201        let sync = self.sync.lock().unwrap();
202        debug!(
203            "[{:?}]{}: cursor={}, current_chunk={}, delta={}",
204            space_descriptor,
205            site,
206            sync.cursor,
207            sync.current_chunk,
208            sync.cursor - sync.current_chunk
209        );
210    }
211
212    fn get_region_start(addr: Address) -> Address {
213        addr.align_down(BYTES_IN_REGION)
214    }
215
216    /// # Safety
217    /// TODO: I am not sure why this is unsafe.
218    pub unsafe fn reset(&self) {
219        let mut guard = self.sync.lock().unwrap();
220        self.common().accounting.reset();
221        self.release_pages(&mut guard);
222        drop(guard);
223    }
224
225    pub unsafe fn get_current_chunk(&self) -> Address {
226        let guard = self.sync.lock().unwrap();
227        guard.current_chunk
228    }
229
230    /*/**
231    * Release all pages associated with this page resource, optionally
232    * zeroing on release and optionally memory protecting on release.
233    */
234     @Inline
235     private void releasePages() {
236     if (contiguous) {
237     // TODO: We will perform unnecessary zeroing if the nursery size has decreased.
238     if (zeroConcurrent) {
239     // Wait for current zeroing to finish.
240     while (zeroingCursor.LT(zeroingSentinel)) { }
241     }
242     // Reset zeroing region.
243     if (cursor.GT(zeroingSentinel)) {
244     zeroingSentinel = cursor;
245     }
246     zeroingCursor = start;
247     cursor = start;
248     currentChunk = Conversions.chunkAlign(start, true);
249     } else { /* Not contiguous */
250     if (!cursor.isZero()) {
251     do {
252     Extent bytes = cursor.diff(currentChunk).toWord().toExtent();
253     releasePages(currentChunk, bytes);
254     } while (moveToNextChunk());
255
256     currentChunk = Address.zero();
257     sentinel = Address.zero();
258     cursor = Address.zero();
259     space.releaseAllChunks();
260     }
261     }
262     }*/
263
264    // top might be Address::ZERO
265    pub fn reset_cursor(&self, mut top: Address) {
266        if self.common.contiguous {
267            let mut guard = self.sync.lock().unwrap();
268            let space_start = match guard.conditional {
269                MonotonePageResourceConditional::Contiguous { start, .. } => start,
270                _ => unreachable!(),
271            };
272            if top.is_zero() {
273                top = space_start;
274            }
275            let cursor = top.align_up(crate::util::constants::BYTES_IN_PAGE);
276            let chunk = chunk_align_down(top);
277            let pages = bytes_to_pages_up(top - space_start);
278            self.common.accounting.reset();
279            self.common.accounting.reserve_and_commit(pages);
280            guard.current_chunk = chunk;
281            guard.cursor = cursor;
282        } else {
283            let mut chunk_start = self.common.get_head_discontiguous_region();
284            let mut release_regions = false;
285            let mut live_size = 0;
286            while !chunk_start.is_zero() {
287                let chunk_end = chunk_start
288                    + (self.common.vm_map.get_contiguous_region_chunks(chunk_start)
289                        << LOG_BYTES_IN_CHUNK);
290                let next_chunk_start = self.common.vm_map.get_next_contiguous_region(chunk_start);
291                if top.is_zero() {
292                    top = chunk_start;
293                }
294                if top >= chunk_start && top < chunk_end {
295                    // This is the last live chunk
296                    debug_assert!(!release_regions);
297                    let mut guard = self.sync.lock().unwrap();
298                    guard.current_chunk = chunk_start;
299                    guard.sentinel = chunk_end;
300                    guard.cursor = top.align_up(BYTES_IN_PAGE);
301                    live_size += top - chunk_start;
302                    // Release all the remaining regions
303                    release_regions = true;
304                } else if release_regions {
305                    // release this region
306                    self.common.release_discontiguous_chunks(chunk_start);
307                } else {
308                    // keep this live region
309                    live_size += chunk_end - chunk_start;
310                }
311                chunk_start = next_chunk_start;
312            }
313            let pages = bytes_to_pages_up(live_size);
314            self.common.accounting.reset();
315            self.common.accounting.reserve_and_commit(pages);
316        }
317    }
318
319    unsafe fn release_pages(&self, guard: &mut MutexGuard<MonotonePageResourceSync>) {
320        // TODO: concurrent zeroing
321        if self.common().contiguous {
322            guard.cursor = match guard.conditional {
323                MonotonePageResourceConditional::Contiguous { start: _start, .. } => _start,
324                _ => unreachable!(),
325            };
326            guard.current_chunk = guard.cursor;
327        } else if !guard.cursor.is_zero() {
328            let bytes = guard.cursor - guard.current_chunk;
329            self.release_pages_extent(guard.current_chunk, bytes);
330            while self.move_to_next_chunk(guard) {
331                let bytes = guard.cursor - guard.current_chunk;
332                self.release_pages_extent(guard.current_chunk, bytes);
333            }
334
335            guard.current_chunk = Address::zero();
336            guard.sentinel = Address::zero();
337            guard.cursor = Address::zero();
338            self.common.release_all_chunks();
339        }
340    }
341
342    /// Iterate over all contiguous memory regions in this space.
343    /// For contiguous space, this iterator should yield only once, and returning a contiguous memory region covering the whole space.
344    pub fn iterate_allocated_regions(&self) -> impl Iterator<Item = (Address, usize)> + '_ {
345        struct Iter<'a, VM: VMBinding> {
346            pr: &'a MonotonePageResource<VM>,
347            contiguous_space: Option<Range<Address>>,
348            discontiguous_start: Address,
349        }
350        impl<VM: VMBinding> Iterator for Iter<'_, VM> {
351            type Item = (Address, usize);
352            fn next(&mut self) -> Option<Self::Item> {
353                if let Some(range) = self.contiguous_space.take() {
354                    Some((range.start, range.end - range.start))
355                } else if self.discontiguous_start.is_zero() {
356                    None
357                } else {
358                    let start = self.discontiguous_start;
359                    self.discontiguous_start = self.pr.vm_map().get_next_contiguous_region(start);
360
361                    let contiguous_region_size = self.pr.vm_map().get_contiguous_region_size(start);
362                    let cursor = self.pr.cursor();
363                    let size = if start < cursor && cursor < start + contiguous_region_size {
364                        // If the current cursor is within the current discontiguous region,
365                        // then return the size till the cursor.
366                        // This is sufficient for sweeping the memory and clearing side metadata.
367                        // Note that if cursor == start,
368                        // it means the cursor is at the end of the previous chunk.
369                        cursor - start
370                    } else {
371                        contiguous_region_size
372                    };
373                    Some((start, size))
374                }
375            }
376        }
377        let sync = self.sync.lock().unwrap();
378        match sync.conditional {
379            MonotonePageResourceConditional::Contiguous { start, .. } => {
380                let cursor = sync.cursor.align_up(BYTES_IN_CHUNK);
381                Iter {
382                    pr: self,
383                    contiguous_space: Some(start..cursor),
384                    discontiguous_start: Address::ZERO,
385                }
386            }
387            MonotonePageResourceConditional::Discontiguous => {
388                let discontiguous_start = self.common.get_head_discontiguous_region();
389                Iter {
390                    pr: self,
391                    contiguous_space: None,
392                    discontiguous_start,
393                }
394            }
395        }
396    }
397
398    fn release_pages_extent(&self, _first: Address, bytes: usize) {
399        let pages = crate::util::conversions::bytes_to_pages_up(bytes);
400        debug_assert!(bytes == crate::util::conversions::pages_to_bytes(pages));
401        // FIXME ZERO_PAGES_ON_RELEASE
402        // FIXME Options.protectOnRelease
403        // FIXME VM.events.tracePageReleased
404    }
405
406    fn move_to_next_chunk(&self, guard: &mut MutexGuard<MonotonePageResourceSync>) -> bool {
407        guard.current_chunk = self
408            .vm_map()
409            .get_next_contiguous_region(guard.current_chunk);
410        if guard.current_chunk.is_zero() {
411            false
412        } else {
413            guard.cursor = guard.current_chunk
414                + self
415                    .vm_map()
416                    .get_contiguous_region_size(guard.current_chunk);
417            true
418        }
419    }
420}
mmtk/util/heap/monotonepageresource.rs

mmtk/util/heap/
monotonepageresource.rs
