mmtk/util/os/
memory.rs

1use bytemuck::NoUninit;
2use std::io::Result;
3
4use crate::util::os::*;
5use crate::vm::*;
6use crate::{
7    util::{address::Address, VMThread},
8    vm::VMBinding,
9};
10
11/// Error returned by mmap-related operations in MMTk.
12#[derive(Debug)]
13pub struct MmapError {
14    /// The start address of the mmap operation that failed.
15    pub error_address: Address,
16    /// The size (in bytes) of the mmap operation that failed.
17    pub bytes: usize,
18    /// Human-readable annotation for the mmap operation.
19    ///
20    /// This is derived from [`MmapAnnotation`] at the call site.
21    pub annotation: String,
22    /// The underlying OS I/O error.
23    pub error: std::io::Error,
24}
25
26impl MmapError {
27    /// Create a new [`MmapError`].
28    pub fn new(
29        error_address: Address,
30        bytes: usize,
31        annotation: &MmapAnnotation<'_>,
32        error: std::io::Error,
33    ) -> Self {
34        Self {
35            error_address,
36            bytes,
37            annotation: annotation.to_string(),
38            error,
39        }
40    }
41}
42
43impl std::fmt::Display for MmapError {
44    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
45        write!(
46            f,
47            "mmap {} (size {}, annotation {}) failed: {}",
48            self.error_address, self.bytes, self.annotation, self.error
49        )
50    }
51}
52
53impl std::error::Error for MmapError {
54    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
55        Some(&self.error)
56    }
57}
58
59/// Result type for mmap operations that can return [`MmapError`].
60pub type MmapResult<T> = std::result::Result<T, MmapError>;
61
62/// Abstraction for OS memory operations.
63pub trait OSMemory {
64    /// Perform a demand-zero mmap.
65    ///
66    /// Fallback: `annotation` is only used for debugging. For platforms that do not support mmap annotations, this parameter can be ignored.
67    /// Fallback: see [`crate::util::os::MmapStrategy`] for fallbacks for `strategy`.
68    fn dzmmap(
69        start: Address,
70        size: usize,
71        strategy: MmapStrategy,
72        annotation: &MmapAnnotation<'_>,
73    ) -> MmapResult<Address>;
74
75    /// Perform a no-reserve mmap at any available address, aligned to `align`.
76    ///
77    /// This API is used for reserving address ranges (typically with `PROT_NONE`) before committing.
78    fn dzmmap_anywhere(
79        size: usize,
80        align: usize,
81        strategy: MmapStrategy,
82        annotation: &MmapAnnotation<'_>,
83    ) -> std::io::Result<Address>;
84
85    /// Handle mmap errors, possibly by signaling the VM about an out-of-memory condition.
86    fn handle_mmap_error<VM: VMBinding>(mmap_error: MmapError, tls: VMThread) {
87        use crate::util::alloc::AllocationError;
88        use std::io::ErrorKind;
89
90        eprintln!(
91            "Failed to mmap from {} to {} (size {}), annotation {}",
92            mmap_error.error_address,
93            mmap_error.error_address.wrapping_add(mmap_error.bytes),
94            mmap_error.bytes,
95            mmap_error.annotation
96        );
97        eprintln!("{}", OS::get_process_memory_maps().unwrap());
98
99        let call_binding_oom = || {
100            // Signal `MmapOutOfMemory`. Expect the VM to abort immediately.
101            trace!("Signal MmapOutOfMemory!");
102            VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory);
103            unreachable!()
104        };
105
106        match mmap_error.error.kind() {
107            // From Rust nightly 2021-05-12, we started to see Rust added this ErrorKind.
108            ErrorKind::OutOfMemory => {
109                call_binding_oom();
110            }
111            // Before Rust had ErrorKind::OutOfMemory, this is how we capture OOM from OS calls.
112            // TODO: We may be able to remove this now.
113            ErrorKind::Other => {
114                // further check the error
115                if let Some(os_errno) = mmap_error.error.raw_os_error() {
116                    if OS::is_mmap_oom(os_errno) {
117                        call_binding_oom();
118                    }
119                }
120            }
121            ErrorKind::AlreadyExists => {
122                panic!("Failed to mmap, the address is already mapped. Should MMTk quarantine the address range first?");
123            }
124            _ => {
125                if let Some(os_errno) = mmap_error.error.raw_os_error() {
126                    if OS::is_mmap_oom(os_errno) {
127                        call_binding_oom();
128                    }
129                }
130            }
131        }
132        panic!("Unexpected mmap failure: {:?}", mmap_error.error)
133    }
134
135    /// Check whether the given OS error number indicates an out-of-memory condition.
136    fn is_mmap_oom(os_errno: i32) -> bool;
137
138    /// Unmap a memory region.
139    fn munmap(start: Address, size: usize) -> Result<()>;
140
141    /// Change the protection of a memory region to the specified protection.
142    fn set_memory_access(start: Address, size: usize, prot: MmapProtection) -> Result<()>;
143
144    /// Checks if the memory has already been mapped. If not, we panic.
145    ///
146    /// Note that the checking may have a side effect that it will map the memory if it was unmapped. So we panic if it was unmapped.
147    /// Be very careful about using this function.
148    ///
149    /// Fallback: As the function is only used for assertions, it can be a no-op, and MMTk will still run and never panics in this function.
150    fn panic_if_unmapped(start: Address, size: usize);
151
152    /// Get the total memory of the system in bytes.
153    fn get_system_total_memory() -> Result<u64> {
154        use sysinfo::MemoryRefreshKind;
155        use sysinfo::{RefreshKind, System};
156
157        // TODO: Note that if we want to get system info somewhere else in the future, we should
158        // refactor this instance into some global struct. sysinfo recommends sharing one instance of
159        // `System` instead of making multiple instances.
160        // See https://docs.rs/sysinfo/0.29.0/sysinfo/index.html#usage for more info
161        //
162        // If we refactor the `System` instance to use it for other purposes, please make sure start-up
163        // time is not affected.  It takes a long time to load all components in sysinfo (e.g. by using
164        // `System::new_all()`).  Some applications, especially short-running scripts, are sensitive to
165        // start-up time.  During start-up, MMTk core only needs the total memory to initialize the
166        // `Options`.  If we only load memory-related components on start-up, it should only take <1ms
167        // to initialize the `System` instance.
168        let sys = System::new_with_specifics(
169            RefreshKind::nothing().with_memory(MemoryRefreshKind::nothing().with_ram()),
170        );
171        Ok(sys.total_memory())
172    }
173}
174
175/// Strategy for performing mmap
176#[derive(Debug, Copy, Clone)]
177pub struct MmapStrategy {
178    /// Whether we should use huge page for this mmapping.
179    /// Fallback: for platforms that do not support huge pages, this option can be ignored.
180    pub huge_page: HugePageSupport,
181    /// The protection flags for mmap.
182    pub prot: MmapProtection,
183    /// Whether this mmap allows replacing existing mappings.
184    /// Fallback: for platforms that cannot replace existing mappings, or always replace existing mappings, this option can be ignored.
185    pub replace: bool,
186    /// Whether this mmap allows reserve/commit physical memory.
187    /// This has to be implemented properly for a platform. Otherwise, we will see huge unrealistic memory consumption.
188    pub reserve: bool,
189}
190
191impl std::default::Default for MmapStrategy {
192    fn default() -> Self {
193        Self {
194            huge_page: HugePageSupport::No,
195            prot: MmapProtection::ReadWrite,
196            replace: false,
197            reserve: true,
198        }
199    }
200}
201
202impl MmapStrategy {
203    /// Create a new strategy
204    pub fn new(
205        huge_page: HugePageSupport,
206        prot: MmapProtection,
207        replace: bool,
208        reserve: bool,
209    ) -> Self {
210        Self {
211            huge_page,
212            prot,
213            replace,
214            reserve,
215        }
216    }
217
218    // Builder methods
219
220    /// Set huge page option.
221    pub fn huge_page(self, huge_page: HugePageSupport) -> Self {
222        Self { huge_page, ..self }
223    }
224
225    /// Set huge page option by a boolean flag.
226    pub fn transparent_hugepages(self, enable: bool) -> Self {
227        let huge_page = if enable {
228            HugePageSupport::TransparentHugePages
229        } else {
230            HugePageSupport::No
231        };
232        Self { huge_page, ..self }
233    }
234
235    /// Set protection option.
236    pub fn prot(self, prot: MmapProtection) -> Self {
237        Self { prot, ..self }
238    }
239
240    /// Set the replace flag.
241    pub fn replace(self, replace: bool) -> Self {
242        Self { replace, ..self }
243    }
244
245    /// Set the reserve flag.
246    pub fn reserve(self, reserve: bool) -> Self {
247        Self { reserve, ..self }
248    }
249
250    #[cfg(test)] // In test mode, we use test settings which allows replacing existing mappings.
251    /// The strategy for MMTk's own internal memory (test)
252    pub const INTERNAL_MEMORY: Self = Self::TEST;
253    #[cfg(not(test))]
254    /// The strategy for MMTk's own internal memory
255    pub const INTERNAL_MEMORY: Self = Self {
256        huge_page: HugePageSupport::No,
257        prot: MmapProtection::ReadWrite,
258        replace: false,
259        reserve: true,
260    };
261
262    /// The strategy for quarantining address ranges.
263    pub const QUARANTINE: Self = Self {
264        huge_page: HugePageSupport::No,
265        prot: MmapProtection::NoAccess,
266        // In test mode, we allow replacing existing mappings for quarantine,
267        // so that we can reuse the same address range for multiple test cases.
268        replace: cfg!(test),
269        reserve: false,
270    };
271
272    /// The strategy for MMTk's test memory
273    #[cfg(test)]
274    pub const TEST: Self = Self {
275        huge_page: HugePageSupport::No,
276        prot: MmapProtection::ReadWrite,
277        replace: true,
278        reserve: true,
279    };
280}
281
282/// The protection flags for Mmap
283#[repr(i32)]
284#[derive(Debug, Copy, Clone)]
285pub enum MmapProtection {
286    /// Allow read + write
287    ReadWrite,
288    /// Allow read + write + code execution
289    ReadWriteExec,
290    /// Do not allow any access
291    NoAccess,
292}
293
294/// Support for huge pages
295#[repr(u8)]
296#[derive(Debug, Copy, Clone, NoUninit)]
297pub enum HugePageSupport {
298    /// No support for huge page
299    No,
300    /// Enable transparent huge pages for the pages that are mapped. This option is only for linux.
301    TransparentHugePages,
302}
303
304/// Annotation for an mmap entry.
305///
306/// Invocations of `mmap_fixed` and other functions that may transitively call `mmap_fixed`
307/// require an annotation that indicates the purpose of the memory mapping.
308///
309/// This is for debugging.  On Linux, mmtk-core will use `prctl` with `PR_SET_VMA` to set the
310/// human-readable name for the given mmap region.  The annotation is ignored on other platforms.
311///
312/// Note that when using `Map32` (even when running on 64-bit architectures), the discontiguous
313/// memory range is shared between different spaces. Spaces may use `mmap` to map new chunks, but
314/// the same chunk may later be reused by other spaces. The annotation only applies when `mmap` is
315/// called for a chunk for the first time, which reflects which space first attempted the mmap, not
316/// which space is currently using the chunk.  Use `crate::policy::space::print_vm_map` to print a
317/// more accurate mapping between address ranges and spaces.
318///
319/// On 32-bit architecture, side metadata are allocated in a chunked fasion.  One single `mmap`
320/// region will contain many different metadata.  In that case, we simply annotate the whole region
321/// with a `MmapAnnotation::SideMeta` where `meta` is `"all"`.
322pub enum MmapAnnotation<'a> {
323    /// The mmap is for a space.
324    Space {
325        /// The name of the space.
326        name: &'a str,
327    },
328    /// The mmap is for a side metadata.
329    SideMeta {
330        /// The name of the space.
331        space: &'a str,
332        /// The name of the side metadata.
333        meta: &'a str,
334    },
335    /// The mmap is for a test case.  Usually constructed using the [`mmap_anno_test!`] macro.
336    Test {
337        /// The source file.
338        file: &'a str,
339        /// The line number.
340        line: u32,
341    },
342    /// For all other use cases.
343    Misc {
344        /// A human-readable descriptive name.
345        name: &'a str,
346    },
347}
348
349/// Construct an `MmapAnnotation::Test` with the current file name and line number.
350#[macro_export]
351macro_rules! mmap_anno_test {
352    () => {
353        &$crate::util::os::MmapAnnotation::Test {
354            file: file!(),
355            line: line!(),
356        }
357    };
358}
359
360// Export this to external crates
361pub use mmap_anno_test;
362
363impl std::fmt::Display for MmapAnnotation<'_> {
364    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
365        match self {
366            MmapAnnotation::Space { name } => write!(f, "mmtk:space:{name}"),
367            MmapAnnotation::SideMeta { space, meta } => write!(f, "mmtk:sidemeta:{space}:{meta}"),
368            MmapAnnotation::Test { file, line } => write!(f, "mmtk:test:{file}:{line}"),
369            MmapAnnotation::Misc { name } => write!(f, "mmtk:misc:{name}"),
370        }
371    }
372}
mmtk/util/os/memory.rs

mmtk/util/os/
memory.rs
