mmtk/util/os/
memory.rs

1use bytemuck::NoUninit;
2use std::io::Result;
3
4use crate::util::os::*;
5use crate::vm::*;
6use crate::{
7    util::{address::Address, VMThread},
8    vm::VMBinding,
9};
10
11/// Error returned by mmap-related operations in MMTk.
12#[derive(Debug)]
13pub struct MmapError {
14    /// The start address of the mmap operation that failed.
15    pub error_address: Address,
16    /// The size (in bytes) of the mmap operation that failed.
17    pub bytes: usize,
18    /// Human-readable annotation for the mmap operation.
19    ///
20    /// This is derived from [`MmapAnnotation`] at the call site.
21    pub annotation: String,
22    /// The underlying OS I/O error.
23    pub error: std::io::Error,
24}
25
26impl MmapError {
27    /// Create a new [`MmapError`].
28    pub fn new(
29        error_address: Address,
30        bytes: usize,
31        annotation: &MmapAnnotation<'_>,
32        error: std::io::Error,
33    ) -> Self {
34        Self {
35            error_address,
36            bytes,
37            annotation: annotation.to_string(),
38            error,
39        }
40    }
41}
42
43impl std::fmt::Display for MmapError {
44    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
45        write!(
46            f,
47            "mmap {} (size {}, annotation {}) failed: {}",
48            self.error_address, self.bytes, self.annotation, self.error
49        )
50    }
51}
52
53impl std::error::Error for MmapError {
54    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
55        Some(&self.error)
56    }
57}
58
59/// Result type for mmap operations that can return [`MmapError`].
60pub type MmapResult<T> = std::result::Result<T, MmapError>;
61
62/// Abstraction for OS memory operations.
63pub trait OSMemory {
64    /// Perform a demand-zero mmap.
65    ///
66    /// Fallback: `annotation` is only used for debugging. For platforms that do not support mmap annotations, this parameter can be ignored.
67    /// Fallback: see [`crate::util::os::MmapStrategy`] for fallbacks for `strategy`.
68    fn dzmmap(
69        start: Address,
70        size: usize,
71        strategy: MmapStrategy,
72        annotation: &MmapAnnotation<'_>,
73    ) -> MmapResult<Address>;
74
75    /// Handle mmap errors, possibly by signaling the VM about an out-of-memory condition.
76    fn handle_mmap_error<VM: VMBinding>(mmap_error: MmapError, tls: VMThread) {
77        use crate::util::alloc::AllocationError;
78        use std::io::ErrorKind;
79
80        eprintln!(
81            "Failed to mmap from {} to {} (size {}), annotation {}",
82            mmap_error.error_address,
83            mmap_error.error_address.wrapping_add(mmap_error.bytes),
84            mmap_error.bytes,
85            mmap_error.annotation
86        );
87        eprintln!("{}", OS::get_process_memory_maps().unwrap());
88
89        let call_binding_oom = || {
90            // Signal `MmapOutOfMemory`. Expect the VM to abort immediately.
91            trace!("Signal MmapOutOfMemory!");
92            VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory);
93            unreachable!()
94        };
95
96        match mmap_error.error.kind() {
97            // From Rust nightly 2021-05-12, we started to see Rust added this ErrorKind.
98            ErrorKind::OutOfMemory => {
99                call_binding_oom();
100            }
101            // Before Rust had ErrorKind::OutOfMemory, this is how we capture OOM from OS calls.
102            // TODO: We may be able to remove this now.
103            ErrorKind::Other => {
104                // further check the error
105                if let Some(os_errno) = mmap_error.error.raw_os_error() {
106                    if OS::is_mmap_oom(os_errno) {
107                        call_binding_oom();
108                    }
109                }
110            }
111            ErrorKind::AlreadyExists => {
112                panic!("Failed to mmap, the address is already mapped. Should MMTk quarantine the address range first?");
113            }
114            _ => {
115                if let Some(os_errno) = mmap_error.error.raw_os_error() {
116                    if OS::is_mmap_oom(os_errno) {
117                        call_binding_oom();
118                    }
119                }
120            }
121        }
122        panic!("Unexpected mmap failure: {:?}", mmap_error.error)
123    }
124
125    /// Check whether the given OS error number indicates an out-of-memory condition.
126    fn is_mmap_oom(os_errno: i32) -> bool;
127
128    /// Unmap a memory region.
129    fn munmap(start: Address, size: usize) -> Result<()>;
130
131    /// Change the protection of a memory region to the specified protection.
132    fn set_memory_access(start: Address, size: usize, prot: MmapProtection) -> Result<()>;
133
134    /// Checks if the memory has already been mapped. If not, we panic.
135    ///
136    /// Note that the checking may have a side effect that it will map the memory if it was unmapped. So we panic if it was unmapped.
137    /// Be very careful about using this function.
138    ///
139    /// Fallback: As the function is only used for assertions, it can be a no-op, and MMTk will still run and never panics in this function.
140    fn panic_if_unmapped(start: Address, size: usize);
141
142    /// Get the total memory of the system in bytes.
143    fn get_system_total_memory() -> Result<u64> {
144        use sysinfo::MemoryRefreshKind;
145        use sysinfo::{RefreshKind, System};
146
147        // TODO: Note that if we want to get system info somewhere else in the future, we should
148        // refactor this instance into some global struct. sysinfo recommends sharing one instance of
149        // `System` instead of making multiple instances.
150        // See https://docs.rs/sysinfo/0.29.0/sysinfo/index.html#usage for more info
151        //
152        // If we refactor the `System` instance to use it for other purposes, please make sure start-up
153        // time is not affected.  It takes a long time to load all components in sysinfo (e.g. by using
154        // `System::new_all()`).  Some applications, especially short-running scripts, are sensitive to
155        // start-up time.  During start-up, MMTk core only needs the total memory to initialize the
156        // `Options`.  If we only load memory-related components on start-up, it should only take <1ms
157        // to initialize the `System` instance.
158        let sys = System::new_with_specifics(
159            RefreshKind::nothing().with_memory(MemoryRefreshKind::nothing().with_ram()),
160        );
161        Ok(sys.total_memory())
162    }
163}
164
165/// Strategy for performing mmap
166#[derive(Debug, Copy, Clone)]
167pub struct MmapStrategy {
168    /// Whether we should use huge page for this mmapping.
169    /// Fallback: for platforms that do not support huge pages, this option can be ignored.
170    pub huge_page: HugePageSupport,
171    /// The protection flags for mmap.
172    pub prot: MmapProtection,
173    /// Whether this mmap allows replacing existing mappings.
174    /// Fallback: for platforms that cannot replace existing mappings, or always replace existing mappings, this option can be ignored.
175    pub replace: bool,
176    /// Whether this mmap allows reserve/commit physical memory.
177    /// This has to be implemented properly for a platform. Otherwise, we will see huge unrealistic memory consumption.
178    pub reserve: bool,
179}
180
181impl std::default::Default for MmapStrategy {
182    fn default() -> Self {
183        Self {
184            huge_page: HugePageSupport::No,
185            prot: MmapProtection::ReadWrite,
186            replace: false,
187            reserve: true,
188        }
189    }
190}
191
192impl MmapStrategy {
193    /// Create a new strategy
194    pub fn new(
195        huge_page: HugePageSupport,
196        prot: MmapProtection,
197        replace: bool,
198        reserve: bool,
199    ) -> Self {
200        Self {
201            huge_page,
202            prot,
203            replace,
204            reserve,
205        }
206    }
207
208    // Builder methods
209
210    /// Set huge page option.
211    pub fn huge_page(self, huge_page: HugePageSupport) -> Self {
212        Self { huge_page, ..self }
213    }
214
215    /// Set huge page option by a boolean flag.
216    pub fn transparent_hugepages(self, enable: bool) -> Self {
217        let huge_page = if enable {
218            HugePageSupport::TransparentHugePages
219        } else {
220            HugePageSupport::No
221        };
222        Self { huge_page, ..self }
223    }
224
225    /// Set protection option.
226    pub fn prot(self, prot: MmapProtection) -> Self {
227        Self { prot, ..self }
228    }
229
230    /// Set the replace flag.
231    pub fn replace(self, replace: bool) -> Self {
232        Self { replace, ..self }
233    }
234
235    /// Set the reserve flag.
236    pub fn reserve(self, reserve: bool) -> Self {
237        Self { reserve, ..self }
238    }
239
240    #[cfg(test)] // In test mode, we use test settings which allows replacing existing mappings.
241    /// The strategy for MMTk's own internal memory (test)
242    pub const INTERNAL_MEMORY: Self = Self::TEST;
243    #[cfg(not(test))]
244    /// The strategy for MMTk's own internal memory
245    pub const INTERNAL_MEMORY: Self = Self {
246        huge_page: HugePageSupport::No,
247        prot: MmapProtection::ReadWrite,
248        replace: false,
249        reserve: true,
250    };
251
252    /// The strategy for MMTk's test memory
253    #[cfg(test)]
254    pub const TEST: Self = Self {
255        huge_page: HugePageSupport::No,
256        prot: MmapProtection::ReadWrite,
257        replace: true,
258        reserve: true,
259    };
260}
261
262/// The protection flags for Mmap
263#[repr(i32)]
264#[derive(Debug, Copy, Clone)]
265pub enum MmapProtection {
266    /// Allow read + write
267    ReadWrite,
268    /// Allow read + write + code execution
269    ReadWriteExec,
270    /// Do not allow any access
271    NoAccess,
272}
273
274/// Support for huge pages
275#[repr(u8)]
276#[derive(Debug, Copy, Clone, NoUninit)]
277pub enum HugePageSupport {
278    /// No support for huge page
279    No,
280    /// Enable transparent huge pages for the pages that are mapped. This option is only for linux.
281    TransparentHugePages,
282}
283
284/// Annotation for an mmap entry.
285///
286/// Invocations of `mmap_fixed` and other functions that may transitively call `mmap_fixed`
287/// require an annotation that indicates the purpose of the memory mapping.
288///
289/// This is for debugging.  On Linux, mmtk-core will use `prctl` with `PR_SET_VMA` to set the
290/// human-readable name for the given mmap region.  The annotation is ignored on other platforms.
291///
292/// Note that when using `Map32` (even when running on 64-bit architectures), the discontiguous
293/// memory range is shared between different spaces. Spaces may use `mmap` to map new chunks, but
294/// the same chunk may later be reused by other spaces. The annotation only applies when `mmap` is
295/// called for a chunk for the first time, which reflects which space first attempted the mmap, not
296/// which space is currently using the chunk.  Use `crate::policy::space::print_vm_map` to print a
297/// more accurate mapping between address ranges and spaces.
298///
299/// On 32-bit architecture, side metadata are allocated in a chunked fasion.  One single `mmap`
300/// region will contain many different metadata.  In that case, we simply annotate the whole region
301/// with a `MmapAnnotation::SideMeta` where `meta` is `"all"`.
302pub enum MmapAnnotation<'a> {
303    /// The mmap is for a space.
304    Space {
305        /// The name of the space.
306        name: &'a str,
307    },
308    /// The mmap is for a side metadata.
309    SideMeta {
310        /// The name of the space.
311        space: &'a str,
312        /// The name of the side metadata.
313        meta: &'a str,
314    },
315    /// The mmap is for a test case.  Usually constructed using the [`mmap_anno_test!`] macro.
316    Test {
317        /// The source file.
318        file: &'a str,
319        /// The line number.
320        line: u32,
321    },
322    /// For all other use cases.
323    Misc {
324        /// A human-readable descriptive name.
325        name: &'a str,
326    },
327}
328
329/// Construct an `MmapAnnotation::Test` with the current file name and line number.
330#[macro_export]
331macro_rules! mmap_anno_test {
332    () => {
333        &$crate::util::os::MmapAnnotation::Test {
334            file: file!(),
335            line: line!(),
336        }
337    };
338}
339
340// Export this to external crates
341pub use mmap_anno_test;
342
343impl std::fmt::Display for MmapAnnotation<'_> {
344    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
345        match self {
346            MmapAnnotation::Space { name } => write!(f, "mmtk:space:{name}"),
347            MmapAnnotation::SideMeta { space, meta } => write!(f, "mmtk:sidemeta:{space}:{meta}"),
348            MmapAnnotation::Test { file, line } => write!(f, "mmtk:test:{file}:{line}"),
349            MmapAnnotation::Misc { name } => write!(f, "mmtk:misc:{name}"),
350        }
351    }
352}
mmtk/util/os/memory.rs

mmtk/util/os/
memory.rs
