mmtk/util/metadata/side_metadata/
layout.rs

1#[cfg(target_pointer_width = "32")]
2use crate::util::heap::layout::vm_layout::VMLayout;
3#[cfg(target_pointer_width = "32")]
4use crate::util::heap::layout::vm_layout::BYTES_IN_CHUNK;
5#[cfg(target_pointer_width = "64")]
6use crate::util::metadata::side_metadata::side_metadata_offset_after;
7use crate::util::metadata::side_metadata::SideMetadataSpec;
8use crate::util::os::{HugePageSupport, MmapAnnotation};
9use crate::util::Address;
10use crate::util::{constants::LOG_BYTES_IN_PAGE, conversions::raw_align_up};
11use crate::MMAPPER;
12use std::sync::OnceLock;
13
14/// The compile-time base offset for global side metadata layout. We treat offsets as relative
15/// (starting from zero) and add the runtime base address when computing actual addresses.
16pub(crate) const GLOBAL_SIDE_METADATA_BASE_OFFSET: usize = 0;
17
18/// The run-time base address for side metadata. This is initialized at startup by mmapping necessary memory address for side metadata,
19/// and should be used as the base when computing actual side metadata addresses.
20/// We use OnceLock to ensure it is only initialized once. To eliminate the cost of accessing OnceLock after initialization, we can use get().unwrap_unchecked().
21/// TODO: use `OncLock::get_unchecked()` once it is stabilized.
22static SIDE_METADATA_BASE_ADDRESS: OnceLock<Address> = OnceLock::new();
23
24/// The upper bound for VM side metadata layout. We need to list all the VM side metadata specs, compute the upper bound, and store it here.
25static VM_SIDE_METADATA_UPPER_BOUND_OFFSET: OnceLock<usize> = OnceLock::new();
26
27// The following steps are needed before using side metadata.
28// The functions are intended as 'pub(super)'. We expect most people to use [`crate::util::metadata::side_metadata::initialize_side_metadata()`] instead, which calls these functions internally.
29
30// Step 1: Call `set_vm-side_metadata_specs()` to register VM side metadata layout. This is needed for the startup reservation to cover VM side metadata.
31
32/// Record VM side metadata layout so startup reservation can cover VM specs.
33/// This must be called before `initialize_side_metadata_base()`.
34pub(super) fn set_vm_side_metadata_specs(specs: &[SideMetadataSpec]) {
35    let mut upper_bound = 0usize;
36    for spec in specs {
37        if spec.uses_contiguous_side_metadata() {
38            upper_bound = upper_bound.max(spec.upper_bound_offset());
39        }
40    }
41    VM_SIDE_METADATA_UPPER_BOUND_OFFSET
42        .set(upper_bound)
43        .unwrap();
44    debug!(
45        "Registered VM side metadata layout: {} specs, upper_bound={}",
46        specs.len(),
47        upper_bound
48    );
49}
50
51// Step 2: Call `initialize_side_metadata_base()` to reserve address space for side metadata.
52
53/// Initialize the side metadata base address by reserving address space with quarantine mmap.
54pub(super) fn initialize_side_metadata_base(
55    specified_base: Address,
56    huge_page_support: HugePageSupport,
57) {
58    #[cfg(target_pointer_width = "64")]
59    {
60        let core_end = super::spec_defs::LAST_LOCAL_SIDE_METADATA_SPEC.upper_bound_offset();
61        let vm_end = *VM_SIDE_METADATA_UPPER_BOUND_OFFSET.get().unwrap();
62        info!(
63            "Initializing side metadata base: vm_specs_registered={} core_end={} vm_end={}",
64            VM_SIDE_METADATA_UPPER_BOUND_OFFSET.get().is_some(),
65            core_end,
66            unsafe { Address::from_usize(vm_end) }
67        );
68    }
69    let total_bytes = side_metadata_reserved_bytes();
70    let pages = total_bytes >> LOG_BYTES_IN_PAGE;
71    let anno = MmapAnnotation::SideMeta {
72        space: "all",
73        meta: "all-quarantined",
74    };
75    info!(
76        "Quarantine side metadata range: total_bytes=0x{:x}, pages=0x{:x}, granularity=0x{:x}",
77        total_bytes,
78        pages,
79        MMAPPER.granularity()
80    );
81    let base = if specified_base.is_zero() {
82        MMAPPER
83            .quarantine_address_range_anywhere(pages, huge_page_support, &anno)
84            .unwrap_or_else(|e| panic!("failed to quarantine side metadata address range: {e}"))
85    } else {
86        MMAPPER
87            .quarantine_address_range(specified_base, pages, huge_page_support, &anno)
88            .unwrap_or_else(|e| {
89                panic!(
90                    "failed to quarantine side metadata address range at {}: {e}",
91                    specified_base
92                )
93            });
94        specified_base
95    };
96    info!(
97        "Side metadata base initialized at {} (range: {} - {})",
98        base,
99        base,
100        base + total_bytes
101    );
102    SIDE_METADATA_BASE_ADDRESS.set(base).unwrap();
103}
104
105/// Tests need to check if side metadata is initialized so they can avoid attempting to initialize it again.
106#[cfg(test)]
107pub(super) fn is_side_metadata_initialized() -> bool {
108    VM_SIDE_METADATA_UPPER_BOUND_OFFSET.get().is_some()
109        && SIDE_METADATA_BASE_ADDRESS.get().is_some()
110}
111
112// With the above functions called, side metadata is initialized, and the following functions can be used to query side metadata layout and addresses.
113
114/// Get the runtime side metadata base address.
115pub fn global_side_metadata_base_address() -> Address {
116    #[cfg(debug_assertions)]
117    {
118        #[cfg(not(any(test, feature = "test_private")))]
119        {
120            assert!(
121                VM_SIDE_METADATA_UPPER_BOUND_OFFSET.get().is_some(),
122                "global_side_metadata_base_address() called before VM side metadata layout was registered"
123            );
124        }
125
126        assert!(
127            SIDE_METADATA_BASE_ADDRESS.get().is_some(),
128            "global_side_metadata_base_address() called before side metadata base was initialized"
129        );
130    }
131
132    // TODO: use `OnceLock::get_unchecked()` once it is stabilized.
133    // Otherwise, even though this uses `unwrap_unchecked`, the compiler still needs to atomically load the
134    // initialization flag inside of `get`. See https://github.com/rust-lang/libs-team/issues/654.
135    unsafe { *SIDE_METADATA_BASE_ADDRESS.get().unwrap_unchecked() }
136}
137
138// Local side metadata start address. This is derived from the end of global side metadata.
139pub(crate) fn local_side_metadata_base_address() -> Address {
140    global_side_metadata_base_address() + global_side_metadata_bytes()
141}
142
143/// Total side metadata bytes that should be reserved at startup (independent of runtime base, without alignment).
144fn total_side_metadata_bytes() -> usize {
145    let vm_end = *VM_SIDE_METADATA_UPPER_BOUND_OFFSET.get().unwrap();
146    #[cfg(target_pointer_width = "64")]
147    {
148        let core_end = super::spec_defs::LAST_LOCAL_SIDE_METADATA_SPEC.upper_bound_offset();
149        debug!(
150            "total_side_metadata_bytes(): core_end={} vm_end={} (registered={})",
151            core_end,
152            vm_end,
153            VM_SIDE_METADATA_UPPER_BOUND_OFFSET.get().is_some()
154        );
155        core_end.max(vm_end)
156    }
157    #[cfg(target_pointer_width = "32")]
158    {
159        let local_bytes =
160            1usize << (VMLayout::LOG_ARCH_ADDRESS_SPACE - LOG_LOCAL_SIDE_METADATA_WORST_CASE_RATIO);
161        global_side_metadata_bytes().max(vm_end) + local_bytes
162    }
163}
164
165/// Total side metadata bytes that should be reserved at startup (independent of runtime base, with alignment).
166pub(crate) fn side_metadata_reserved_bytes() -> usize {
167    raw_align_up(total_side_metadata_bytes(), MMAPPER.granularity())
168}
169
170/// Base address of VO bit, public to VM bindings which may need to use this.
171#[cfg(target_pointer_width = "64")]
172pub fn vo_bit_side_metadata_addr() -> Address {
173    crate::util::metadata::vo_bit::vo_bit_side_metadata_addr()
174}
175
176/// The base address for the global side metadata space available to VM bindings, to be used for the per-object metadata.
177/// VM bindings must use this to avoid overlap with core internal global side metadata.
178pub fn global_side_metadata_vm_base_address() -> Address {
179    super::spec_defs::LAST_GLOBAL_SIDE_METADATA_SPEC.upper_bound_address_for_contiguous()
180}
181
182/// Total global side metadata bytes (independent of the runtime base address).
183pub(crate) fn global_side_metadata_bytes() -> usize {
184    let end = super::spec_defs::LAST_GLOBAL_SIDE_METADATA_SPEC.upper_bound_offset();
185    end.max(*VM_SIDE_METADATA_UPPER_BOUND_OFFSET.get().unwrap())
186}
187
188// constants
189
190/// This constant represents the worst-case ratio of source data size to global side metadata.
191/// A value of 2 means the space required for global side metadata must be less than 1/4th of the source data.
192/// So, a value of `n` means this ratio must be less than $2^-n$.
193#[cfg(target_pointer_width = "32")]
194pub(super) const LOG_GLOBAL_SIDE_METADATA_WORST_CASE_RATIO: usize = 3;
195#[cfg(target_pointer_width = "64")]
196pub(super) const LOG_GLOBAL_SIDE_METADATA_WORST_CASE_RATIO: usize = 1;
197
198/// This constant represents the worst-case ratio of source data size to global+local side metadata.
199/// A value of 1 means the space required for global+local side metadata must be less than 1/2nd of the source data.
200/// So, a value of `n` means this ratio must be less than $2^-n$.
201#[cfg(target_pointer_width = "32")]
202pub(super) const LOG_LOCAL_SIDE_METADATA_WORST_CASE_RATIO: usize = 3;
203#[cfg(target_pointer_width = "64")]
204pub(super) const LOG_LOCAL_SIDE_METADATA_WORST_CASE_RATIO: usize = 1;
205
206#[cfg(target_pointer_width = "32")]
207pub(super) const LOCAL_SIDE_METADATA_PER_CHUNK: usize =
208    BYTES_IN_CHUNK >> LOG_LOCAL_SIDE_METADATA_WORST_CASE_RATIO;
209
210/// The base offset for the global side metadata available to VM bindings.
211pub const GLOBAL_SIDE_METADATA_VM_BASE_OFFSET: usize =
212    super::spec_defs::LAST_GLOBAL_SIDE_METADATA_SPEC.upper_bound_offset();
213
214/// The base address for the local side metadata space available to VM bindings, to be used for the per-object metadata.
215/// VM bindings must use this to avoid overlap with core internal local side metadata.
216pub const LOCAL_SIDE_METADATA_VM_BASE_OFFSET: usize =
217    super::spec_defs::LAST_LOCAL_SIDE_METADATA_SPEC.upper_bound_offset();
218
219#[cfg(target_pointer_width = "32")]
220pub(super) const LOCAL_SIDE_METADATA_BASE_OFFSET_FOR_LAYOUT: usize = 0;
221#[cfg(target_pointer_width = "64")]
222pub(super) const LOCAL_SIDE_METADATA_BASE_OFFSET_FOR_LAYOUT: usize =
223    side_metadata_offset_after(&super::spec_defs::LAST_GLOBAL_SIDE_METADATA_SPEC);
mmtk/util/metadata/side_metadata/layout.rs

mmtk/util/metadata/side_metadata/
layout.rs
