mmtk/util/
object_forwarding.rs

1use crate::util::copy::*;
2use crate::util::metadata::MetadataSpec;
3use crate::util::{constants, ObjectReference};
4use crate::vm::ObjectModel;
5use crate::vm::VMBinding;
6use std::sync::atomic::Ordering;
7
8const FORWARDING_NOT_TRIGGERED_YET: u8 = 0b00;
9const BEING_FORWARDED: u8 = 0b10;
10const FORWARDED: u8 = 0b11;
11const FORWARDING_MASK: u8 = 0b11;
12#[allow(unused)]
13const FORWARDING_BITS: usize = 2;
14
15// copy address mask
16#[cfg(target_pointer_width = "64")]
17const FORWARDING_POINTER_MASK: usize = 0x00ff_ffff_ffff_fff8;
18#[cfg(target_pointer_width = "32")]
19const FORWARDING_POINTER_MASK: usize = 0xffff_fffc;
20
21/// Attempt to become the worker thread who will forward the object.
22/// The successful worker will set the object forwarding bits to BEING_FORWARDED, preventing other workers from forwarding the same object.
23pub fn attempt_to_forward<VM: VMBinding>(object: ObjectReference) -> u8 {
24    loop {
25        let old_value = get_forwarding_status::<VM>(object);
26        if old_value != FORWARDING_NOT_TRIGGERED_YET
27            || VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC
28                .compare_exchange_metadata::<VM, u8>(
29                    object,
30                    old_value,
31                    BEING_FORWARDED,
32                    None,
33                    Ordering::SeqCst,
34                    Ordering::Relaxed,
35                )
36                .is_ok()
37        {
38            return old_value;
39        }
40    }
41}
42
43/// Spin-wait for the object's forwarding to become complete and then read the forwarding pointer to the new object.
44///
45/// # Arguments:
46///
47/// * `object`: the forwarded/being_forwarded object.
48/// * `forwarding_bits`: the last state of the forwarding bits before calling this function.
49///
50/// Returns a reference to the new object.
51///
52pub fn spin_and_get_forwarded_object<VM: VMBinding>(
53    object: ObjectReference,
54    forwarding_bits: u8,
55) -> ObjectReference {
56    let mut forwarding_bits = forwarding_bits;
57    while forwarding_bits == BEING_FORWARDED {
58        forwarding_bits = get_forwarding_status::<VM>(object);
59    }
60
61    if forwarding_bits == FORWARDED {
62        read_forwarding_pointer::<VM>(object)
63    } else {
64        // For some policies (such as Immix), we can have interleaving such that one thread clears
65        // the forwarding word while another thread was stuck spinning in the above loop.
66        // See: https://github.com/mmtk/mmtk-core/issues/579
67        debug_assert!(
68            forwarding_bits == FORWARDING_NOT_TRIGGERED_YET,
69            "Invalid/Corrupted forwarding word {:x} for object {}",
70            forwarding_bits,
71            object,
72        );
73        object
74    }
75}
76
77/// Copy an object and set the forwarding state.
78///
79/// The caller can use `on_after_forwarding` to set extra metadata (including VO bits, mark bits,
80/// etc.) after the object is copied, but before the forwarding state is changed to `FORWARDED`. The
81/// atomic memory operation that sets the forwarding bits to `FORWARDED` has the `SeqCst` order.  It
82/// will guarantee that if another GC worker thread that attempts to forward the same object sees
83/// the forwarding bits being `FORWARDED`, it is guaranteed to see those extra metadata set.
84///
85/// Arguments:
86///
87/// *   `object`: The object to copy.
88/// *   `semantics`: The copy semantics.
89/// *   `copy_context`: A reference ot the `CopyContext` instance of the current GC worker.
90/// *   `on_after_forwarding`: A callback function that is called after `object` is copied, but
91///     before the forwarding bits are set.  Its argument is a reference to the new copy of
92///     `object`.
93pub fn forward_object<VM: VMBinding>(
94    object: ObjectReference,
95    semantics: CopySemantics,
96    copy_context: &mut GCWorkerCopyContext<VM>,
97    on_after_forwarding: impl FnOnce(ObjectReference),
98) -> ObjectReference {
99    let new_object = VM::VMObjectModel::copy(object, semantics, copy_context);
100    on_after_forwarding(new_object);
101    if let Some(shift) = forwarding_bits_offset_in_forwarding_pointer::<VM>() {
102        VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC.store_atomic::<VM, usize>(
103            object,
104            new_object.to_raw_address().as_usize() | ((FORWARDED as usize) << shift),
105            None,
106            Ordering::SeqCst,
107        )
108    } else {
109        write_forwarding_pointer::<VM>(object, new_object);
110        VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC.store_atomic::<VM, u8>(
111            object,
112            FORWARDED,
113            None,
114            Ordering::SeqCst,
115        );
116    }
117    new_object
118}
119
120/// Return the forwarding bits for a given `ObjectReference`.
121pub fn get_forwarding_status<VM: VMBinding>(object: ObjectReference) -> u8 {
122    VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC.load_atomic::<VM, u8>(
123        object,
124        None,
125        Ordering::SeqCst,
126    )
127}
128
129pub fn is_forwarded<VM: VMBinding>(object: ObjectReference) -> bool {
130    get_forwarding_status::<VM>(object) == FORWARDED
131}
132
133fn is_being_forwarded<VM: VMBinding>(object: ObjectReference) -> bool {
134    get_forwarding_status::<VM>(object) == BEING_FORWARDED
135}
136
137pub fn is_forwarded_or_being_forwarded<VM: VMBinding>(object: ObjectReference) -> bool {
138    get_forwarding_status::<VM>(object) != FORWARDING_NOT_TRIGGERED_YET
139}
140
141pub fn state_is_forwarded_or_being_forwarded(forwarding_bits: u8) -> bool {
142    forwarding_bits != FORWARDING_NOT_TRIGGERED_YET
143}
144
145pub fn state_is_being_forwarded(forwarding_bits: u8) -> bool {
146    forwarding_bits == BEING_FORWARDED
147}
148
149/// Zero the forwarding bits of an object.
150/// This function is used on new objects.
151pub fn clear_forwarding_bits<VM: VMBinding>(object: ObjectReference) {
152    VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC.store_atomic::<VM, u8>(
153        object,
154        0,
155        None,
156        Ordering::SeqCst,
157    )
158}
159
160/// Read the forwarding pointer of an object.
161/// This function is called on forwarded/being_forwarded objects.
162pub fn read_forwarding_pointer<VM: VMBinding>(object: ObjectReference) -> ObjectReference {
163    debug_assert!(
164        is_forwarded_or_being_forwarded::<VM>(object),
165        "read_forwarding_pointer called for object {:?} that has not started forwarding!",
166        object,
167    );
168
169    // We write the forwarding poiner. We know it is an object reference.
170    unsafe {
171        // We use "unchecked" convertion becasue we guarantee the forwarding pointer we stored
172        // previously is from a valid `ObjectReference` which is never zero.
173        ObjectReference::from_raw_address_unchecked(crate::util::Address::from_usize(
174            VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC.load_atomic::<VM, usize>(
175                object,
176                Some(FORWARDING_POINTER_MASK),
177                Ordering::SeqCst,
178            ),
179        ))
180    }
181}
182
183/// Write the forwarding pointer of an object.
184/// This function is called on being_forwarded objects.
185pub fn write_forwarding_pointer<VM: VMBinding>(
186    object: ObjectReference,
187    new_object: ObjectReference,
188) {
189    debug_assert!(
190        is_being_forwarded::<VM>(object),
191        "write_forwarding_pointer called for object {:?} that is not being forwarded! Forwarding state = 0x{:x}",
192        object,
193        get_forwarding_status::<VM>(object),
194    );
195
196    trace!("write_forwarding_pointer({}, {})", object, new_object);
197    VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC.store_atomic::<VM, usize>(
198        object,
199        new_object.to_raw_address().as_usize(),
200        Some(FORWARDING_POINTER_MASK),
201        Ordering::SeqCst,
202    )
203}
204
205/// (This function is only used internal to the `util` module)
206///
207/// This function checks whether the forwarding pointer and forwarding bits can be written in the same atomic operation.
208///
209/// Returns `None` if this is not possible.
210/// Otherwise, returns `Some(shift)`, where `shift` is the left shift needed on forwarding bits.
211///
212#[cfg(target_endian = "little")]
213pub(super) fn forwarding_bits_offset_in_forwarding_pointer<VM: VMBinding>() -> Option<isize> {
214    use std::ops::Deref;
215    // if both forwarding bits and forwarding pointer are in-header
216    match (
217        VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC.deref(),
218        VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC.deref(),
219    ) {
220        (MetadataSpec::InHeader(fp), MetadataSpec::InHeader(fb)) => {
221            let maybe_shift = fb.bit_offset - fp.bit_offset;
222            if maybe_shift >= 0 && maybe_shift < constants::BITS_IN_WORD as isize {
223                Some(maybe_shift)
224            } else {
225                None
226            }
227        }
228        _ => None,
229    }
230}
231
232#[cfg(target_endian = "big")]
233pub(super) fn forwarding_bits_offset_in_forwarding_pointer<VM: VMBinding>() -> Option<isize> {
234    unimplemented!()
235}
236
237pub(crate) fn debug_print_object_forwarding_info<VM: VMBinding>(object: ObjectReference) {
238    let forwarding_bits = get_forwarding_status::<VM>(object);
239    println!(
240        "forwarding bits = {:?}, forwarding pointer = {:?}",
241        forwarding_bits,
242        if state_is_forwarded_or_being_forwarded(forwarding_bits) {
243            Some(read_forwarding_pointer::<VM>(object))
244        } else {
245            None
246        }
247    )
248}
mmtk/util/object_forwarding.rs

mmtk/util/
object_forwarding.rs
