mmtk/policy/marksweepspace/native_ms/
global.rs

1use std::sync::{
2    atomic::{AtomicUsize, Ordering},
3    Arc,
4};
5
6use crate::{
7    policy::{marksweepspace::native_ms::*, sft::GCWorkerMutRef},
8    scheduler::{GCWorkScheduler, GCWorker, WorkBucketStage},
9    util::{
10        copy::CopySemantics,
11        epilogue,
12        heap::{BlockPageResource, PageResource},
13        metadata::{self, side_metadata::SideMetadataSpec, MetadataSpec},
14        object_enum::{self, ObjectEnumerator},
15        ObjectReference,
16    },
17    vm::{ActivePlan, VMBinding},
18};
19
20#[cfg(feature = "vo_bit")]
21use crate::util::Address;
22
23use crate::plan::tracing::{ObjectQueue, OptionObjectQueue};
24use crate::policy::sft::SFT;
25use crate::policy::space::{CommonSpace, Space};
26use crate::util::alloc::allocator::AllocationOptions;
27use crate::util::constants::LOG_BYTES_IN_PAGE;
28use crate::util::heap::chunk_map::*;
29use crate::util::linear_scan::Region;
30use crate::util::VMThread;
31use crate::vm::ObjectModel;
32use crate::vm::Scanning;
33use std::sync::Mutex;
34
35/// The result for `MarkSweepSpace.acquire_block()`. `MarkSweepSpace` will attempt
36/// to allocate from abandoned blocks first. If none found, it will get a new block
37/// from the page resource.
38pub enum BlockAcquireResult {
39    Exhausted,
40    /// A new block we just acquired from the page resource
41    Fresh(Block),
42    /// An available block. The block can be directly used if there is any free cell in it.
43    AbandonedAvailable(Block),
44    /// An unswept block. The block needs to be swept first before it can be used.
45    AbandonedUnswept(Block),
46}
47
48/// A mark sweep space.
49///
50/// The space and each free list allocator own some block lists.
51/// A block that is in use belongs to exactly one of the block lists. In this case,
52/// whoever owns a block list has exclusive access on the blocks in the list.
53/// There should be no data race to access blocks. A thread should NOT access a block list
54/// if it does not own the block list.
55///
56/// The table below roughly describes what we do in each phase.
57///
58/// | Phase          | Allocator local block lists                     | Global abandoned block lists                 | Chunk map |
59/// |----------------|-------------------------------------------------|----------------------------------------------|-----------|
60/// | Allocation     | Alloc from local                                | Move blocks from global to local block lists | -         |
61/// |                | Lazy: sweep local blocks                        |                                              |           |
62/// | GC - Prepare   | -                                               | -                                            | Find used chunks, reset block mark, bzero mark bit |
63/// | GC - Trace     | Trace object and mark blocks.                   | Trace object and mark blocks.                | -         |
64/// |                | No block list access.                           | No block list access.                        |           |
65/// | GC - Release   | Lazy: Move blocks to local unswept list         | Lazy: Move blocks to global unswept list     | _         |
66/// |                | Eager: Sweep local blocks                       | Eager: Sweep global blocks                   |           |
67/// |                | Both: Return local blocks to a temp global list |                                              |           |
68/// | GC - End of GC | -                                               | Merge the temp global lists                  | -         |
69pub struct MarkSweepSpace<VM: VMBinding> {
70    pub common: CommonSpace<VM>,
71    pr: BlockPageResource<VM, Block>,
72    /// Allocation status for all chunks in MS space
73    chunk_map: ChunkMap,
74    /// Work packet scheduler
75    scheduler: Arc<GCWorkScheduler<VM>>,
76    /// Abandoned blocks. If a mutator dies, all its blocks go to this abandoned block
77    /// lists. We reuse blocks in these lists in the mutator phase.
78    /// The space needs to do the release work for these block lists.
79    abandoned: Mutex<AbandonedBlockLists>,
80    /// Abandoned blocks during a GC. Each allocator finishes doing release work, and returns
81    /// their local blocks to the global lists. Thus we do not need to do release work for
82    /// these block lists in the space. These lists are only filled in the release phase,
83    /// and will be moved to the abandoned lists above at the end of a GC.
84    abandoned_in_gc: Mutex<AbandonedBlockLists>,
85    /// Count the number of pending `ReleaseMarkSweepSpace` and `ReleaseMutator` work packets during
86    /// the `Release` stage.
87    pending_release_packets: AtomicUsize,
88}
89
90unsafe impl<VM: VMBinding> Sync for MarkSweepSpace<VM> {}
91
92pub struct AbandonedBlockLists {
93    pub available: BlockLists,
94    pub unswept: BlockLists,
95    pub consumed: BlockLists,
96}
97
98impl AbandonedBlockLists {
99    fn new() -> Self {
100        Self {
101            available: new_empty_block_lists(),
102            unswept: new_empty_block_lists(),
103            consumed: new_empty_block_lists(),
104        }
105    }
106
107    fn sweep_later<VM: VMBinding>(&mut self, space: &MarkSweepSpace<VM>) {
108        for i in 0..MI_BIN_FULL {
109            // Release free blocks
110            self.available[i].release_blocks(space);
111            self.consumed[i].release_blocks(space);
112            if cfg!(not(feature = "eager_sweeping")) {
113                self.unswept[i].release_blocks(space);
114            } else {
115                // If we do eager sweeping, we should have no unswept blocks.
116                debug_assert!(self.unswept[i].is_empty());
117            }
118
119            // For eager sweeping, that's it.  We just release unmarked blocks, and leave marked
120            // blocks to be swept later in the `SweepChunk` work packet.
121
122            // For lazy sweeping, we move blocks from available and consumed to unswept.  When an
123            // allocator tries to use them, they will sweep the block.
124            if cfg!(not(feature = "eager_sweeping")) {
125                self.unswept[i].append(&mut self.available[i]);
126                self.unswept[i].append(&mut self.consumed[i]);
127            }
128        }
129    }
130
131    fn recycle_blocks(&mut self) {
132        for i in 0..MI_BIN_FULL {
133            for block in self.consumed[i].iter() {
134                if block.has_free_cells() {
135                    self.consumed[i].remove(block);
136                    self.available[i].push(block);
137                }
138            }
139        }
140    }
141
142    fn merge(&mut self, other: &mut Self) {
143        for i in 0..MI_BIN_FULL {
144            self.available[i].append(&mut other.available[i]);
145            self.unswept[i].append(&mut other.unswept[i]);
146            self.consumed[i].append(&mut other.consumed[i]);
147        }
148    }
149
150    #[cfg(debug_assertions)]
151    fn assert_empty(&self) {
152        for i in 0..MI_BIN_FULL {
153            assert!(self.available[i].is_empty());
154            assert!(self.unswept[i].is_empty());
155            assert!(self.consumed[i].is_empty());
156        }
157    }
158}
159
160impl<VM: VMBinding> SFT for MarkSweepSpace<VM> {
161    fn name(&self) -> &'static str {
162        self.common.name
163    }
164
165    fn is_live(&self, object: crate::util::ObjectReference) -> bool {
166        VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.is_marked::<VM>(object, Ordering::SeqCst)
167    }
168
169    #[cfg(feature = "object_pinning")]
170    fn pin_object(&self, _object: ObjectReference) -> bool {
171        false
172    }
173
174    #[cfg(feature = "object_pinning")]
175    fn unpin_object(&self, _object: ObjectReference) -> bool {
176        false
177    }
178
179    #[cfg(feature = "object_pinning")]
180    fn is_object_pinned(&self, _object: ObjectReference) -> bool {
181        false
182    }
183
184    fn is_movable(&self) -> bool {
185        false
186    }
187
188    #[cfg(feature = "sanity")]
189    fn is_sane(&self) -> bool {
190        true
191    }
192
193    fn initialize_object_metadata(&self, _object: crate::util::ObjectReference, _bytes: usize) {
194        #[cfg(feature = "vo_bit")]
195        crate::util::metadata::vo_bit::set_vo_bit(_object);
196    }
197
198    #[cfg(feature = "vo_bit")]
199    fn is_mmtk_object(&self, addr: Address) -> Option<ObjectReference> {
200        crate::util::metadata::vo_bit::is_vo_bit_set_for_addr(addr)
201    }
202
203    #[cfg(feature = "vo_bit")]
204    fn find_object_from_internal_pointer(
205        &self,
206        ptr: Address,
207        max_search_bytes: usize,
208    ) -> Option<ObjectReference> {
209        // We don't need to search more than the max object size in the mark sweep space.
210        let search_bytes = usize::min(MAX_OBJECT_SIZE, max_search_bytes);
211        crate::util::metadata::vo_bit::find_object_from_internal_pointer::<VM>(ptr, search_bytes)
212    }
213
214    fn sft_trace_object(
215        &self,
216        queue: &mut OptionObjectQueue,
217        object: ObjectReference,
218        _worker: GCWorkerMutRef,
219    ) -> ObjectReference {
220        self.trace_object(queue, object)
221    }
222}
223
224impl<VM: VMBinding> Space<VM> for MarkSweepSpace<VM> {
225    fn as_space(&self) -> &dyn Space<VM> {
226        self
227    }
228
229    fn as_sft(&self) -> &(dyn SFT + Sync + 'static) {
230        self
231    }
232
233    fn get_page_resource(&self) -> &dyn crate::util::heap::PageResource<VM> {
234        &self.pr
235    }
236
237    fn maybe_get_page_resource_mut(&mut self) -> Option<&mut dyn PageResource<VM>> {
238        Some(&mut self.pr)
239    }
240
241    fn initialize_sft(&self, sft_map: &mut dyn crate::policy::sft_map::SFTMap) {
242        self.common().initialize_sft(self.as_sft(), sft_map)
243    }
244
245    fn common(&self) -> &CommonSpace<VM> {
246        &self.common
247    }
248
249    fn release_multiple_pages(&mut self, _start: crate::util::Address) {
250        todo!()
251    }
252
253    fn enumerate_objects(&self, enumerator: &mut dyn ObjectEnumerator) {
254        object_enum::enumerate_blocks_from_chunk_map::<Block>(enumerator, &self.chunk_map);
255    }
256
257    fn clear_side_log_bits(&self) {
258        let log_bit = VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.extract_side_spec();
259        for chunk in self.chunk_map.all_chunks() {
260            log_bit.bzero_metadata(chunk.start(), Chunk::BYTES);
261        }
262    }
263
264    fn set_side_log_bits(&self) {
265        let log_bit = VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.extract_side_spec();
266        for chunk in self.chunk_map.all_chunks() {
267            log_bit.bset_metadata(chunk.start(), Chunk::BYTES);
268        }
269    }
270}
271
272impl<VM: VMBinding> crate::policy::gc_work::PolicyTraceObject<VM> for MarkSweepSpace<VM> {
273    fn trace_object<Q: ObjectQueue, const KIND: crate::policy::gc_work::TraceKind>(
274        &self,
275        queue: &mut Q,
276        object: ObjectReference,
277        _copy: Option<CopySemantics>,
278        _worker: &mut GCWorker<VM>,
279    ) -> ObjectReference {
280        self.trace_object(queue, object)
281    }
282
283    fn may_move_objects<const KIND: crate::policy::gc_work::TraceKind>() -> bool {
284        false
285    }
286}
287
288// We cannot allocate objects that are larger than the max bin size.
289#[allow(dead_code)]
290pub const MAX_OBJECT_SIZE: usize = crate::policy::marksweepspace::native_ms::MI_LARGE_OBJ_SIZE_MAX;
291
292impl<VM: VMBinding> MarkSweepSpace<VM> {
293    // Allow ptr_arg as we want to keep the function signature the same as for malloc marksweep
294    #[allow(clippy::ptr_arg)]
295    pub fn extend_global_side_metadata_specs(_specs: &mut Vec<SideMetadataSpec>) {
296        // MarkSweepSpace does not need any special global specs. This method exists, as
297        // we need this method for MallocSpace, and we want those two spaces to be used interchangably.
298    }
299
300    pub fn new(args: crate::policy::space::PlanCreateSpaceArgs<VM>) -> MarkSweepSpace<VM> {
301        let scheduler = args.scheduler.clone();
302        let vm_map = args.vm_map;
303        let is_discontiguous = args.vmrequest.is_discontiguous();
304        let local_specs = {
305            metadata::extract_side_metadata(&[
306                MetadataSpec::OnSide(Block::NEXT_BLOCK_TABLE),
307                MetadataSpec::OnSide(Block::PREV_BLOCK_TABLE),
308                MetadataSpec::OnSide(Block::FREE_LIST_TABLE),
309                MetadataSpec::OnSide(Block::SIZE_TABLE),
310                #[cfg(feature = "malloc_native_mimalloc")]
311                MetadataSpec::OnSide(Block::LOCAL_FREE_LIST_TABLE),
312                #[cfg(feature = "malloc_native_mimalloc")]
313                MetadataSpec::OnSide(Block::THREAD_FREE_LIST_TABLE),
314                MetadataSpec::OnSide(Block::BLOCK_LIST_TABLE),
315                MetadataSpec::OnSide(Block::TLS_TABLE),
316                MetadataSpec::OnSide(Block::MARK_TABLE),
317                *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC,
318            ])
319        };
320        let common = CommonSpace::new(args.into_policy_args(false, false, local_specs));
321        let space_index = common.descriptor.get_index();
322        MarkSweepSpace {
323            pr: if is_discontiguous {
324                BlockPageResource::new_discontiguous(
325                    Block::LOG_PAGES,
326                    vm_map,
327                    scheduler.num_workers(),
328                )
329            } else {
330                BlockPageResource::new_contiguous(
331                    Block::LOG_PAGES,
332                    common.start,
333                    common.extent,
334                    vm_map,
335                    scheduler.num_workers(),
336                )
337            },
338            common,
339            chunk_map: ChunkMap::new(space_index),
340            scheduler,
341            abandoned: Mutex::new(AbandonedBlockLists::new()),
342            abandoned_in_gc: Mutex::new(AbandonedBlockLists::new()),
343            pending_release_packets: AtomicUsize::new(0),
344        }
345    }
346
347    /// Mark an object non-atomically.  If multiple GC worker threads attempt to mark the same
348    /// object, more than one of them may return `true`.
349    fn attempt_mark_non_atomic(&self, object: ObjectReference) -> bool {
350        if !VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.is_marked::<VM>(object, Ordering::SeqCst) {
351            VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.mark::<VM>(object, Ordering::SeqCst);
352            true
353        } else {
354            false
355        }
356    }
357
358    /// Mark an object atomically.
359    fn attempt_mark_atomic(&self, object: ObjectReference) -> bool {
360        let mark_state = 1u8;
361
362        loop {
363            let old_value = VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.load_atomic::<VM, u8>(
364                object,
365                None,
366                Ordering::SeqCst,
367            );
368            if old_value == mark_state {
369                return false;
370            }
371
372            if VM::VMObjectModel::LOCAL_MARK_BIT_SPEC
373                .compare_exchange_metadata::<VM, u8>(
374                    object,
375                    old_value,
376                    mark_state,
377                    None,
378                    Ordering::SeqCst,
379                    Ordering::SeqCst,
380                )
381                .is_ok()
382            {
383                break;
384            }
385        }
386        true
387    }
388
389    /// Mark an object.  Return `true` if the object is newly marked.  Return `false` if the object
390    /// was already marked.
391    fn attempt_mark(&self, object: ObjectReference) -> bool {
392        if VM::VMScanning::UNIQUE_OBJECT_ENQUEUING {
393            self.attempt_mark_atomic(object)
394        } else {
395            self.attempt_mark_non_atomic(object)
396        }
397    }
398
399    fn trace_object<Q: ObjectQueue>(
400        &self,
401        queue: &mut Q,
402        object: ObjectReference,
403    ) -> ObjectReference {
404        debug_assert!(
405            self.in_space(object),
406            "Cannot mark an object {} that was not alloced by free list allocator.",
407            object,
408        );
409        if self.attempt_mark(object) {
410            let block = Block::containing(object);
411            block.set_state(BlockState::Marked);
412            queue.enqueue(object);
413        }
414        object
415    }
416
417    pub fn record_new_block(&self, block: Block) {
418        block.init();
419        self.chunk_map.set_allocated(block.chunk(), true);
420    }
421
422    pub fn prepare(&mut self, _full_heap: bool) {
423        #[cfg(debug_assertions)]
424        self.abandoned_in_gc.lock().unwrap().assert_empty();
425
426        // # Safety: MarkSweepSpace reference is always valid within this collection cycle.
427        let space = unsafe { &*(self as *const Self) };
428        let work_packets = self
429            .chunk_map
430            .generate_tasks(|chunk| Box::new(PrepareChunkMap { space, chunk }));
431        self.scheduler.work_buckets[crate::scheduler::WorkBucketStage::Prepare]
432            .bulk_add(work_packets);
433    }
434
435    pub fn release(&mut self) {
436        let num_mutators = VM::VMActivePlan::number_of_mutators();
437        // all ReleaseMutator work packets plus the ReleaseMarkSweepSpace packet
438        self.pending_release_packets
439            .store(num_mutators + 1, Ordering::SeqCst);
440
441        // Do work in separate work packet in order not to slow down the `Release` work packet which
442        // blocks all `ReleaseMutator` packets.
443        let space = unsafe { &*(self as *const Self) };
444        let work_packet = ReleaseMarkSweepSpace { space };
445        self.scheduler.work_buckets[crate::scheduler::WorkBucketStage::Release].add(work_packet);
446    }
447
448    pub fn end_of_gc(&mut self) {
449        epilogue::debug_assert_counter_zero(
450            &self.pending_release_packets,
451            "pending_release_packets",
452        );
453    }
454
455    /// Release a block.
456    pub fn release_block(&self, block: Block) {
457        self.block_clear_metadata(block);
458
459        block.deinit();
460        self.pr.release_block(block);
461    }
462
463    pub fn block_clear_metadata(&self, block: Block) {
464        for metadata_spec in Block::METADATA_SPECS {
465            metadata_spec.set_zero_atomic(block.start(), Ordering::SeqCst);
466        }
467        #[cfg(feature = "vo_bit")]
468        crate::util::metadata::vo_bit::bzero_vo_bit(block.start(), Block::BYTES);
469    }
470
471    pub fn acquire_block(
472        &self,
473        tls: VMThread,
474        size: usize,
475        align: usize,
476        alloc_options: AllocationOptions,
477    ) -> BlockAcquireResult {
478        {
479            let mut abandoned = self.abandoned.lock().unwrap();
480            let bin = mi_bin::<VM>(size, align);
481
482            {
483                let abandoned_available = &mut abandoned.available;
484                if !abandoned_available[bin].is_empty() {
485                    let block = abandoned_available[bin].pop().unwrap();
486                    return BlockAcquireResult::AbandonedAvailable(block);
487                }
488            }
489
490            {
491                let abandoned_unswept = &mut abandoned.unswept;
492                if !abandoned_unswept[bin].is_empty() {
493                    let block = abandoned_unswept[bin].pop().unwrap();
494                    return BlockAcquireResult::AbandonedUnswept(block);
495                }
496            }
497        }
498
499        let acquired = self.acquire(tls, Block::BYTES >> LOG_BYTES_IN_PAGE, alloc_options);
500        if acquired.is_zero() {
501            BlockAcquireResult::Exhausted
502        } else {
503            BlockAcquireResult::Fresh(Block::from_unaligned_address(acquired))
504        }
505    }
506
507    pub fn get_abandoned_block_lists(&self) -> &Mutex<AbandonedBlockLists> {
508        &self.abandoned
509    }
510
511    pub fn get_abandoned_block_lists_in_gc(&self) -> &Mutex<AbandonedBlockLists> {
512        &self.abandoned_in_gc
513    }
514
515    pub fn release_packet_done(&self) {
516        let old = self.pending_release_packets.fetch_sub(1, Ordering::SeqCst);
517        if old == 1 {
518            if cfg!(feature = "eager_sweeping") {
519                // When doing eager sweeping, we start sweeing now.
520                // After sweeping, we will recycle blocks.
521                let work_packets = self.generate_sweep_tasks();
522                self.scheduler.work_buckets[WorkBucketStage::Release].bulk_add(work_packets);
523            } else {
524                // When doing lazy sweeping, we recycle blocks now.
525                self.recycle_blocks();
526            }
527        }
528    }
529
530    fn generate_sweep_tasks(&self) -> Vec<Box<dyn GCWork<VM>>> {
531        let space = unsafe { &*(self as *const Self) };
532        let epilogue = Arc::new(RecycleBlocks {
533            space,
534            counter: AtomicUsize::new(0),
535        });
536        let tasks = self.chunk_map.generate_tasks(|chunk| {
537            Box::new(SweepChunk {
538                space,
539                chunk,
540                epilogue: epilogue.clone(),
541            })
542        });
543        epilogue.counter.store(tasks.len(), Ordering::SeqCst);
544        tasks
545    }
546
547    fn recycle_blocks(&self) {
548        {
549            let mut abandoned = self.abandoned.try_lock().unwrap();
550            let mut abandoned_in_gc = self.abandoned_in_gc.try_lock().unwrap();
551
552            if cfg!(feature = "eager_sweeping") {
553                // When doing eager sweeping, previously consumed blocks may become available after
554                // sweeping.  We recycle them.
555                abandoned.recycle_blocks();
556                abandoned_in_gc.recycle_blocks();
557            }
558
559            abandoned.merge(&mut abandoned_in_gc);
560
561            #[cfg(debug_assertions)]
562            abandoned_in_gc.assert_empty();
563        }
564
565        // BlockPageResource uses worker-local block queues to eliminate contention when releasing
566        // blocks, similar to how the MarkSweepSpace caches blocks in `abandoned_in_gc` before
567        // returning to the global pool.  We flush the BlockPageResource, too.
568        self.pr.flush_all();
569    }
570}
571
572use crate::scheduler::GCWork;
573use crate::MMTK;
574
575struct PrepareChunkMap<VM: VMBinding> {
576    space: &'static MarkSweepSpace<VM>,
577    chunk: Chunk,
578}
579
580impl<VM: VMBinding> GCWork<VM> for PrepareChunkMap<VM> {
581    fn do_work(&mut self, _worker: &mut GCWorker<VM>, _mmtk: &'static MMTK<VM>) {
582        debug_assert!(self.space.chunk_map.get(self.chunk).unwrap().is_allocated());
583        // number of allocated blocks.
584        let mut n_occupied_blocks = 0;
585        self.chunk
586            .iter_region::<Block>()
587            .filter(|block| block.get_state() != BlockState::Unallocated)
588            .for_each(|block| {
589                // Clear block mark
590                block.set_state(BlockState::Unmarked);
591                // Count occupied blocks
592                n_occupied_blocks += 1
593            });
594        if n_occupied_blocks == 0 {
595            // Set this chunk as free if there is no live blocks.
596            self.space.chunk_map.set_allocated(self.chunk, false)
597        } else {
598            // Otherwise this chunk is occupied, and we reset the mark bit if it is on the side.
599            if let MetadataSpec::OnSide(side) = *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC {
600                side.bzero_metadata(self.chunk.start(), Chunk::BYTES);
601            }
602        }
603    }
604}
605
606struct ReleaseMarkSweepSpace<VM: VMBinding> {
607    space: &'static MarkSweepSpace<VM>,
608}
609
610impl<VM: VMBinding> GCWork<VM> for ReleaseMarkSweepSpace<VM> {
611    fn do_work(&mut self, _worker: &mut GCWorker<VM>, _mmtk: &'static MMTK<VM>) {
612        {
613            let mut abandoned = self.space.abandoned.lock().unwrap();
614            abandoned.sweep_later(self.space);
615        }
616
617        self.space.release_packet_done();
618    }
619}
620
621/// Chunk sweeping work packet.  Only used by eager sweeping to sweep marked blocks after unmarked
622/// blocks have been released.
623struct SweepChunk<VM: VMBinding> {
624    space: &'static MarkSweepSpace<VM>,
625    chunk: Chunk,
626    /// A destructor invoked when all `SweepChunk` packets are finished.
627    epilogue: Arc<RecycleBlocks<VM>>,
628}
629
630impl<VM: VMBinding> GCWork<VM> for SweepChunk<VM> {
631    fn do_work(&mut self, _worker: &mut GCWorker<VM>, _mmtk: &'static MMTK<VM>) {
632        assert!(self.space.chunk_map.get(self.chunk).unwrap().is_allocated());
633
634        // number of allocated blocks.
635        let mut allocated_blocks = 0;
636        // Iterate over all allocated blocks in this chunk.
637        for block in self
638            .chunk
639            .iter_region::<Block>()
640            .filter(|block| block.get_state() != BlockState::Unallocated)
641        {
642            // We have released unmarked blocks in `ReleaseMarkSweepSpace` and `ReleaseMutator`.
643            // We shouldn't see any unmarked blocks now.
644            debug_assert_eq!(block.get_state(), BlockState::Marked);
645            block.sweep::<VM>();
646            allocated_blocks += 1;
647        }
648        probe!(mmtk, sweep_chunk_ms, allocated_blocks);
649        // Set this chunk as free if there is not live blocks.
650        if allocated_blocks == 0 {
651            self.space.chunk_map.set_allocated(self.chunk, false);
652        }
653        self.epilogue.finish_one_work_packet();
654    }
655}
656
657struct RecycleBlocks<VM: VMBinding> {
658    space: &'static MarkSweepSpace<VM>,
659    counter: AtomicUsize,
660}
661
662impl<VM: VMBinding> RecycleBlocks<VM> {
663    fn finish_one_work_packet(&self) {
664        if 1 == self.counter.fetch_sub(1, Ordering::SeqCst) {
665            self.space.recycle_blocks()
666        }
667    }
668}
669
670impl<VM: VMBinding> Drop for RecycleBlocks<VM> {
671    fn drop(&mut self) {
672        epilogue::debug_assert_counter_zero(&self.counter, "RecycleBlocks::counter");
673    }
674}
mmtk/policy/marksweepspace/native_ms/global.rs

mmtk/policy/marksweepspace/native_ms/
global.rs
