1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
use super::{Block, BlockState};
use crate::util::alloc::allocator;
use crate::util::linear_scan::Region;
use crate::vm::VMBinding;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;

/// List of blocks owned by the allocator
#[repr(C)]
pub struct BlockList {
    pub first: Option<Block>,
    pub last: Option<Block>,
    pub size: usize,
    pub lock: AtomicBool,
}

impl std::fmt::Debug for BlockList {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(f, "BlockList {:?}", self.iter().collect::<Vec<Block>>())
    }
}

impl BlockList {
    const fn new(size: usize) -> BlockList {
        BlockList {
            first: None,
            last: None,
            size,
            lock: AtomicBool::new(false),
        }
    }

    /// List has no blocks
    pub fn is_empty(&self) -> bool {
        self.first.is_none()
    }

    /// Remove a block from the list
    pub fn remove(&mut self, block: Block) {
        match (block.load_prev_block(), block.load_next_block()) {
            (None, None) => {
                self.first = None;
                self.last = None;
            }
            (None, Some(next)) => {
                next.clear_prev_block();
                self.first = Some(next);
                next.store_block_list(self);
            }
            (Some(prev), None) => {
                prev.clear_next_block();
                self.last = Some(prev);
                prev.store_block_list(self);
            }
            (Some(prev), Some(next)) => {
                prev.store_next_block(next);
                next.store_prev_block(prev);
            }
        }
    }

    /// Pop the first block in the list
    pub fn pop(&mut self) -> Option<Block> {
        if let Some(head) = self.first {
            if let Some(next) = head.load_next_block() {
                self.first = Some(next);
                next.clear_prev_block();
                next.store_block_list(self);
            } else {
                self.first = None;
                self.last = None;
            }
            head.clear_next_block();
            head.clear_prev_block();
            Some(head)
        } else {
            None
        }
    }

    /// Push block to the front of the list
    pub fn push(&mut self, block: Block) {
        if self.is_empty() {
            block.clear_next_block();
            block.clear_prev_block();
            self.first = Some(block);
            self.last = Some(block);
        } else {
            let self_head = self.first.unwrap();
            block.store_next_block(self_head);
            self_head.store_prev_block(block);
            block.clear_prev_block();
            self.first = Some(block);
        }
        block.store_block_list(self);
    }

    /// Moves all the blocks of `other` into `self`, leaving `other` empty.
    pub fn append(&mut self, other: &mut BlockList) {
        debug_assert_eq!(self.size, other.size);
        if !other.is_empty() {
            debug_assert!(
                other.first.unwrap().load_prev_block().is_none(),
                "The other list's head has prev block: prev{} -> head{}",
                other.first.unwrap().load_prev_block().unwrap().start(),
                other.first.unwrap().start()
            );
            if self.is_empty() {
                self.first = other.first;
                self.last = other.last;
            } else {
                debug_assert!(
                    self.first.unwrap().load_prev_block().is_none(),
                    "Current list's head has prev block: prev{} -> head{}",
                    self.first.unwrap().load_prev_block().unwrap().start(),
                    self.first.unwrap().start()
                );
                let self_tail = self.last.unwrap();
                let other_head = other.first.unwrap();
                self_tail.store_next_block(other_head);
                other_head.store_prev_block(self_tail);
                self.last = other.last;
            }
            let mut cursor = other.first;
            while let Some(block) = cursor {
                block.store_block_list(self);
                cursor = block.load_next_block();
            }
            other.reset();
        }
    }

    /// Remove all blocks
    fn reset(&mut self) {
        self.first = None;
        self.last = None;
    }

    /// Lock the list. The MiMalloc allocator mostly uses thread-local block lists, and those operations on the list
    /// do not need synchronisation. However, in cases where a block list may be accessed by multiple threads, we need
    /// to lock the list before accessing it.
    ///
    /// Our current sole use for locking is parallel sweeping. During the Release phase, multiple GC worker threads can
    /// sweep chunks and release mutators at the same time, and the same `BlockList` can be reached by traversing blocks in a chunk,
    /// and also by traversing blocks held by a mutator.  This lock is necessary to prevent
    /// multiple GC workers from mutating the same `BlockList` instance.
    pub fn lock(&mut self) {
        let mut success = false;
        while !success {
            success = self
                .lock
                .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
                .is_ok();
        }
    }

    /// Unlock list. See the comments on the lock method.
    pub fn unlock(&mut self) {
        self.lock.store(false, Ordering::SeqCst);
    }

    /// Get an iterator for the block list.
    pub fn iter(&self) -> BlockListIterator {
        BlockListIterator { cursor: self.first }
    }

    /// Release unmarked blocks, and sweep other blocks in the block list. Used by eager sweeping.
    pub fn release_and_sweep_blocks<VM: VMBinding>(&self, space: &super::MarkSweepSpace<VM>) {
        for block in self.iter() {
            // We should not have unallocated blocks in a block list
            debug_assert_ne!(block.get_state(), BlockState::Unallocated);
            if !block.attempt_release(space) {
                block.sweep::<VM>();
            }
        }
    }

    /// Release unmarked blocks, and do not sweep any blocks. Used by lazy sweeping
    pub fn release_blocks<VM: VMBinding>(&self, space: &super::MarkSweepSpace<VM>) {
        for block in self.iter() {
            // We should not have unallocated blocks in a block list
            debug_assert_ne!(block.get_state(), BlockState::Unallocated);
            block.attempt_release(space);
        }
    }
}

pub struct BlockListIterator {
    cursor: Option<Block>,
}

impl Iterator for BlockListIterator {
    type Item = Block;

    fn next(&mut self) -> Option<Self::Item> {
        let ret = self.cursor;
        if let Some(cur) = self.cursor {
            self.cursor = cur.load_next_block();
        }
        ret
    }
}

/// Log2 of pointer size
const MI_INTPTR_SHIFT: usize = crate::util::constants::LOG_BYTES_IN_ADDRESS as usize;
/// pointer size in bytes
const MI_INTPTR_SIZE: usize = 1 << MI_INTPTR_SHIFT;
/// pointer size in bits
const MI_INTPTR_BITS: usize = MI_INTPTR_SIZE * 8;
/// Number of bins in BlockLists. Reserve bin0 as an empty bin.
pub(crate) const MI_BIN_FULL: usize = MAX_BIN + 1;
/// The largest valid bin.
pub(crate) const MAX_BIN: usize = 48;

/// Largest object size allowed with our mimalloc implementation, in bytes
pub(crate) const MI_LARGE_OBJ_SIZE_MAX: usize =
    crate::util::rust_util::min_of_usize(Block::BYTES, MAX_BIN_SIZE);
/// Largest object size in words
const MI_LARGE_OBJ_WSIZE_MAX: usize = MI_LARGE_OBJ_SIZE_MAX / MI_INTPTR_SIZE;
/// The object size for the last bin. We should not try allocate objects larger than this with the allocator.
pub(crate) const MAX_BIN_SIZE: usize = 8192 * MI_INTPTR_SIZE;

/// All the bins for the block lists
// Each block list takes roughly 8bytes * 4 * 49 = 1658 bytes. It is more reasonable to heap allocate them, and
// just put them behind a boxed pointer.
pub type BlockLists = Box<[BlockList; MAX_BIN + 1]>;

/// Create an empty set of block lists of different size classes (bins)
pub(crate) fn new_empty_block_lists() -> BlockLists {
    let ret = Box::new([
        BlockList::new(MI_INTPTR_SIZE),
        BlockList::new(MI_INTPTR_SIZE),
        BlockList::new(2 * MI_INTPTR_SIZE),
        BlockList::new(3 * MI_INTPTR_SIZE),
        BlockList::new(4 * MI_INTPTR_SIZE),
        BlockList::new(5 * MI_INTPTR_SIZE),
        BlockList::new(6 * MI_INTPTR_SIZE),
        BlockList::new(7 * MI_INTPTR_SIZE),
        BlockList::new(8 * MI_INTPTR_SIZE), /* 8 */
        BlockList::new(10 * MI_INTPTR_SIZE),
        BlockList::new(12 * MI_INTPTR_SIZE),
        BlockList::new(14 * MI_INTPTR_SIZE),
        BlockList::new(16 * MI_INTPTR_SIZE),
        BlockList::new(20 * MI_INTPTR_SIZE),
        BlockList::new(24 * MI_INTPTR_SIZE),
        BlockList::new(28 * MI_INTPTR_SIZE),
        BlockList::new(32 * MI_INTPTR_SIZE), /* 16 */
        BlockList::new(40 * MI_INTPTR_SIZE),
        BlockList::new(48 * MI_INTPTR_SIZE),
        BlockList::new(56 * MI_INTPTR_SIZE),
        BlockList::new(64 * MI_INTPTR_SIZE),
        BlockList::new(80 * MI_INTPTR_SIZE),
        BlockList::new(96 * MI_INTPTR_SIZE),
        BlockList::new(112 * MI_INTPTR_SIZE),
        BlockList::new(128 * MI_INTPTR_SIZE), /* 24 */
        BlockList::new(160 * MI_INTPTR_SIZE),
        BlockList::new(192 * MI_INTPTR_SIZE),
        BlockList::new(224 * MI_INTPTR_SIZE),
        BlockList::new(256 * MI_INTPTR_SIZE),
        BlockList::new(320 * MI_INTPTR_SIZE),
        BlockList::new(384 * MI_INTPTR_SIZE),
        BlockList::new(448 * MI_INTPTR_SIZE),
        BlockList::new(512 * MI_INTPTR_SIZE), /* 32 */
        BlockList::new(640 * MI_INTPTR_SIZE),
        BlockList::new(768 * MI_INTPTR_SIZE),
        BlockList::new(896 * MI_INTPTR_SIZE),
        BlockList::new(1024 * MI_INTPTR_SIZE),
        BlockList::new(1280 * MI_INTPTR_SIZE),
        BlockList::new(1536 * MI_INTPTR_SIZE),
        BlockList::new(1792 * MI_INTPTR_SIZE),
        BlockList::new(2048 * MI_INTPTR_SIZE), /* 40 */
        BlockList::new(2560 * MI_INTPTR_SIZE),
        BlockList::new(3072 * MI_INTPTR_SIZE),
        BlockList::new(3584 * MI_INTPTR_SIZE),
        BlockList::new(4096 * MI_INTPTR_SIZE),
        BlockList::new(5120 * MI_INTPTR_SIZE),
        BlockList::new(6144 * MI_INTPTR_SIZE),
        BlockList::new(7168 * MI_INTPTR_SIZE),
        BlockList::new(8192 * MI_INTPTR_SIZE), /* 48 */
    ]);

    debug_assert_eq!(
        ret[MAX_BIN].size, MAX_BIN_SIZE,
        "MAX_BIN_SIZE = {}, actual max bin size  = {}, please update the constants",
        MAX_BIN_SIZE, ret[MAX_BIN].size
    );

    ret
}

/// Returns how many pages the block lists uses.
#[allow(unused)]
pub(crate) fn pages_used_by_blocklists(lists: &BlockLists) -> usize {
    let mut pages = 0;
    for bin in 1..=MAX_BIN {
        let list = &lists[bin];

        // walk the blocks
        let mut cursor = list.first;
        while let Some(block) = cursor {
            pages += Block::BYTES >> crate::util::constants::LOG_BYTES_IN_PAGE;
            cursor = block.load_next_block();
        }
    }

    pages
}

/// Align a byte size to a size in machine words
/// i.e. byte size == `wsize*sizeof(void*)`
/// adapted from _mi_wsize_from_size in mimalloc
fn mi_wsize_from_size(size: usize) -> usize {
    (size + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE
}

pub fn mi_bin<VM: VMBinding>(size: usize, align: usize) -> usize {
    let size = allocator::get_maximum_aligned_size::<VM>(size, align);
    mi_bin_from_size(size)
}

fn mi_bin_from_size(size: usize) -> usize {
    // adapted from _mi_bin in mimalloc
    let mut wsize: usize = mi_wsize_from_size(size);
    debug_assert!(wsize <= MI_LARGE_OBJ_WSIZE_MAX);
    let bin: u8;
    if wsize <= 1 {
        bin = 1;
    } else if wsize <= 8 {
        bin = wsize as u8;
        // bin = ((wsize + 1) & !1) as u8; // round to double word sizes
    } else {
        wsize -= 1;
        let b = (MI_INTPTR_BITS - 1 - usize::leading_zeros(wsize) as usize) as u8; // note: wsize != 0
        bin = ((b << 2) + ((wsize >> (b - 2)) & 0x03) as u8) - 3;
    }
    bin as usize
}

#[cfg(test)]
mod tests {
    use super::*;

    fn get_bin_size_range(bin: usize, bins: &BlockLists) -> Option<(usize, usize)> {
        if bin == 0 || bin > MAX_BIN {
            None
        } else if bin == 1 {
            Some((0, bins[1].size))
        } else {
            Some((bins[bin - 1].size, bins[bin].size))
        }
    }

    #[test]
    fn test_mi_bin() {
        let block_lists = new_empty_block_lists();
        for size in 0..=MAX_BIN_SIZE {
            let bin = mi_bin_from_size(size);
            let bin_range = get_bin_size_range(bin, &block_lists);
            assert!(bin_range.is_some(), "Invalid bin {} for size {}", bin, size);
            assert!(
                size >= bin_range.unwrap().0 && bin < bin_range.unwrap().1,
                "Assigning size={} to bin={} ({:?}) incorrect",
                size,
                bin,
                bin_range.unwrap()
            );
        }
    }
}