Use a translated bitmap to enable more efficient marking

perf-delta.nix

@@ -36,7 +36,7 @@ let
         do
           # ignore all errors
           echo -n $file
-          if timeout 10s moc $file -no-check-ir -ref-system-api -o $file.wasm 2>/dev/null
+          if timeout 10s moc $file --force-gc --compacting-gc -no-check-ir -ref-system-api -o $file.wasm 2>/dev/null
           then echo " failed (ignored)"
           else echo " ok"
           fi

rts/motoko-rts-tests/src/bitmap.rs

@@ -2,7 +2,7 @@ use crate::memory::TestMemory;
 
 use motoko_rts::constants::WORD_SIZE;
 use motoko_rts::gc::mark_compact::bitmap::{
-    alloc_bitmap, get_bit, iter_bits, set_bit, BITMAP_ITER_END,
+    alloc_bitmap, free_bitmap, get_bit, iter_bits, set_bit, BITMAP_ITER_END,
 };
 use motoko_rts::memory::Memory;
 use motoko_rts::types::{Bytes, Words};
@@ -69,6 +69,7 @@ fn test_set_get<M: Memory>(mem: &mut M, mut bits: Vec<u16>) -> Result<(), String
         alloc_bitmap(
             mem,
             Bytes((u32::from(*bits.iter().max().unwrap()) + 1) * WORD_SIZE),
+            0,
         );
 
         for bit in &bits {
@@ -98,6 +99,8 @@ fn test_set_get<M: Memory>(mem: &mut M, mut bits: Vec<u16>) -> Result<(), String
 
             last_bit = Some(bit);
         }
+
+        free_bitmap()
     }
 
     Ok(())
@@ -111,7 +114,7 @@ fn test_bit_iter<M: Memory>(mem: &mut M, bits: HashSet<u16>) -> TestCaseResult {
     .to_bytes();
 
     unsafe {
-        alloc_bitmap(mem, heap_size);
+        alloc_bitmap(mem, heap_size, 0);
 
         for bit in bits.iter() {
             set_bit(u32::from(*bit));
@@ -154,6 +157,8 @@ fn test_bit_iter<M: Memory>(mem: &mut M, bits: HashSet<u16>) -> TestCaseResult {
                 .into(),
             ));
         }
+
+        free_bitmap()
     }
 
     Ok(())

rts/motoko-rts-tests/src/gc/heap.rs

@@ -204,11 +204,23 @@ impl MotokoHeapInner {
             map.len(),
         );
 
-        let mut heap: Vec<u8> = vec![0; heap_size];
+        // The Worst-case unalignment w.r.t. 32-byte alignment is 28 (assuming
+        // that we have general word alignment). So we over-allocate 28 bytes.
+        let mut heap: Vec<u8> = vec![0; heap_size + 28];
+
+        // MarkCompact assumes that the dynamic heap starts at a 32-byte multiple
+        let realign = match gc {
+            GC::Copying => 0,
+            GC::MarkCompact => (32 - (heap.as_ptr() as usize + static_heap_size_bytes) % 32) % 32,
+        };
+        assert_eq!(realign % 4, 0);
 
         // Maps `ObjectIdx`s into their offsets in the heap
-        let object_addrs: FxHashMap<ObjectIdx, usize> =
-            create_dynamic_heap(map, continuation_table, &mut heap[static_heap_size_bytes..]);
+        let object_addrs: FxHashMap<ObjectIdx, usize> = create_dynamic_heap(
+            map,
+            continuation_table,
+            &mut heap[static_heap_size_bytes + realign..heap_size + realign],
+        );
 
         // Closure table pointer is the last word in static heap
         let continuation_table_ptr_offset = static_heap_size_bytes - WORD_SIZE;
@@ -217,15 +229,15 @@ impl MotokoHeapInner {
             &object_addrs,
             continuation_table_ptr_offset,
             static_heap_size_bytes + dynamic_heap_size_without_continuation_table_bytes,
-            &mut heap[..static_heap_size_bytes],
+            &mut heap[realign..static_heap_size_bytes + realign],
         );
 
         MotokoHeapInner {
             heap: heap.into_boxed_slice(),
-            heap_base_offset: static_heap_size_bytes,
-            heap_ptr_offset: total_heap_size_bytes,
-            static_root_array_offset: 0,
-            continuation_table_ptr_offset: continuation_table_ptr_offset,
+            heap_base_offset: static_heap_size_bytes + realign,
+            heap_ptr_offset: total_heap_size_bytes + realign,
+            static_root_array_offset: realign,
+            continuation_table_ptr_offset: continuation_table_ptr_offset + realign,
         }
     }
 

rts/motoko-rts/src/debug.rs

@@ -31,21 +31,21 @@ pub unsafe fn dump_heap(
     print_heap(heap_base, hp);
 }
 
-pub(crate) unsafe fn print_continuation_table(closure_tbl_loc: *mut Value) {
+pub(crate) unsafe fn print_continuation_table(continuation_tbl_loc: *mut Value) {
     if !crate::continuation_table::table_initialized() {
         println!(100, "Continuation table not initialized");
         return;
     }
 
-    let arr = (*closure_tbl_loc).as_array() as *mut Array;
+    let arr = (*continuation_tbl_loc).as_array();
     let len = (*arr).len;
 
     if len == 0 {
-        println!(50, "Closure table empty");
+        println!(50, "Continuation table empty");
         return;
     }
 
-    println!(50, "Closure table: {}", len);
+    println!(50, "Continuation table: {}", len);
 
     let mut buf = [0u8; 1000];
     let mut write_buf = WriteBuf::new(&mut buf);
@@ -59,7 +59,7 @@ pub(crate) unsafe fn print_continuation_table(closure_tbl_loc: *mut Value) {
             write_buf.reset();
         }
     }
-    println!(50, "End of closure table");
+    println!(50, "End of continuation table");
 }
 
 pub(crate) unsafe fn print_static_roots(static_roots: Value) {

rts/motoko-rts/src/gc/mark_compact.rs

@@ -38,7 +38,7 @@ unsafe fn compacting_gc<M: Memory>(mem: &mut M) {
 
     compacting_gc_internal(
         mem,
-        ic::get_heap_base(),
+        ic::get_aligned_heap_base(),
         // get_hp
         || ic::HP as usize,
         // set_hp
@@ -72,6 +72,8 @@ pub unsafe fn compacting_gc_internal<
 ) {
     let old_hp = get_hp() as u32;
 
+    assert_eq!(heap_base % 32, 0);
+
     mark_compact(
         mem,
         set_hp,
@@ -98,16 +100,16 @@ unsafe fn mark_compact<M: Memory, SetHp: Fn(u32)>(
 ) {
     let mem_size = Bytes(heap_end - heap_base);
 
-    alloc_bitmap(mem, mem_size);
+    alloc_bitmap(mem, mem_size, heap_base / WORD_SIZE);
     alloc_mark_stack(mem);
 
     mark_static_roots(mem, static_roots, heap_base);
 
     if (*continuation_table_ptr_loc).is_ptr() {
         // TODO: No need to check if continuation table is already marked
-        mark_object(mem, *continuation_table_ptr_loc, heap_base);
+        mark_object(mem, *continuation_table_ptr_loc);
         // Similar to `mark_root_mutbox_fields`, `continuation_table_ptr_loc` is in static heap so
-        // it will be readable when we unthread continuation table
+        // it will be readable when we unthread the continuation table
         thread(continuation_table_ptr_loc);
     }
 
@@ -125,21 +127,21 @@ unsafe fn mark_static_roots<M: Memory>(mem: &mut M, static_roots: Value, heap_ba
     // Static objects are not in the dynamic heap so don't need marking.
     for i in 0..root_array.len() {
         let obj = root_array.get(i).as_obj();
-        // Root array should only has pointers to other static MutBoxes
+        // Root array should only have pointers to other static MutBoxes
         debug_assert_eq!(obj.tag(), TAG_MUTBOX); // check tag
         debug_assert!((obj as u32) < heap_base); // check that MutBox is static
         mark_root_mutbox_fields(mem, obj as *mut MutBox, heap_base);
     }
 }
 
-unsafe fn mark_object<M: Memory>(mem: &mut M, obj: Value, heap_base: u32) {
+unsafe fn mark_object<M: Memory>(mem: &mut M, obj: Value) {
     let obj_tag = obj.tag();
     let obj = obj.get_ptr() as u32;
 
     // Check object alignment to avoid undefined behavior. See also static_checks module.
     debug_assert_eq!(obj % WORD_SIZE, 0);
 
-    let obj_idx = (obj - heap_base) / WORD_SIZE;
+    let obj_idx = obj / WORD_SIZE;
 
     if get_bit(obj_idx) {
         // Already marked
@@ -159,7 +161,7 @@ unsafe fn mark_stack<M: Memory>(mem: &mut M, heap_base: u32) {
 unsafe fn mark_fields<M: Memory>(mem: &mut M, obj: *mut Obj, obj_tag: Tag, heap_base: u32) {
     visit_pointer_fields(obj, obj_tag, heap_base as usize, |field_addr| {
         let field_value = *field_addr;
-        mark_object(mem, field_value, heap_base);
+        mark_object(mem, field_value);
 
         // Thread if backwards or self pointer
         if field_value.get_ptr() <= obj as usize {
@@ -175,7 +177,7 @@ unsafe fn mark_root_mutbox_fields<M: Memory>(mem: &mut M, mutbox: *mut MutBox, h
     if pointer_to_dynamic_heap(field_addr, heap_base as usize) {
         // TODO: We should be able to omit the "already marked" check here as no two root MutBox
         // can point to the same object (I think)
-        mark_object(mem, *field_addr, heap_base);
+        mark_object(mem, *field_addr);
         // It's OK to thread forward pointers here as the static objects won't be moved, so we will
         // be able to unthread objects pointed by these fields later.
         thread(field_addr);
@@ -197,7 +199,7 @@ unsafe fn update_refs<SetHp: Fn(u32)>(set_hp: SetHp, heap_base: u32) {
     let mut bitmap_iter = iter_bits();
     let mut bit = bitmap_iter.next();
     while bit != BITMAP_ITER_END {
-        let p = (heap_base + (bit * WORD_SIZE)) as *mut Obj;
+        let p = (bit * WORD_SIZE) as *mut Obj;
         let p_new = free;
 
         // Update backwards references to the object's new location and restore object header