From b52df2b4521ee2114a9a2c36e39ca61bead476ad Mon Sep 17 00:00:00 2001
From: Jesse Brault <jbrault@mac.com>
Date: Fri, 17 Oct 2025 18:11:02 -0500
Subject: [PATCH] Rework gc api.

---
 src/lib.rs        |   4 +-
 src/vm/mem/mod.rs | 451 ++++++++++++++++++++++++++++++++++------------
 2 files changed, 334 insertions(+), 121 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index 930efd2..f80ffc7 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,9 +1,12 @@
 #![feature(new_range_api)]
+#![feature(coerce_unsized)]
+#![feature(unsize)]
 #![allow(warnings)]
 extern crate core;
 
 pub mod ast;
 pub mod diagnostic;
+pub mod ir;
 pub mod module;
 pub mod name_analysis;
 pub mod object_file;
@@ -11,4 +14,3 @@ pub mod parser;
 pub mod std_core;
 pub mod util;
 pub mod vm;
-pub mod ir;
diff --git a/src/vm/mem/mod.rs b/src/vm/mem/mod.rs
index f20f478..d865a72 100644
--- a/src/vm/mem/mod.rs
+++ b/src/vm/mem/mod.rs
@@ -1,90 +1,250 @@
+use std::any::TypeId;
 use std::cell::RefCell;
-use std::marker::PhantomData;
-use std::ops::Mul;
+use std::marker::Unsize;
+use std::ops::{CoerceUnsized, Deref, DerefMut, Mul};
+use std::sync::OnceLock;
 
-#[derive(Debug)]
-pub struct Gc<T> {
-    inner: *mut GcInner,
-    data: *const T,
-    _phantom: PhantomData<T>,
+pub trait Trace {
+    fn add_grays(&self, grays: &mut Vec<GcAny>);
 }
 
-#[derive(Debug)]
-pub struct GcInner {
-    color: GcColor,
-    size: usize,
-}
-
-impl GcInner {
-    pub fn new(size: usize) -> Self {
-        Self {
-            color: GcColor::White,
-            size,
+impl Trace for RefCell<DvmObject> {
+    fn add_grays(&self, grays: &mut Vec<GcAny>) {
+        for field in self.borrow().fields() {
+            match field {
+                DvmValue::Object(dvm_object) => grays.push(dvm_object.as_any()),
+                DvmValue::NativeObject(native_object) => grays.push(native_object.clone()),
+                _ => {}
+            }
         }
     }
 }
 
-#[derive(PartialEq, Eq, Copy, Clone, Debug)]
+impl Trace for String {
+    fn add_grays(&self, grays: &mut Vec<GcAny>) {
+        // no-op
+    }
+}
+
+#[derive(Clone)]
+pub struct Gc<T: Trace> {
+    inner: *mut GcInner<T>,
+}
+
+impl<T: Trace> Gc<T> {
+    pub fn new(data: T) -> Self {
+        let size = size_of_val(&data);
+        let inner = Box::into_raw(Box::new(GcInner::new(data, size)));
+        Self { inner }
+    }
+
+    fn from_inner(inner: *mut GcInner<T>) -> Self {
+        Self { inner }
+    }
+
+    pub fn borrow(&self) -> GcRef<T> {
+        unsafe {
+            (*self.inner).borrow_count += 1;
+            GcRef::new(self.inner)
+        }
+    }
+}
+
+impl<T: Trace + 'static> Gc<T> {
+    pub fn as_any(&self) -> GcAny {
+        GcAny::new::<T>(self.inner)
+    }
+}
+
+pub struct GcRef<T: Trace> {
+    inner: *mut GcInner<T>,
+}
+
+impl<T: Trace> GcRef<T> {
+    fn new(inner: *mut GcInner<T>) -> Self {
+        Self { inner }
+    }
+}
+
+impl<T: Trace> Deref for GcRef<T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        unsafe { &(*self.inner).data }
+    }
+}
+
+impl<T: Trace> DerefMut for GcRef<T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        unsafe { &mut (*self.inner).data }
+    }
+}
+
+impl<T: Trace> Drop for GcRef<T> {
+    fn drop(&mut self) {
+        unsafe {
+            (*self.inner).borrow_count -= 1;
+        }
+    }
+}
+
+struct GcInner<T: Trace> {
+    size: usize,
+    color: GcColor,
+    borrow_count: usize,
+    next: Option<GcAny>,
+    data: T,
+}
+
+impl<T: Trace> GcInner<T> {
+    fn new(data: T, size: usize) -> Self {
+        Self {
+            size,
+            color: GcColor::White,
+            borrow_count: 0,
+            next: None,
+            data,
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug)]
 pub enum GcColor {
     White,
     Black,
 }
 
-impl<T> Gc<T> {
-    pub fn new(data: T) -> Self {
-        let inner = Box::into_raw(Box::new(GcInner::new(size_of::<T>())));
-        let boxed_data = Box::into_raw(Box::new(data)) as *const T;
+struct GcAnyMeta {
+    trace: unsafe fn(*mut (), &mut Vec<GcAny>),
+    size: unsafe fn(*mut ()) -> usize,
+    borrow_count: unsafe fn(*mut ()) -> usize,
+    color: unsafe fn(*mut ()) -> GcColor,
+    set_color: unsafe fn(*mut (), GcColor),
+    next: unsafe fn(*mut ()) -> Option<GcAny>,
+    set_next: unsafe fn(*mut (), GcAny),
+    dealloc: unsafe fn(*mut ()),
+    type_id: TypeId,
+}
+
+#[derive(Clone)]
+pub struct GcAny {
+    inner: *mut (), // *mut GcInner<T>
+    meta: &'static GcAnyMeta,
+}
+
+impl GcAny {
+    pub fn new<T: Trace + 'static>(gc_inner: *mut GcInner<T>) -> Self {
+        unsafe fn trace_impl<T: Trace>(gc_inner: *mut (), grays: &mut Vec<GcAny>) {
+            (*(gc_inner as *mut GcInner<T>)).data.add_grays(grays);
+        }
+
+        unsafe fn size_impl<T: Trace>(gc_inner: *mut ()) -> usize {
+            (*(gc_inner as *mut GcInner<T>)).size
+        }
+
+        unsafe fn borrow_count_impl<T: Trace>(gc_inner: *mut ()) -> usize {
+            (*(gc_inner as *mut GcInner<T>)).borrow_count
+        }
+
+        unsafe fn color_impl<T: Trace>(gc_inner: *mut ()) -> GcColor {
+            (*(gc_inner as *mut GcInner<T>)).color
+        }
+
+        unsafe fn set_color_impl<T: Trace>(gc_inner: *mut (), color: GcColor) {
+            (*(gc_inner as *mut GcInner<T>)).color = color;
+        }
+
+        unsafe fn next_impl<T: Trace>(gc_inner: *mut ()) -> Option<GcAny> {
+            (*(gc_inner as *mut GcInner<T>)).next.clone()
+        }
+
+        unsafe fn set_next_impl<T: Trace>(gc_inner: *mut (), next: GcAny) {
+            (*(gc_inner as *mut GcInner<T>)).next = Some(next);
+        }
+
+        unsafe fn dealloc_impl<T: Trace>(gc_inner: *mut ()) {
+            drop(Box::from_raw(gc_inner as *mut GcInner<T>));
+        }
+
+        static VTABLE: OnceLock<GcAnyMeta> = OnceLock::new();
+        let v_table = VTABLE.get_or_init(|| GcAnyMeta {
+            trace: trace_impl::<T>,
+            size: size_impl::<T>,
+            borrow_count: borrow_count_impl::<T>,
+            color: color_impl::<T>,
+            set_color: set_color_impl::<T>,
+            next: next_impl::<T>,
+            set_next: set_next_impl::<T>,
+            dealloc: dealloc_impl::<T>,
+            type_id: TypeId::of::<T>(),
+        });
         Self {
-            inner,
-            data: boxed_data,
-            _phantom: PhantomData,
+            meta: v_table,
+            inner: gc_inner as *mut (),
         }
     }
 
-    pub fn set_color(&mut self, color: GcColor) {
+    fn trace(&self, grays: &mut Vec<GcAny>) {
         unsafe {
-            (*self.inner).color = color;
+            (self.meta.trace)(self.inner, grays);
         }
     }
 
-    pub fn color(&self) -> GcColor {
-        unsafe { (*self.inner).color }
+    fn size(&self) -> usize {
+        unsafe { (self.meta.size)(self.inner) }
     }
 
-    pub fn size(&self) -> usize {
-        unsafe { (*self.inner).size }
+    fn borrow_count(&self) -> usize {
+        unsafe { (self.meta.borrow_count)(self.inner) }
     }
 
-    pub fn data(&self) -> &T {
-        unsafe { &*self.data }
+    fn color(&self) -> GcColor {
+        unsafe { (self.meta.color)(self.inner) }
     }
 
-    pub fn dealloc(&mut self) {
+    fn set_color(&self, color: GcColor) {
         unsafe {
-            drop(Box::from_raw(self.inner));
-            drop(Box::from_raw(self.data as *mut T));
+            (self.meta.set_color)(self.inner, color);
+        }
+    }
+
+    fn next(&self) -> Option<GcAny> {
+        unsafe { (self.meta.next)(self.inner) }
+    }
+
+    fn set_next(&self, next: GcAny) {
+        unsafe { (self.meta.set_next)(self.inner, next) }
+    }
+
+    fn dealloc(&mut self) {
+        unsafe {
+            if self.borrow_count() > 0 {
+                panic!("Attempt to dealloc a GcAny whose borrow count > 0");
+            }
+            (self.meta.dealloc)(self.inner)
+        }
+    }
+
+    pub fn is<T: 'static>(&self) -> bool {
+        self.meta.type_id == TypeId::of::<T>()
+    }
+
+    pub fn downcast<T: Trace + 'static>(&self) -> Option<Gc<T>> {
+        if self.is::<T>() {
+            unsafe { Some(Gc::from_inner(self.inner as *mut GcInner<T>)) }
+        } else {
+            None
         }
     }
 }
 
-impl<T> Clone for Gc<T> {
-    fn clone(&self) -> Self {
-        Self {
-            inner: self.inner,
-            data: self.data,
-            _phantom: PhantomData,
-        }
-    }
-}
-
-#[derive(Debug)]
 enum DvmValue {
     Primitive(usize),
     Object(Gc<RefCell<DvmObject>>),
+    NativeObject(GcAny),
     Nil,
 }
 
-#[derive(Debug)]
 struct DvmObject {
     fields: Vec<DvmValue>,
 }
@@ -103,103 +263,154 @@ impl DvmObject {
     }
 }
 
-fn collect_garbage(
-    next_gc: &mut usize,
-    current_size: &mut usize,
-    stack: &Vec<DvmValue>,
-    gcs: &mut Vec<Gc<RefCell<DvmObject>>>,
-) {
-    let mut gray_stack: Vec<Gc<RefCell<DvmObject>>> = vec![];
+pub struct GcHeap {
+    head: Option<GcAny>,
+    tail: Option<GcAny>,
+    next_gc: usize,
+    current_size: usize,
+}
+
+impl GcHeap {
+    pub fn new() -> Self {
+        Self {
+            head: None,
+            tail: None,
+            next_gc: 1024 * 1024,
+            current_size: 0,
+        }
+    }
+
+    pub fn alloc<T: Trace + 'static>(&mut self, t: T) -> Gc<T> {
+        let gc = Gc::new(t);
+        if self.tail.is_some() {
+            let tail = self.tail.take().unwrap();
+            tail.set_next(gc.as_any());
+            self.tail = Some(tail);
+        } else {
+            self.head = Some(gc.as_any());
+            self.tail = Some(gc.as_any());
+        }
+        gc
+    }
+
+    pub fn head(&self) -> Option<GcAny> {
+        self.head.clone()
+    }
+
+    pub fn set_head(&mut self, head: GcAny) {
+        self.head = Some(head.clone());
+        if self.tail.is_none() {
+            self.tail = Some(head);
+        }
+    }
+
+    pub fn clear_head(&mut self) {
+        self.head = None;
+    }
+
+    pub fn current_size(&self) -> usize {
+        self.current_size
+    }
+
+    pub fn set_current_size(&mut self, size: usize) {
+        self.current_size = size;
+    }
+
+    pub fn subtract_current_size(&mut self, size: usize) {
+        self.current_size -= size;
+    }
+
+    pub fn next_gc(&self) -> usize {
+        self.next_gc
+    }
+
+    pub fn set_next_gc(&mut self, next_gc: usize) {
+        self.next_gc = next_gc;
+    }
+}
+
+impl Drop for GcHeap {
+    fn drop(&mut self) {
+        let mut current = self.head().take();
+        while let Some(gc_any) = &mut current {
+            let next = gc_any.next();
+            gc_any.dealloc();
+            current = next;
+        }
+    }
+}
+
+fn collect_garbage(stack: &Vec<DvmValue>, heap: &mut GcHeap) {
+    if heap.current_size() < heap.next_gc() {
+        return;
+    }
+
+    let mut gray_stack: Vec<GcAny> = vec![];
 
     for stack_value in stack {
         match stack_value {
             DvmValue::Object(gc) => {
-                gray_stack.push(gc.clone());
+                gray_stack.push(gc.as_any());
+            }
+            DvmValue::NativeObject(gc_any) => {
+                gray_stack.push(gc_any.clone());
             }
             _ => {}
         }
     }
 
     while let Some(mut current) = gray_stack.pop() {
-        for field in current.data().borrow().fields() {
-            match field {
-                DvmValue::Object(object) => {
-                    if object.color() == GcColor::White {
-                        gray_stack.push(object.clone());
-                    }
-                }
-                _ => {}
-            }
-        }
+        current.trace(&mut gray_stack);
         current.set_color(GcColor::Black);
     }
 
     let mut collected_size: usize = 0;
 
-    for i in (0..gcs.len()).rev() {
-        if gcs[i].color() == GcColor::White {
-            let mut gc = gcs.remove(i);
-            let size = gc.size();
-            *current_size -= size;
-            collected_size += size;
-            gc.dealloc();
+    let mut previous: Option<GcAny> = None;
+    let mut current = heap.head();
+    while let Some(ref mut current_gc_erased) = current {
+        if let GcColor::White = current_gc_erased.color() {
+            if let Some(ref previous_gc_erased) = previous {
+                // attach previous to next if both are Some
+                let next = current_gc_erased.next();
+                if let Some(next_gc_erased) = next {
+                    previous_gc_erased.set_next(next_gc_erased);
+                }
+            }
+            let size = current_gc_erased.size();
+            heap.subtract_current_size(size);
+            collected_size = size;
+            current_gc_erased.dealloc();
         } else {
-            gcs[i].set_color(GcColor::White);
+            current_gc_erased.set_color(GcColor::White);
+            previous = Some(current_gc_erased.clone());
         }
     }
 
-    *next_gc = std::cmp::max(1024 * 1024, current_size.mul(2));
-    println!(
-        "collected_size: {}, current_size: {}, next_gc: {}",
-        collected_size, current_size, next_gc
-    );
-}
-
-fn maybe_collect_garbage(
-    next_gc: &mut usize,
-    current_size: &mut usize,
-    stack: &Vec<DvmValue>,
-    gcs: &mut Vec<Gc<RefCell<DvmObject>>>,
-) {
-    if current_size > next_gc {
-        collect_garbage(next_gc, current_size, stack, gcs);
-    }
-}
-
-fn alloc_dvm_object(
-    next_gc: &mut usize,
-    current_size: &mut usize,
-    stack: &Vec<DvmValue>,
-    gcs: &mut Vec<Gc<RefCell<DvmObject>>>,
-) -> Gc<RefCell<DvmObject>> {
-    maybe_collect_garbage(next_gc, current_size, stack, gcs);
-    let mut object = DvmObject::new();
-    object.fields_mut().push(DvmValue::Nil);
-    let gc = Gc::new(RefCell::new(object));
-    *current_size += gc.size();
-    gcs.push(gc.clone());
-    gc
-}
-
-fn set_field(obj: &mut Gc<RefCell<DvmObject>>, field_index: usize, value: DvmValue) {
-    let mut binding = obj.data().borrow_mut();
-    binding.fields_mut()[field_index] = value;
+    heap.set_next_gc(std::cmp::max(1024 * 1024, heap.current_size()));
 }
 
 #[cfg(test)]
 mod tests {
-    use crate::vm::mem::{alloc_dvm_object, collect_garbage, DvmObject, DvmValue, Gc};
-    use std::cell::RefCell;
+    use crate::vm::mem::{DvmValue, GcHeap};
 
     #[test]
-    fn test() {
+    fn simple_allocate() {
         let mut stack: Vec<DvmValue> = vec![];
-        let mut heap: Vec<Gc<RefCell<DvmObject>>> = vec![];
-        let mut next_gc: usize = 1024 * 1024;
-        let mut current_size: usize = 0;
-        for i in 0..100_000_000 {
-            let mut o = alloc_dvm_object(&mut next_gc, &mut current_size, &stack, &mut heap);
+        let mut heap = GcHeap::new();
+        for i in 0..100 {
+            let o = heap.alloc(String::new());
+            o.borrow().push_str("hello");
+        }
+    }
+
+    #[test]
+    fn downcast_as_string() {
+        let mut heap = GcHeap::new();
+        let t = heap.alloc(String::from("hello world"));
+        let a = t.as_any();
+        if let Some(s) = a.downcast::<String>() {
+            assert_eq!(*s.borrow(), "hello world");
         }
-        collect_garbage(&mut next_gc, &mut current_size, &stack, &mut heap);
     }
 }