From 303607c140c2eb17053b347506042ef2f2a334ed Mon Sep 17 00:00:00 2001 From: Nia Weiss Date: Sun, 18 Jul 2021 14:46:20 -0400 Subject: [PATCH] runtime (gc): add a new "list" GC This memory manager uses a simple linked list to track active allocations, and measures the gaps between them to find available memory. Performance is not great for large heaps, but is sufficient for microcontrollers with single-digit kilobytes of memory. --- compileopts/options.go | 2 +- compileopts/options_test.go | 2 +- main.go | 2 +- src/runtime/gc_globals_conservative.go | 2 +- src/runtime/gc_globals_precise.go | 2 +- src/runtime/gc_list.go | 467 +++++++++++++++++++++++++ src/runtime/gc_stack_portable.go | 2 +- src/runtime/gc_stack_raw.go | 2 +- src/runtime/runtime_unix_heap.go | 2 +- targets/avr.json | 2 +- 10 files changed, 476 insertions(+), 9 deletions(-) create mode 100644 src/runtime/gc_list.go diff --git a/compileopts/options.go b/compileopts/options.go index 7e7bfcafc4..17eb13a569 100644 --- a/compileopts/options.go +++ b/compileopts/options.go @@ -7,7 +7,7 @@ import ( ) var ( - validGCOptions = []string{"none", "leaking", "extalloc", "conservative"} + validGCOptions = []string{"none", "leaking", "extalloc", "conservative", "list"} validSchedulerOptions = []string{"none", "tasks", "coroutines"} validSerialOptions = []string{"none", "uart", "usb"} validPrintSizeOptions = []string{"none", "short", "full"} diff --git a/compileopts/options_test.go b/compileopts/options_test.go index 1ff532cc4f..e3d07178a4 100644 --- a/compileopts/options_test.go +++ b/compileopts/options_test.go @@ -9,7 +9,7 @@ import ( func TestVerifyOptions(t *testing.T) { - expectedGCError := errors.New(`invalid gc option 'incorrect': valid values are none, leaking, extalloc, conservative`) + expectedGCError := errors.New(`invalid gc option 'incorrect': valid values are none, leaking, extalloc, conservative, list`) expectedSchedulerError := errors.New(`invalid scheduler option 'incorrect': valid values are none, tasks, coroutines`) expectedPrintSizeError := errors.New(`invalid size option 'incorrect': valid values are none, short, full`) expectedPanicStrategyError := errors.New(`invalid panic option 'incorrect': valid values are print, trap`) diff --git a/main.go b/main.go index 7c3ee96df8..43647cc76d 100644 --- a/main.go +++ b/main.go @@ -1006,7 +1006,7 @@ func main() { command := os.Args[1] opt := flag.String("opt", "z", "optimization level: 0, 1, 2, s, z") - gc := flag.String("gc", "", "garbage collector to use (none, leaking, extalloc, conservative)") + gc := flag.String("gc", "", "garbage collector to use (none, leaking, extalloc, conservative, list)") panicStrategy := flag.String("panic", "print", "panic strategy (print, trap)") scheduler := flag.String("scheduler", "", "which scheduler to use (none, coroutines, tasks)") serial := flag.String("serial", "", "which serial output to use (none, uart, usb)") diff --git a/src/runtime/gc_globals_conservative.go b/src/runtime/gc_globals_conservative.go index 564765515b..d66bb39370 100644 --- a/src/runtime/gc_globals_conservative.go +++ b/src/runtime/gc_globals_conservative.go @@ -1,4 +1,4 @@ -// +build gc.conservative gc.extalloc +// +build gc.conservative gc.extalloc gc.list // +build baremetal tinygo.wasm package runtime diff --git a/src/runtime/gc_globals_precise.go b/src/runtime/gc_globals_precise.go index f79b71fdc9..6feb2cc605 100644 --- a/src/runtime/gc_globals_precise.go +++ b/src/runtime/gc_globals_precise.go @@ -1,4 +1,4 @@ -// +build gc.conservative gc.extalloc +// +build gc.conservative gc.extalloc gc.list // +build !baremetal,!tinygo.wasm package runtime diff --git a/src/runtime/gc_list.go b/src/runtime/gc_list.go new file mode 100644 index 0000000000..ff3bf9650b --- /dev/null +++ b/src/runtime/gc_list.go @@ -0,0 +1,467 @@ +// +build gc.list + +// This memory manager is partially inspired by the memory allocator included in avr-libc. +// Unlike avr-libc malloc, this memory manager stores an allocation list instead of a free list. +// There is an overhead of 2 pointer-widths/allocation (up from avr-libc's 1 pointer-width/allocation). +// The allocation list is stored in ascending address order. +// The set of free spans is implicitly derived from the gaps between allocations. +// This allocator has an effective allocation complexity of O(n) and worst-case GC complexity of O(n^2). +// Due to architectural quirks, as well as tiny heaps, this should almost always be faster than the standard conservative collector on AVR. + +package runtime + +import ( + "internal/task" + "runtime/interrupt" + "unsafe" +) + +// Set gcDebug to true to print debug information. +const ( + gcDebug = false // print debug info + gcAsserts = gcDebug // perform sanity checks +) + +// setHeapEnd is called to expand the heap. The heap can only grow, not shrink. +func setHeapEnd(newHeapEnd uintptr) { + if gcAsserts && newHeapEnd <= heapEnd { + panic("gc: setHeapEnd didn't grow the heap") + } + + heapEnd = newHeapEnd + + // There is no other metadata to update since the free space is just computed from the heap bounds and the allocations list. +} + +// Initialize the memory allocator. +func initHeap() { + // No initialization is required: the only state is the allocation list, which starts empty. + // This function must exist anyway since other GCs need initialization. + // It should be inlined and optimized away by the compiler. +} + +var gcRunning bool + +// alloc tries to find some free space on the heap, possibly doing a garbage +// collection cycle if needed. If no space is free, it panics. +//go:noinline +func alloc(size uintptr) unsafe.Pointer { + if gcAsserts && gcRunning { + runtimePanic("alloc while GC is running") + } + + if size == 0 { + return unsafe.Pointer(&zeroSizedAlloc) + } + + var ranGC bool +tryAlloc: + // Search for available memory. + alloc := findMem(size) + if alloc == nil { + // There is no free span large enough for the allocation. + + if ranGC { + // Even after running the GC, there is not enough memory. + if growHeap() { + // The heap was able to grow to fit more data. + // Try allocating again. + // When doing a huge allocation this will loop until the heap is big enough. + goto tryAlloc + } + + // There is no more available memory. + runtimePanic("out of memory") + } + + // Run the garbage collector and try again. + GC() + ranGC = true + goto tryAlloc + } + + // Zero the allocation. + ptr := unsafe.Pointer(uintptr(unsafe.Pointer(alloc)) + unsafe.Sizeof(allocHeader{})) + memzero(ptr, size) + + return ptr +} + +// zeroSizedAlloc is just a sentinel that gets returned when allocating 0 bytes. +var zeroSizedAlloc uint8 + +// findMem searches the heap for a free span large enough to contain an allocation of the specified size. +// If there are no sufficiently large free spans available, this returns nil. +func findMem(size uintptr) *allocHeader { + // This memory allocator implementation is effectively the same algorithm applied by avr-libc. + // It loops through the set of all free spans, and selects the smallest span that is large enough to fit the allocation. + + allocSize := unsafe.Sizeof(allocHeader{}) + size + + // best* store the best-fit free span. + var bestDst **allocHeader + var bestStart uintptr + var bestSize uintptr + + start := heapStart + dst := &activeAllocs.first +searchLoop: + for { + // Find the allocation after this free span. + alloc := *dst + + // Find the end of this free span. + var end uintptr + if alloc != nil { + // The allocation terminates the free span. + end = uintptr(unsafe.Pointer(alloc)) + } else { + // The free span ends at the end of the heap. + end = heapEnd + } + + // Calculate the size of the free span. + freeSpanSize := end - start + + switch { + case freeSpanSize == allocSize: + // This span is a perfect fit. + bestDst = dst + bestStart = start + break searchLoop + case freeSpanSize > allocSize && (bestDst == nil || bestSize > allocSize): + // This span is a better fit than the previous best. + bestDst = dst + bestStart = start + bestSize = freeSpanSize + } + + // Move to the next free span. + if alloc == nil { + // That was the last free region. + break searchLoop + } + start = align(uintptr(unsafe.Pointer(alloc)) + unsafe.Sizeof(allocHeader{}) + alloc.len) + dst = &alloc.next + } + + if bestDst == nil { + // There is no suitable allocation. + return nil + } + + mem := (*allocHeader)(unsafe.Pointer(bestStart)) + *bestDst, mem.next = mem, *bestDst + mem.len = size + return mem +} + +func free(ptr unsafe.Pointer) { + // TODO: free allocations on request, when the compiler knows they're unused. +} + +// GC performs a garbage collection cycle. +func GC() { + if gcAsserts { + gcRunning = true + } + + if gcDebug { + println("running collection cycle...") + } + + // Mark in-use allocations. + remaining := mark() + + if gcDebug { + println("found unreferenced allocations:") + activeAllocs.dump() + } + + // Update the active allocations list. + activeAllocs = remaining + + if gcDebug { + println("sorting active allocations") + } + + activeAllocs.sort() + + if gcDebug { + println("garbage collection cycle complete") + } + + if gcAsserts { + gcRunning = false + } +} + +// mark all in-use allocations and save them as a sorted list. +func mark() allocList { + if gcDebug { + println("running GC mark phase...") + println("pre-mark allocations:") + activeAllocs.dump() + } + + // Mark all allocations referenced by currently-executing functions. + markStack() + + // Mark all allocations referenced by global variables. + markGlobals() + + // Save remaining allocations to a list after they have been marked and scanned. + var remainingAllocs allocList + + // Channel operations in interrupts may move task pointers around while we are marking. + // Therefore we need to scan the runqueue seperately. + var markedTaskQueue task.Queue +runqueueScan: + if baremetal && hasScheduler { + for !runqueue.Empty() { + // Pop the next task off of the runqueue. + t := runqueue.Pop() + + // Mark the task if it has not already been marked. + markRoot(uintptr(unsafe.Pointer(&runqueue)), uintptr(unsafe.Pointer(t))) + + // Push the task onto our temporary queue. + markedTaskQueue.Push(t) + } + } + + // Mark allocations referenced by other marked allocations. + for { + // Pop an allocation off of the scan list. + alloc := scanList.pop() + if alloc == nil { + break + } + + // Scan the allocation. + alloc.scan() + + // Save the allocation in the remaining allocations list. + remainingAllocs.push(alloc) + } + + if baremetal && hasScheduler { + // Restore the runqueue. + i := interrupt.Disable() + if !runqueue.Empty() { + // Something new came in while finishing the mark. + interrupt.Restore(i) + if gcDebug { + println("scanning tasks queued during GC") + } + goto runqueueScan + } + runqueue = markedTaskQueue + interrupt.Restore(i) + } + + if gcDebug { + println("GC mark phase complete") + println("remaining allocations:") + remainingAllocs.dump() + } + + return remainingAllocs +} + +// sort the allocation list in ascending address order. +func (list *allocList) sort() { + // Sort the list by repeatedly moving the lowest-address element to the front. + // This is selection sort (https://en.wikipedia.org/wiki/Selection_sort). + for dst := &list.first; *dst != nil; dst = &((*dst).next) { + // Find the next lowest-address allocation. + src := dst + min := *src + for ref, alloc := src, min; alloc != nil; ref, alloc = &alloc.next, alloc.next { + if uintptr(unsafe.Pointer(alloc)) < uintptr(unsafe.Pointer(min)) { + src, min = ref, alloc + } + } + + // Remove the min from the list. + *src = min.next + + // Insert the min at dst. + *dst, min.next = min, *dst + } + + if gcDebug { + println("sorted allocations:") + list.dump() + } + + if gcAsserts && list.first != nil { + prev := list.first + for alloc := list.first.next; alloc != nil; alloc, prev = alloc.next, alloc { + if uintptr(unsafe.Pointer(alloc)) <= uintptr(unsafe.Pointer(prev)) { + runtimePanic("alloc list not sorted") + } + } + } + + if gcAsserts { + var prevEnd uintptr + for alloc := list.first; alloc != nil; alloc, prevEnd = alloc.next, uintptr(unsafe.Pointer(alloc))+unsafe.Sizeof(allocHeader{})+alloc.len { + start := uintptr(unsafe.Pointer(alloc)) + if start < prevEnd { + runtimePanic("overlapping allocations") + } + } + } +} + +// scan the allocation for pointers to other allocations. +func (alloc *allocHeader) scan() { + start := uintptr(unsafe.Pointer(alloc)) + unsafe.Sizeof(allocHeader{}) + end := start + alloc.len + + if gcDebug { + println("scanning allocation at", start, "of", alloc.len, "bytes") + } + + markRoots(start, end) +} + +// markRoots reads all pointers from start to end (exclusive) and if they look +// like a heap pointer and are unmarked, marks them and scans that object as +// well (recursively). The start and end parameters must be valid pointers and +// must be aligned. +func markRoots(start, end uintptr) { + if gcDebug { + println("mark from", start, "to", end, int(end-start)) + } + if gcAsserts { + if start >= end { + runtimePanic("gc: unexpected range to mark") + } + } + + for addr := start; addr+unsafe.Sizeof(unsafe.Pointer(nil)) <= end; addr += unsafe.Alignof(addr) { + root := *(*uintptr)(unsafe.Pointer(addr)) + markRoot(addr, root) + } +} + +// mark a GC root at the address addr. +func markRoot(addr, root uintptr) { + if !looksLikePointer(root) { + // This is not a pointer. + // Skip searching for it. + return + } + + // Search the list of unmarked allocations for this root address. + alloc := activeAllocs.search(root) + if alloc == nil { + // The pointer is either a false positive (not a pointer) or already marked. + // Either way, there is nothing to do. + return + } + if gcDebug { + println("found unmarked pointer", root, "at address", addr) + } + + // Push the allocation onto the scan list. + scanList.push(alloc) +} + +// looksLikePointer returns whether this could be a pointer. Currently, it +// simply returns whether it lies anywhere in the heap. Go allows interior +// pointers so we can't check alignment or anything like that. +func looksLikePointer(ptr uintptr) bool { + return ptr >= heapStart && ptr < heapEnd +} + +// activeAllocs is a list of allocations which are currently in use. +// During the mark pass, marked allocations are removed from here and placed onto the scan list. +// This list is stored in ascending address order. +var activeAllocs allocList + +// scanList is a list of allocations which have been marked but not yet scanned. +var scanList allocList + +// allocList is a list of allocations. +type allocList struct { + first *allocHeader +} + +// push an allocation to the front of the list. +func (list *allocList) push(node *allocHeader) { + list.first, node.next = node, list.first +} + +// pop an allocation from the front of the list. +// If the list is empty, this will return nil. +func (list *allocList) pop() *allocHeader { + if list.first == nil { + return nil + } + + popped := list.first + list.first = popped.next + + return popped +} + +// search finds and removes the allocation containing an address. +// This assumes that the list is sorted. +// If no allocation contains this address, this will return nil. +func (list *allocList) search(addr uintptr) *allocHeader { + // While traversing the list, keep the pointer to the pointer to the current allocation list entry. + // This is used later to remove it from the list. + prev := &list.first + for alloc := list.first; alloc != nil; alloc, prev = alloc.next, &alloc.next { + // Find the start of the memory after the allocation header. + start := uintptr(unsafe.Pointer(alloc)) + unsafe.Sizeof(allocHeader{}) + if addr < start { + return nil + } + + // Check if the address is before the end of the allocation. + // This check calculates the distance between the start of the allocation and compares that to the length. + // This works around an overflow when the last byte of an allocation is ^uintptr(0). + if addr-start >= alloc.len { + continue + } + + // The address is within this allocation. + // Remove the allocation from the list by replacing the reference to it with the next allocation. + *prev = alloc.next + alloc.next = nil + + return alloc + } + + return nil +} + +// dump the contents of the list to the debug output. +func (list *allocList) dump() { + for alloc := list.first; alloc != nil; alloc = alloc.next { + start := uintptr(unsafe.Pointer(alloc)) + unsafe.Sizeof(allocHeader{}) + println("-", "[", start, ",", start+alloc.len, "):", alloc.len, "bytes") + } +} + +// allocHeader is a header placed before an allocation in order to store metadata for the garbage collector. +// It is used to form a linked list of allocations. +type allocHeader struct { + // next is a pointer to the next allocation. + next *allocHeader + + // len is the length of the body of this node in bytes. + len uintptr +} + +func KeepAlive(x interface{}) { + // Unimplemented. Only required with SetFinalizer(). +} + +func SetFinalizer(obj interface{}, finalizer interface{}) { + // Unimplemented. +} diff --git a/src/runtime/gc_stack_portable.go b/src/runtime/gc_stack_portable.go index d4a046374f..052f9955ee 100644 --- a/src/runtime/gc_stack_portable.go +++ b/src/runtime/gc_stack_portable.go @@ -1,4 +1,4 @@ -// +build gc.conservative gc.extalloc +// +build gc.conservative gc.extalloc gc.list // +build tinygo.wasm package runtime diff --git a/src/runtime/gc_stack_raw.go b/src/runtime/gc_stack_raw.go index 01b07d9bbf..16563b085f 100644 --- a/src/runtime/gc_stack_raw.go +++ b/src/runtime/gc_stack_raw.go @@ -1,4 +1,4 @@ -// +build gc.conservative gc.extalloc +// +build gc.conservative gc.extalloc gc.list // +build !tinygo.wasm package runtime diff --git a/src/runtime/runtime_unix_heap.go b/src/runtime/runtime_unix_heap.go index cee1f1e1a0..2a9643e538 100644 --- a/src/runtime/runtime_unix_heap.go +++ b/src/runtime/runtime_unix_heap.go @@ -1,7 +1,7 @@ // +build darwin linux,!baremetal,!wasi freebsd,!baremetal // +build !nintendoswitch -// +build gc.conservative gc.leaking +// +build gc.conservative gc.leaking gc.list package runtime diff --git a/targets/avr.json b/targets/avr.json index 2ec83a7aa7..617a95ba5f 100644 --- a/targets/avr.json +++ b/targets/avr.json @@ -3,7 +3,7 @@ "build-tags": ["avr", "baremetal", "linux", "arm"], "goos": "linux", "goarch": "arm", - "gc": "conservative", + "gc": "list", "linker": "avr-gcc", "scheduler": "none", "default-stack-size": 256,