diff --git a/build-polkawasm.sh b/build-polkawasm.sh index dd222a229e..f4087aa096 100755 --- a/build-polkawasm.sh +++ b/build-polkawasm.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -docker build --tag polkawasm/tinygo:dev -f Dockerfile.polkawasm . \ No newline at end of file +docker build --tag polkawasm/tinygo:dev-ext -f Dockerfile.polkawasm . \ No newline at end of file diff --git a/compileopts/config.go b/compileopts/config.go index 2358ef93dd..c5f9831b94 100644 --- a/compileopts/config.go +++ b/compileopts/config.go @@ -98,14 +98,19 @@ func (c *Config) GC() string { if c.Target.GC != "" { return c.Target.GC } - return "conservative" + for _, tag := range c.Target.BuildTags { + if tag == "baremetal" || tag == "wasm" { + return "conservative" + } + } + return "extalloc" } // NeedsStackObjects returns true if the compiler should insert stack objects // that can be traced by the garbage collector. func (c *Config) NeedsStackObjects() bool { switch c.GC() { - case "conservative": + case "conservative", "extalloc": for _, tag := range c.BuildTags() { if tag == "tinygo.wasm" { return true diff --git a/main.go b/main.go index 9df30ef69e..df0e5324b1 100644 --- a/main.go +++ b/main.go @@ -1369,7 +1369,7 @@ func main() { command := os.Args[1] opt := flag.String("opt", "z", "optimization level: 0, 1, 2, s, z") - gc := flag.String("gc", "", "garbage collector to use (none, leaking, conservative)") + gc := flag.String("gc", "", "garbage collector to use (none, leaking, conservative, extalloc)") panicStrategy := flag.String("panic", "print", "panic strategy (print, trap)") scheduler := flag.String("scheduler", "", "which scheduler to use (none, tasks, asyncify)") serial := flag.String("serial", "", "which serial output to use (none, uart, usb)") diff --git a/src/runtime/gc_extalloc.go b/src/runtime/gc_extalloc.go new file mode 100644 index 0000000000..5b53093f16 --- /dev/null +++ b/src/runtime/gc_extalloc.go @@ -0,0 +1,612 @@ +//go:build gc.extalloc + +package runtime + +import "unsafe" + +// This garbage collector implementation allows TinyGo to use an external memory allocator. +// It appends a header to the end of every allocation which the garbage collector uses for tracking purposes. +// This is also a conservative collector. + +const ( + gcDebug = false + gcAsserts = false +) + +func initHeap() {} + +// memTreap is a treap which is used to track allocations for the garbage collector. +type memTreap struct { + root *memTreapNode +} + +// printNode recursively prints a subtree at a given indentation depth. +func (t *memTreap) printNode(n *memTreapNode, depth int) { + for i := 0; i < depth; i++ { + print(" ") + } + println(n, n.priority()) + if n == nil { + return + } + if gcAsserts && n.parent == nil && t.root != n { + runtimePanic("parent missing") + } + t.printNode(n.left, depth+1) + t.printNode(n.right, depth+1) +} + +// print the treap. +func (t *memTreap) print() { + println("treap:") + t.printNode(t.root, 1) +} + +// empty returns whether the treap contains any nodes. +func (t *memTreap) empty() bool { + return t.root == nil +} + +// minAddr returns the lowest address contained in an allocation in the treap. +func (t *memTreap) minAddr() uintptr { + // Find the rightmost node. + n := t.root + for n.right != nil { + n = n.right + } + + // The lowest address is the base of the rightmost node. + return uintptr(unsafe.Pointer(&n.base)) +} + +// maxAddr returns the highest address contained in an allocation in the treap. +func (t *memTreap) maxAddr() uintptr { + // Find the leftmost node. + n := t.root + for n.left != nil { + n = n.left + } + + // The highest address is the end of the leftmost node. + return uintptr(unsafe.Pointer(&n.base)) + n.size +} + +// rotateRight does a right rotation of p and q. +// https://en.wikipedia.org/wiki/Tree_rotation#/media/File:Tree_rotation.png +func (t *memTreap) rotateRight(p, q *memTreapNode) { + if t.root == q { + t.root = p + } else { + *q.parentSlot() = p + } + + //a := p.left + b := p.right + //c := q.right + + p.parent = q.parent + p.right = q + + q.parent = p + q.left = b + + if b != nil { + b.parent = q + } +} + +// rotateLeft does a left rotation of p and q. +// https://en.wikipedia.org/wiki/Tree_rotation#/media/File:Tree_rotation.png +func (t *memTreap) rotateLeft(p, q *memTreapNode) { + if t.root == p { + t.root = q + } else { + *p.parentSlot() = q + } + + //a := p.left + b := q.left + //c := q.right + + q.parent = p.parent + q.left = p + + p.parent = q + p.right = b + + if b != nil { + b.parent = p + } +} + +// rotate rotates a lower node up to its parent. +// The node n must be a child of m, and will be the parent of m after the rotation. +func (t *memTreap) rotate(n, m *memTreapNode) { + // https://en.wikipedia.org/wiki/Tree_rotation#/media/File:Tree_rotation.png + if uintptr(unsafe.Pointer(n)) > uintptr(unsafe.Pointer(m)) { + t.rotateRight(n, m) + } else { + t.rotateLeft(m, n) + } +} + +// insert a node into the treap. +func (t *memTreap) insert(n *memTreapNode) { + if gcAsserts && (n.parent != nil || n.left != nil || n.right != nil) { + runtimePanic("tried to insert unzeroed treap node") + } + + if t.root == nil { + // This is the first node, and can be inserted directly into the root. + t.root = n + return + } + + // Insert like a regular binary search tree. + for n.parent = t.root; *n.parentSlot() != nil; n.parent = *n.parentSlot() { + } + *n.parentSlot() = n + + // Rotate the tree to restore the heap invariant. + priority := n.priority() + for n.parent != nil && priority > n.parent.priority() { + t.rotate(n, n.parent) + } +} + +// lookupAddr finds the treap node with the allocation containing the specified address. +// If the address is not contained in any allocations in this treap, nil is returned. +// NOTE: fields of memTreapNodes are not considered part of the allocations. +func (t *memTreap) lookupAddr(addr uintptr) *memTreapNode { + n := t.root + for n != nil && !n.contains(addr) { + if addr > uintptr(unsafe.Pointer(n)) { + n = n.left + } else { + n = n.right + } + } + + return n +} + +// replace a node with another node on the treap. +func (t *memTreap) replace(old, new *memTreapNode) { + if gcAsserts && (old == nil || new == nil) { + if gcDebug { + println("tried to replace:", old, "->", new) + } + runtimePanic("invalid replacement") + } + if gcAsserts && old.parent == nil && old != t.root { + if gcDebug { + println("tried to replace:", old, "->", new) + t.print() + } + runtimePanic("corrupted tree") + } + new.parent = old.parent + if old == t.root { + t.root = new + } else { + *new.parentSlot() = new + } +} + +// remove a node from the treap. +// This does not free the allocation. +func (t *memTreap) remove(n *memTreapNode) { +scan: + for { + switch { + case n.left == nil && n.right == nil && n.parent == nil: + // This is the only node - uproot it. + t.root = nil + break scan + case n.left == nil && n.right == nil: + // There are no nodes beneath here, so just remove this node from the parent. + *n.parentSlot() = nil + break scan + case n.left != nil && n.right == nil: + t.replace(n, n.left) + break scan + case n.right != nil && n.left == nil: + t.replace(n, n.right) + break scan + default: + // Rotate this node downward. + if n.left.priority() > n.right.priority() { + t.rotate(n.left, n) + } else { + t.rotate(n.right, n) + } + } + } + + n.left = nil + n.right = nil + n.parent = nil +} + +// memTreapNode is a treap node used to track allocations for the garbage collector. +// This struct is prepended to every allocation. +type memTreapNode struct { + parent, left, right *memTreapNode + size uintptr + base struct{} +} + +// priority computes a pseudo-random priority value for this treap node. +// This value is a fibonacci hash (https://en.wikipedia.org/wiki/Hash_function#Fibonacci_hashing) of the node's memory address. +func (n *memTreapNode) priority() uintptr { + // Select fibonacci multiplier for this bit-width. + var fibonacciMultiplier uint64 + switch 8 * unsafe.Sizeof(uintptr(0)) { + case 16: + fibonacciMultiplier = 40503 + case 32: + fibonacciMultiplier = 2654435769 + case 64: + fibonacciMultiplier = 11400714819323198485 + default: + runtimePanic("invalid size of uintptr") + } + + // Hash the pointer. + return uintptr(fibonacciMultiplier) * uintptr(unsafe.Pointer(n)) +} + +// contains returns whether this allocation contains a given address. +func (n *memTreapNode) contains(addr uintptr) bool { + return addr >= uintptr(unsafe.Pointer(&n.base)) && addr < uintptr(unsafe.Pointer(&n.base))+n.size +} + +// parentSlot returns a pointer to the parent's reference to this node. +func (n *memTreapNode) parentSlot() **memTreapNode { + if uintptr(unsafe.Pointer(n)) > uintptr(unsafe.Pointer(n.parent)) { + return &n.parent.left + } else { + return &n.parent.right + } +} + +// memScanQueue is a queue of memTreapNodes. +type memScanQueue struct { + head, tail *memTreapNode +} + +// push adds an allocation onto the queue. +func (q *memScanQueue) push(n *memTreapNode) { + if gcAsserts && (n.left != nil || n.right != nil || n.parent != nil) { + runtimePanic("tried to push a treap node that is in use") + } + + if q.head == nil { + q.tail = n + } else { + q.head.left = n + } + n.right = q.head + q.head = n +} + +// pop removes the next allocation from the queue. +func (q *memScanQueue) pop() *memTreapNode { + n := q.tail + q.tail = n.left + if q.tail == nil { + q.head = nil + } + n.left = nil + n.right = nil + return n +} + +// empty returns whether the queue contains any allocations. +func (q *memScanQueue) empty() bool { + return q.tail == nil +} + +// allocations is a treap containing all allocations. +var allocations memTreap + +// usedMem is the total amount of allocated memory (including the space taken up by memory treap nodes). +var usedMem uintptr + +// firstPtr and lastPtr are the bounds of memory used by the heap. +// They are computed before the collector starts marking, and are used to quickly eliminate false positives. +var firstPtr, lastPtr uintptr + +// scanQueue is a queue of marked allocations to scan. +var scanQueue memScanQueue + +// mark searches for an allocation containing the given address and marks it if found. +func mark(addr uintptr) bool { + if addr < firstPtr || addr > lastPtr { + // Pointer is outside of allocated bounds. + return false + } + + node := allocations.lookupAddr(addr) + if node != nil { + if gcDebug { + println("mark:", addr) + } + allocations.remove(node) + scanQueue.push(node) + } + + return node != nil +} + +func markRoot(addr uintptr, root uintptr) { + marked := mark(root) + if gcDebug { + if marked { + println("marked root:", root, "at", addr) + } else if addr != 0 { + println("did not mark root:", root, "at", addr) + } + } +} + +func markRoots(start uintptr, end uintptr) { + scan(start, end) +} + +// scan loads all pointer-aligned words and marks any pointers that it finds. +func scan(start uintptr, end uintptr) { + // Align start and end pointers. + start = (start + unsafe.Alignof(unsafe.Pointer(nil)) - 1) &^ (unsafe.Alignof(unsafe.Pointer(nil)) - 1) + end &^= unsafe.Alignof(unsafe.Pointer(nil)) - 1 + + // Mark all pointers. + for ptr := start; ptr < end; ptr += unsafe.Alignof(unsafe.Pointer(nil)) { + mark(*(*uintptr)(unsafe.Pointer(ptr))) + } +} + +// scan marks all allocations referenced by this allocation. +// This should only be invoked by the garbage collector. +func (n *memTreapNode) scan() { + start := uintptr(unsafe.Pointer(&n.base)) + end := start + n.size + scan(start, end) +} + +// destroy removes and frees all allocations in the treap. +func (t *memTreap) destroy() { + n := t.root + for n != nil { + switch { + case n.left != nil: + // Destroy the left subtree. + n = n.left + case n.right != nil: + // Destroy the right subtree. + n = n.right + default: + // This is a leaf node, so delete it and jump back to the parent. + + // Save the parent to jump back to. + parent := n.parent + + if parent != nil { + *n.parentSlot() = nil + } else { + t.root = nil + } + + // Update used memory. + usedMem -= unsafe.Sizeof(memTreapNode{}) + n.size + if gcDebug { + println("collecting:", &n.base, "size:", n.size) + println("used memory:", usedMem) + } + + // Free the node. + extfree(unsafe.Pointer(n)) + + // Jump back to the parent node. + n = parent + } + } +} + +// gcrunning is used by gcAsserts to determine whether the garbage collector is running. +// This is used to detect if the collector is invoking itself or trying to allocate memory. +var gcrunning bool + +// activeMem is a treap used to store marked allocations which have already been scanned. +// This is only used when the garbage collector is running. +var activeMem memTreap + +func GC() { + if gcDebug { + println("running GC") + } + if allocations.empty() { + // Skip collection because the heap is empty. + if gcDebug { + println("nothing to collect") + } + return + } + + if gcAsserts { + if gcrunning { + runtimePanic("GC called itself") + } + gcrunning = true + } + + if gcDebug { + println("pre-GC allocations:") + allocations.print() + } + + // Before scanning, find the lowest and highest allocated pointers. + // These can be quickly compared against to eliminate most false positives. + firstPtr, lastPtr = allocations.minAddr(), allocations.maxAddr() + + // Start by scanning all of the global variables and the stack. + markGlobals() + markStack() + + // Scan all referenced allocations, building a new treap with marked allocations. + // The marking process deletes the allocations from the old allocations treap, so they are only queued once. + for !scanQueue.empty() { + // Pop a marked node off of the scan queue. + n := scanQueue.pop() + + // Scan and mark all nodes that this references. + n.scan() + + // Insert this node into the new treap. + activeMem.insert(n) + } + + // The allocations treap now only contains unreferenced nodes. Destroy them all. + allocations.destroy() + if gcAsserts && !allocations.empty() { + runtimePanic("failed to fully destroy allocations") + } + + // Replace the allocations treap with the new treap. + allocations = activeMem + activeMem = memTreap{} + + if gcDebug { + println("GC finished") + } + + if gcAsserts { + gcrunning = false + } +} + +// heapBound is used to control the growth of the heap. +// When the heap exceeds this size, the garbage collector is run. +// If the garbage collector cannot free up enough memory, the bound is doubled until the allocation fits. +var heapBound uintptr = 4 * unsafe.Sizeof(memTreapNode{}) + +// zeroSizedAlloc is just a sentinel that gets returned when allocating 0 bytes. +var zeroSizedAlloc uint8 + +// alloc tries to find some free space on the heap, possibly doing a garbage +// collection cycle if needed. If no space is free, it panics. +//go:noinline +func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer { + if size == 0 { + return unsafe.Pointer(&zeroSizedAlloc) + } + + if gcAsserts && gcrunning { + runtimePanic("allocated inside the garbage collector") + } + + // Calculate size of allocation including treap node. + allocSize := unsafe.Sizeof(memTreapNode{}) + size + + var gcRan bool + for { + // Try to bound heap growth. + if usedMem+allocSize < usedMem { + if gcDebug { + println("current mem:", usedMem, "alloc size:", allocSize) + } + runtimePanic("target heap size exceeds address space size") + } + if usedMem+allocSize > heapBound { + if !gcRan { + // Run the garbage collector before growing the heap. + if gcDebug { + println("heap reached size limit") + } + GC() + gcRan = true + continue + } else { + // Grow the heap bound to fit the allocation. + for heapBound != 0 && usedMem+allocSize > heapBound { + heapBound <<= 1 + } + if heapBound == 0 { + // This is only possible on hosted 32-bit systems. + // Allow the heap bound to encompass everything. + heapBound = ^uintptr(0) + } + if gcDebug { + println("raising heap size limit to", heapBound) + } + } + } + + // Allocate the memory. + ptr := extalloc(allocSize) + if ptr == nil { + if gcDebug { + println("extalloc failed") + } + if gcRan { + // Garbage collector was not able to free up enough memory. + runtimePanic("out of memory") + } else { + // Run the garbage collector and try again. + GC() + gcRan = true + continue + } + } + + // Initialize the memory treap node. + node := (*memTreapNode)(ptr) + *node = memTreapNode{ + size: size, + } + + // Insert allocation into the allocations treap. + allocations.insert(node) + + // Extract the user's section of the allocation. + ptr = unsafe.Pointer(&node.base) + if gcAsserts && !node.contains(uintptr(ptr)) { + runtimePanic("node is not self-contained") + } + if gcAsserts { + check := allocations.lookupAddr(uintptr(ptr)) + if check == nil { + if gcDebug { + println("failed to find:", ptr) + allocations.print() + } + runtimePanic("bad insert") + } + } + + // Zero the allocation. + memzero(ptr, size) + + // Update used memory. + usedMem += allocSize + + if gcDebug { + println("allocated:", uintptr(ptr), "size:", size) + println("used memory:", usedMem) + } + + return ptr + } +} + +func free(ptr unsafe.Pointer) { + // Currently unimplemented due to bugs in coroutine lowering. +} + +func KeepAlive(x interface{}) { + // Unimplemented. Only required with SetFinalizer(). +} + +func SetFinalizer(obj interface{}, finalizer interface{}) { + // Unimplemented. +} diff --git a/src/runtime/gc_globals.go b/src/runtime/gc_globals.go index 85efa4a0b4..e28d628297 100644 --- a/src/runtime/gc_globals.go +++ b/src/runtime/gc_globals.go @@ -1,4 +1,4 @@ -//go:build gc.conservative && (baremetal || tinygo.wasm) +//go:build gc.conservative && (baremetal || tinygo.wasm || polkawasm) package runtime diff --git a/src/runtime/gc_stack_portable.go b/src/runtime/gc_stack_portable.go index 995975809a..ebb469a502 100644 --- a/src/runtime/gc_stack_portable.go +++ b/src/runtime/gc_stack_portable.go @@ -1,4 +1,4 @@ -//go:build gc.conservative && tinygo.wasm +//go:build (gc.conservative || gc.extalloc) && tinygo.wasm package runtime diff --git a/src/runtime/runtime_polkawasm.go b/src/runtime/runtime_polkawasm.go index a4b708fe36..7336b6a18f 100644 --- a/src/runtime/runtime_polkawasm.go +++ b/src/runtime/runtime_polkawasm.go @@ -75,3 +75,19 @@ func procPin() { //go:linkname procUnpin sync/atomic.runtime_procUnpin func procUnpin() { } + +//go:wasm-module env +//go:export ext_allocator_malloc_version_1 +func extalloc(size uintptr) unsafe.Pointer + +//go:wasm-module env +//go:export ext_allocator_free_version_1 +func extfree(ptr unsafe.Pointer) + +func markGlobals() { + markRoots(globalsStart, globalsEnd) +} + +func setHeapEnd(newHeapEnd uintptr) { + // Nothing to do here, this function is never actually called. +} diff --git a/targets/polkawasm.json b/targets/polkawasm.json index 9063b50ea2..31ff26bebd 100644 --- a/targets/polkawasm.json +++ b/targets/polkawasm.json @@ -5,7 +5,7 @@ "build-tags": ["tinygo.wasm", "polkawasm", "runtime_memhash_leveldb"], "goos": "linux", "goarch": "arm", - "gc": "conservative", + "gc": "extalloc", "opt": "0", "linker": "wasm-ld", "libc": "",