diff --git a/NOTICE b/NOTICE index 93a368305e..d745fe7ceb 100644 --- a/NOTICE +++ b/NOTICE @@ -29,6 +29,7 @@ under the licensing terms detailed in LICENSE: * Valeria Viana Gusmao * Gabor Greif * Martin Fredriksson +* David Schneider * forcepusher * Piotr Oleś * Saúl Cabrera diff --git a/cli/index.js b/cli/index.js index 36eeb4715e..ad0eb45af6 100644 --- a/cli/index.js +++ b/cli/index.js @@ -300,6 +300,7 @@ export async function main(argv, options) { switch (opts.runtime) { case "stub": runtime = 0; break; case "minimal": runtime = 1; break; + case "minimal-mt": runtime = 4; break; /* incremental */ default: runtime = 2; break; } diff --git a/std/assembly/rt/index-incremental.ts b/std/assembly/rt/index-incremental.ts index 4730344b4f..f18c8791db 100644 --- a/std/assembly/rt/index-incremental.ts +++ b/std/assembly/rt/index-incremental.ts @@ -1,2 +1,3 @@ -import "rt/tlsf"; -import "rt/itcms"; +import "rt/tlsf-base"; +import "rt/tlsf-st"; +import "rt/itcms"; diff --git a/std/assembly/rt/index-minimal-mt.ts b/std/assembly/rt/index-minimal-mt.ts new file mode 100644 index 0000000000..3595ba2dd6 --- /dev/null +++ b/std/assembly/rt/index-minimal-mt.ts @@ -0,0 +1,3 @@ +import "rt/tlsf-base"; +import "rt/tlsf-mt"; +import "rt/tcms"; diff --git a/std/assembly/rt/index-minimal.ts b/std/assembly/rt/index-minimal.ts index cf88ee158f..c020f7ed40 100644 --- a/std/assembly/rt/index-minimal.ts +++ b/std/assembly/rt/index-minimal.ts @@ -1,2 +1,3 @@ -import "rt/tlsf"; -import "rt/tcms"; +import "rt/tlsf-base"; +import "rt/tlsf-st"; +import "rt/tcms"; diff --git a/std/assembly/rt/tlsf.ts b/std/assembly/rt/tlsf-base.ts similarity index 96% rename from std/assembly/rt/tlsf.ts rename to std/assembly/rt/tlsf-base.ts index df437b82cb..f325252ea9 100644 --- a/std/assembly/rt/tlsf.ts +++ b/std/assembly/rt/tlsf-base.ts @@ -5,6 +5,9 @@ import { E_ALLOCATION_TOO_LARGE } from "../util/error"; // === The TLSF (Two-Level Segregate Fit) memory allocator === // see: http://www.gii.upv.es/tlsf/ +// Split into single- and multi-threaded versions, the multi-threaded version just adds basic locks around +// allocation and deallocation. + // - `ffs(x)` is equivalent to `ctz(x)` with x != 0 // - `fls(x)` is equivalent to `sizeof(x) * 8 - clz(x) - 1` @@ -137,7 +140,10 @@ import { E_ALLOCATION_TOO_LARGE } from "../util/error"; @inline const ROOT_SIZE: usize = HL_END + sizeof(); // @ts-ignore: decorator -@lazy export let ROOT: Root = changetype(0); // unsafe initializion below +@lazy export let ROOT: Root = changetype(memory.data(ROOT_SIZE)); // unsafe initializion below + +// @ts-ignore: decorator +@inline export const ROOT_INIT: usize = memory.data(4); /** Gets the second level map of the specified first level. */ // @ts-ignore: decorator @@ -460,13 +466,13 @@ function prepareSize(size: usize): usize { } /** Initializes the root structure. */ -function initialize(): void { +export function TLSFinitialize(): void { if (isDefined(ASC_RTRACE)) oninit(__heap_base); let rootOffset = (__heap_base + AL_MASK) & ~AL_MASK; let pagesBefore = memory.size(); let pagesNeeded = ((((rootOffset + ROOT_SIZE) + 0xffff) & ~0xffff) >>> 16); if (pagesNeeded > pagesBefore && memory.grow(pagesNeeded - pagesBefore) < 0) unreachable(); - let root = changetype(rootOffset); + let root = ROOT; root.flMap = 0; SETTAIL(root, changetype(0)); for (let fl: usize = 0; fl < FL_BITS; ++fl) { @@ -483,7 +489,7 @@ function initialize(): void { } else { addMemory(root, memStart, memory.size() << 16); } - ROOT = root; + store(ROOT_INIT, 1); } /** Allocates a block of the specified size. */ @@ -536,7 +542,7 @@ export function reallocateBlock(root: Root, block: Block, size: usize): Block { } /** Moves a block to a new one of the specified size. */ -function moveBlock(root: Root, block: Block, newSize: usize): Block { +export function moveBlock(root: Root, block: Block, newSize: usize): Block { let newBlock = allocateBlock(root, newSize); memory.copy(changetype(newBlock) + BLOCK_OVERHEAD, changetype(block) + BLOCK_OVERHEAD, block.mmInfo & ~TAGS_MASK); if (changetype(block) >= __heap_base) { @@ -554,7 +560,7 @@ export function freeBlock(root: Root, block: Block): void { } /** Checks that a used block is valid to be freed or reallocated. */ -function checkUsedBlock(ptr: usize): Block { +export function checkUsedBlock(ptr: usize): Block { let block = changetype(ptr - BLOCK_OVERHEAD); assert( ptr != 0 && !(ptr & AL_MASK) && // must exist and be aligned @@ -563,27 +569,3 @@ function checkUsedBlock(ptr: usize): Block { return block; } -// @ts-ignore: decorator -@global @unsafe -export function __alloc(size: usize): usize { - if (!ROOT) initialize(); - return changetype(allocateBlock(ROOT, size)) + BLOCK_OVERHEAD; -} - -// @ts-ignore: decorator -@global @unsafe -export function __realloc(ptr: usize, size: usize): usize { - if (!ROOT) initialize(); - return (ptr < __heap_base - ? changetype(moveBlock(ROOT, checkUsedBlock(ptr), size)) - : changetype(reallocateBlock(ROOT, checkUsedBlock(ptr), size)) - ) + BLOCK_OVERHEAD; -} - -// @ts-ignore: decorator -@global @unsafe -export function __free(ptr: usize): void { - if (ptr < __heap_base) return; - if (!ROOT) initialize(); - freeBlock(ROOT, checkUsedBlock(ptr)); -} diff --git a/std/assembly/rt/tlsf-mt.ts b/std/assembly/rt/tlsf-mt.ts new file mode 100644 index 0000000000..c1d3a1155a --- /dev/null +++ b/std/assembly/rt/tlsf-mt.ts @@ -0,0 +1,45 @@ +import {BLOCK_OVERHEAD} from "./common"; +import {allocateBlock, freeBlock, reallocateBlock, ROOT, ROOT_INIT, TLSFinitialize, checkUsedBlock, moveBlock} from "./tlsf-base"; +import {TlsfMutex_lock, TlsfMutex_unlock} from './tlsf-mutex' + +const mutex_ptr = memory.data(4, 16); + +// @ts-ignore: decorator +@global @unsafe +export function __alloc(size: usize): usize { + TlsfMutex_lock(mutex_ptr); + + if (!load(ROOT_INIT)) TLSFinitialize(); + let r: usize = changetype(allocateBlock(ROOT, size)) + BLOCK_OVERHEAD; + + TlsfMutex_unlock(mutex_ptr); + return r; +} + +// @ts-ignore: decorator +@global @unsafe +export function __realloc(ptr: usize, size: usize): usize { + TlsfMutex_lock(mutex_ptr); + + if (!load(ROOT_INIT)) TLSFinitialize(); + let r: usize = (ptr < __heap_base + ? changetype(moveBlock(ROOT, checkUsedBlock(ptr), size)) + : changetype(reallocateBlock(ROOT, checkUsedBlock(ptr), size)) + ) + BLOCK_OVERHEAD; + + TlsfMutex_unlock(mutex_ptr); + return r; +} + +// @ts-ignore: decorator +@global @unsafe +export function __free(ptr: usize): void { + if (ptr < __heap_base) return; + + TlsfMutex_lock(mutex_ptr); + + if (!load(ROOT_INIT)) TLSFinitialize(); + freeBlock(ROOT, checkUsedBlock(ptr)); + + TlsfMutex_unlock(mutex_ptr); +} diff --git a/std/assembly/rt/tlsf-mutex.ts b/std/assembly/rt/tlsf-mutex.ts new file mode 100644 index 0000000000..d323b209b8 --- /dev/null +++ b/std/assembly/rt/tlsf-mutex.ts @@ -0,0 +1,30 @@ +// This just implements a super-simple lock for tlsf-mt.ts + +enum TlsfMutexState { + unlocked, + locked +} + +// Basic spinlock. Spinning is not a performance issue since this only takes as long as an allocation +// @ts-ignore: decorator +@inline +export function TlsfMutex_lock(mutex_ptr: usize): void { + for (; ;) { + // If we succesfully atomically compare and exchange unlocked for locked, we have the mutex + if (atomic.cmpxchg(mutex_ptr, TlsfMutexState.unlocked, TlsfMutexState.locked) === TlsfMutexState.unlocked) + return; + // Wait for unlocked state to try for locked + for (; ;) { + if (atomic.load(mutex_ptr) === TlsfMutexState.unlocked) break; + } + } +} + +// @ts-ignore: decorator +@inline +export function TlsfMutex_unlock(mutex_ptr: usize): void { + if (atomic.cmpxchg(mutex_ptr, TlsfMutexState.locked, TlsfMutexState.unlocked) !== TlsfMutexState.locked) { + // This only happens if someone else unlocked our mutex, or we did it more than once... + throw new Error('Is this the right thing to do here? Mutex in inconsistent state'); + } +} diff --git a/std/assembly/rt/tlsf-st.ts b/std/assembly/rt/tlsf-st.ts new file mode 100644 index 0000000000..95dd0e1600 --- /dev/null +++ b/std/assembly/rt/tlsf-st.ts @@ -0,0 +1,28 @@ +import {BLOCK_OVERHEAD} from "./common"; +import {allocateBlock, freeBlock, reallocateBlock, ROOT, ROOT_INIT, TLSFinitialize, checkUsedBlock, moveBlock} from "./tlsf-base"; + +// @ts-ignore: decorator +@global @unsafe +export function __alloc(size: usize): usize { + if (!load(ROOT_INIT)) TLSFinitialize(); + + return changetype(allocateBlock(ROOT, size)) + BLOCK_OVERHEAD; +} + +// @ts-ignore: decorator +@global @unsafe +export function __realloc(ptr: usize, size: usize): usize { + if (!load(ROOT_INIT)) TLSFinitialize(); + return (ptr < __heap_base + ? changetype(moveBlock(ROOT, checkUsedBlock(ptr), size)) + : changetype(reallocateBlock(ROOT, checkUsedBlock(ptr), size)) + ) + BLOCK_OVERHEAD; +} + +// @ts-ignore: decorator +@global @unsafe +export function __free(ptr: usize): void { + if (ptr < __heap_base) return; + if (!load(ROOT_INIT)) TLSFinitialize(); + freeBlock(ROOT, checkUsedBlock(ptr)); +}