[wasm] Add custom mmap/munmap implementation for anonymous mappings

src/mono/mono/sgen/sgen-internal.c

@@ -273,7 +273,7 @@ sgen_report_internal_mem_usage (void)
 void
 sgen_init_internal_allocator (void)
 {
-	int i, size;
+	int i;
 
 	for (i = 0; i < INTERNAL_MEM_MAX; ++i)
 		fixed_type_allocator_indexes [i] = -1;
@@ -284,7 +284,10 @@ sgen_init_internal_allocator (void)
 		mono_lock_free_allocator_init_allocator (&allocators [i], &size_classes [i], MONO_MEM_ACCOUNT_SGEN_INTERNAL);
 	}
 
-	for (size = mono_pagesize (); size <= LOCK_FREE_ALLOC_SB_MAX_SIZE; size <<= 1) {
+	// FIXME: This whole algorithm is broken on WASM due to its 64KB page size.
+	// Previously SB_MAX_SIZE was < mono_pagesize, so none of this ran.
+#ifndef HOST_WASM
+	for (int size = mono_pagesize (); size <= LOCK_FREE_ALLOC_SB_MAX_SIZE; size <<= 1) {
 		int max_size = (LOCK_FREE_ALLOC_SB_USABLE_SIZE (size) / 2) & ~(SIZEOF_VOID_P - 1);
 		/*
 		 * we assert that allocator_sizes contains the biggest possible object size
@@ -297,6 +300,7 @@ sgen_init_internal_allocator (void)
 		if (size < LOCK_FREE_ALLOC_SB_MAX_SIZE)
 			g_assert (block_size (max_size + 1) == size << 1);
 	}
+#endif
 }
 
 #endif

src/mono/mono/sgen/sgen-marksweep.c

@@ -33,6 +33,7 @@
 #include "mono/sgen/sgen-client.h"
 #include "mono/utils/mono-memory-model.h"
 #include "mono/utils/mono-proclib.h"
+#include "mono/utils/options.h"
 
 static int ms_block_size;
 
@@ -2133,12 +2134,15 @@ major_free_swept_blocks (size_t section_reserve)
 {
 	SGEN_ASSERT (0, sweep_state == SWEEP_STATE_SWEPT, "Sweeping must have finished before freeing blocks");
 
-#if defined(HOST_WIN32) || defined(HOST_ORBIS) || defined (HOST_WASM)
+#if defined(HOST_WIN32) || defined(HOST_ORBIS)
 		/*
 		 * sgen_free_os_memory () asserts in mono_vfree () because windows doesn't like freeing the middle of
 		 * a VirtualAlloc ()-ed block.
 		 */
 		return;
+#elif defined(HOST_WASM)
+	if (!mono_opt_wasm_mmap)
+		return;
 #endif
 
 	{

src/mono/mono/utils/CMakeLists.txt

@@ -206,7 +206,7 @@ set(utils_arch_sources "${utils_arch_sources};mono-hwcap-riscv.c")
 elseif(TARGET_S390X)
 set(utils_arch_sources "${utils_arch_sources};mono-hwcap-s390x.c")
 elseif(TARGET_WASM)
-set(utils_arch_sources "${utils_arch_sources};mono-hwcap-wasm.c;mono-mmap-wasm.c")
+set(utils_arch_sources "${utils_arch_sources};mono-hwcap-wasm.c;mono-mmap-wasm.c;mono-wasm-pagemgr.c")
 elseif(TARGET_WASI)
 set(utils_arch_sources "${utils_arch_sources};mono-hwcap-wasm.c")
 elseif(TARGET_POWERPC OR TARGET_POWERPC64)

src/mono/mono/utils/lock-free-alloc.h

@@ -45,6 +45,7 @@ typedef struct {
 	MonoMemAccountType account_type;
 } MonoLockFreeAllocator;
 
+// FIXME: On WASM the page size is 64KB, so this isn't enough.
 #define LOCK_FREE_ALLOC_SB_MAX_SIZE					16384
 #define LOCK_FREE_ALLOC_SB_HEADER_SIZE				(sizeof (gpointer))
 #define LOCK_FREE_ALLOC_SB_USABLE_SIZE(block_size)	((block_size) - LOCK_FREE_ALLOC_SB_HEADER_SIZE)

src/mono/mono/utils/mono-mmap-wasm.c

@@ -22,6 +22,9 @@
 #include "mono-proclib.h"
 #include <mono/utils/mono-threads.h>
 #include <mono/utils/atomic.h>
+#include <mono/utils/options.h>
+
+#include "mono-wasm-pagemgr.h"
 
 #define BEGIN_CRITICAL_SECTION do { \
 	MonoThreadInfo *__info = mono_thread_info_current_unchecked (); \
@@ -34,6 +37,9 @@
 int
 mono_pagesize (void)
 {
+	if (mono_opt_wasm_mmap)
+		return MWPM_PAGE_SIZE;
+
 	static int saved_pagesize = 0;
 
 	if (saved_pagesize)
@@ -108,7 +114,16 @@ valloc_impl (void *addr, size_t size, int flags, MonoMemAccountType type)
 	mflags |= MAP_PRIVATE;
 
 	BEGIN_CRITICAL_SECTION;
-	ptr = mmap (addr, size, prot, mflags, -1, 0);
+	if (mono_opt_wasm_mmap) {
+		// FIXME: Make this work if the requested address range is free
+		if ((flags & MONO_MMAP_FIXED) && addr)
+			return NULL;
+
+		ptr = mwpm_alloc_range (size, 1);
+		if (!ptr)
+			return NULL;
+	} else
+		ptr = mmap (addr, size, prot, mflags, -1, 0);
 	END_CRITICAL_SECTION;
 
 	if (ptr == MAP_FAILED)
@@ -142,6 +157,10 @@ typedef struct {
 void*
 mono_valloc_aligned (size_t size, size_t alignment, int flags, MonoMemAccountType type)
 {
+	// We don't need padding if the alignment is compatible with the page size
+	if (mono_opt_wasm_mmap && ((MWPM_PAGE_SIZE % alignment) == 0))
+		return valloc_impl (NULL, size, flags, type);
+
 	/* Allocate twice the memory to be able to put the block on an aligned address */
 	char *mem = (char *) valloc_impl (NULL, size + alignment, flags, type);
 	char *aligned;
@@ -175,13 +194,22 @@ mono_vfree (void *addr, size_t length, MonoMemAccountType type)
 		 * mono_valloc_align (), free the original mapping.
 		 */
 		BEGIN_CRITICAL_SECTION;
-		munmap (info->addr, info->size);
+		if (mono_opt_wasm_mmap)
+			mwpm_free_range (info->addr, info->size);
+		else
+			munmap (info->addr, info->size);
 		END_CRITICAL_SECTION;
 		g_free (info);
 		g_hash_table_remove (valloc_hash, addr);
 	} else {
+		// FIXME: We could be trying to unmap part of an aligned mapping, in which case the
+		//  hash lookup failed because addr isn't exactly the start of the mapping.
+		// Ideally if the custom page manager is enabled, we won't have done aligned alloc.
 		BEGIN_CRITICAL_SECTION;
-		munmap (addr, length);
+		if (mono_opt_wasm_mmap)
+			mwpm_free_range (addr, length);
+		else
+			munmap (addr, length);
 		END_CRITICAL_SECTION;
 	}
 

src/mono/mono/utils/mono-mmap.h

@@ -22,7 +22,8 @@ enum {
 	MONO_MMAP_ANON    = 1 << 6,
 	MONO_MMAP_FIXED   = 1 << 7,
 	MONO_MMAP_32BIT   = 1 << 8,
-	MONO_MMAP_JIT     = 1 << 9
+	MONO_MMAP_JIT     = 1 << 9,
+	MONO_MMAP_NOZERO  = 1 << 10,
 };
 
 typedef enum {