From 096c3c939b678a1008d460a5a5cb35b1e05997b9 Mon Sep 17 00:00:00 2001
From: Eduardo Bart <edub4rt@gmail.com>
Date: Sat, 8 Feb 2020 16:30:04 -0300
Subject: [PATCH] Introduce allocator system

---
 docs/pages/overview.md                   |   7 +-
 examples/linkedlist.nelua                |  13 +-
 examples/overview.nelua                  |   5 +-
 lib/allocators/allocator_interface.nelua | 109 +++++++++++++++
 lib/allocators/gc_allocator.nelua        |  41 ++++++
 lib/allocators/generic_allocator.nelua   |  49 +++++++
 lib/gc.nelua                             |  61 +++++---
 lib/memory.nelua                         | 128 ++---------------
 lib/myarraytable.nelua                   | 168 +++++++++++++----------
 nelua/ppcontext.lua                      |   4 +-
 tests/memory_test.nelua                  | 168 ++++++++++++-----------
 11 files changed, 455 insertions(+), 298 deletions(-)
 create mode 100644 lib/allocators/allocator_interface.nelua
 create mode 100644 lib/allocators/gc_allocator.nelua
 create mode 100644 lib/allocators/generic_allocator.nelua

diff --git a/docs/pages/overview.md b/docs/pages/overview.md
index f011d67a..1beb2db1 100644
--- a/docs/pages/overview.md
+++ b/docs/pages/overview.md
@@ -790,14 +790,15 @@ print($ap) -- outputs 3
 Memory can be allocated using C malloc and free.
 
 ```nelua
-require 'C.stdlib'
+require 'memory'
+require 'allocators.generic_allocator'
 
 local Person = @record{name: string, age: integer}
-local p = (@Person*)(C.malloc(#Person))
+local p: Person* = generic_allocator.new(@Person)
 p.name = "John"
 p.age = 20
 print(p.name, p.age)
-C.free(p)
+generic_allocator.delete(p)
 p = nilptr
 ```
 
diff --git a/examples/linkedlist.nelua b/examples/linkedlist.nelua
index f3ae19fc..679702ef 100644
--- a/examples/linkedlist.nelua
+++ b/examples/linkedlist.nelua
@@ -1,6 +1,9 @@
 ## strict = true
 
 require 'memory'
+require 'allocators.generic_allocator'
+
+local allocator = @generic_allocator
 
 ## local memoize = require 'nelua.utils.memoize'
 ## local list = hygienize(memoize(function(T)
@@ -12,7 +15,7 @@ require 'memory'
   }
 
   local function new_node(prev: ListNode*, next: ListNode*, value: T): ListNode*
-    local node = memory.new(@ListNode)
+    local node = allocator.new(@ListNode)
     node.prev = prev
     node.next = next
     node.value = value
@@ -77,7 +80,7 @@ require 'memory'
       node.next.prev = node.prev
     end
     local next = node.next
-    memory.delete(node)
+    allocator.delete(node)
     return next
   end
 
@@ -91,7 +94,7 @@ require 'memory'
     if unlikely(self.tail == node) then
       self.tail = nilptr
     end
-    memory.delete(node)
+    allocator.delete(node)
   end
 
   function List:remove_last()
@@ -104,14 +107,14 @@ require 'memory'
     if unlikely(self.head == node) then
       self.head = nilptr
     end
-    memory.delete(node)
+    allocator.delete(node)
   end
 
   function List:clear()
     local it = self.head
     while it do
       local next = it.next
-      memory.delete(it)
+      allocator.delete(it)
       it = next
     end
     self.head = nilptr
diff --git a/examples/overview.nelua b/examples/overview.nelua
index 2d32631e..4fa60cbf 100644
--- a/examples/overview.nelua
+++ b/examples/overview.nelua
@@ -393,14 +393,15 @@ do -- Dereferencing and referencing
 end
 
 require 'memory'
+require 'allocators.generic_allocator'
 
 do -- Allocating memory
   local Person = @record{name: string, age: integer}
-  local p: Person* = memory.new(@Person)
+  local p: Person* = generic_allocator.new(@Person)
   p.name = "John"
   p.age = 20
   print(p.name, p.age)
-  memory.delete(p)
+  generic_allocator.delete(p)
   p = nilptr
 end
 
diff --git a/lib/allocators/allocator_interface.nelua b/lib/allocators/allocator_interface.nelua
new file mode 100644
index 00000000..6d7c62c1
--- /dev/null
+++ b/lib/allocators/allocator_interface.nelua
@@ -0,0 +1,109 @@
+##[[
+strict = true
+unitname = 'nelua'
+
+--------------------------------------------------------------------------------
+-- compile time checks utilities
+local function check_span_subtype(v)
+  staticassert(v.type:is_span(),
+    "in argument '%s': expected 'span(T)' but got type '%s'",
+    v.name, v.type)
+  return v.type.subtype
+end
+
+local function check_type_match(a, b)
+  staticassert(a.type == b.type,
+    "argument '%s' type '%s' differs from argument '%s' type '%s",
+    a.name, b.name, a.type, b.type)
+end
+
+local function check_span_subtype_match(s, x)
+  local stype = check_span_subtype(s)
+  staticassert(stype == x.type,
+    "in argument '%s': type '%s' differs from argument '%s' type '%s",
+    s.name, x.name, stype, x.type)
+end
+]]
+
+## function implement_allocator_interface(allocator)
+
+local allocator = #[allocator]#
+
+function allocator.spanalloc(T: type, size: usize)
+  local s: span(T)
+  if likely(size > 0_u) then
+    s.data = (@T*)(allocator.alloc(size * #T))
+    check(s.data, 'allocator.spanalloc: allocation fail')
+    s.size = size
+  end
+  return s
+end
+
+function allocator.spanalloc0(T: type, size: usize)
+  local s: span(T)
+  if likely(size > 0_u) then
+    s.data = (@T*)(allocator.alloc0(size * #T))
+    check(s.data, 'allocator.spanalloc0: allocation fail')
+    s.size = size
+  end
+  return s
+end
+
+function allocator.spanrealloc(s: auto, size: usize)
+  local T: type = #[check_span_subtype(s)]#
+  if unlikely(size == 0) then
+    allocator.dealloc(s.data)
+    s.data = nilptr
+    s.size = 0
+  else
+    s.data = (@T*)(allocator.realloc(s.data, size * #T))
+    check(s.data, 'allocator.spanrealloc0: allocation fail')
+    s.size = size
+  end
+  return s
+end
+
+function allocator.spanrealloc0(s: auto, size: usize)
+  local T: type = #[check_span_subtype(s)]#
+  if unlikely(size == 0) then
+    allocator.dealloc(s.data)
+    s.data = nilptr
+    s.size = 0
+  else
+    s.data = (@T*)(allocator.realloc(s.data, size * #T))
+    check(s.data, 'allocator.spanrealloc0: allocation fail')
+    if likely(size > s.size) then
+      memset(&s[s.size], 0, (size - s.size) * #T)
+    end
+    s.size = size
+  end
+  return s
+end
+
+function allocator.spandealloc(s: auto)
+  ## check_span_subtype(s)
+  allocator.dealloc(s.data)
+end
+
+function allocator.new(T: type, size: auto)
+  ## if not size.type:is_nil() then
+    ## staticassert(size.type:is_integral(), 'allocator.new: size must be an integral type')
+    ## if not size.type:is_unsigned() then
+      check(size > 0, 'allocator.new: size must be greater than 0')
+    ## end
+    return allocator.spanalloc0(T, (@usize)(size))
+  ## else
+    return (@T*)(allocator.alloc0(#T))
+  ## end
+end
+
+function allocator.delete(s: auto)
+  ## staticassert(s.type:is_pointer() or s.type:is_span(), "allocator.delete: invalid type '%s'", s.type)
+  ## if s.type:is_span() then
+    allocator.spandealloc(s)
+  ## else -- pointer
+    allocator.dealloc(s)
+  ## end
+end
+
+## end
diff --git a/lib/allocators/gc_allocator.nelua b/lib/allocators/gc_allocator.nelua
new file mode 100644
index 00000000..fd8d6b64
--- /dev/null
+++ b/lib/allocators/gc_allocator.nelua
@@ -0,0 +1,41 @@
+## strict = true
+## unitname = 'nelua'
+
+require 'gc'
+require 'allocators.allocator_interface'
+
+global gc_allocator = @record{}
+
+function gc_allocator.alloc(size: usize): pointer
+  check(size > 0_u, 'gc_allocator.alloc: size cannot be zero')
+  local p: pointer = gc:alloc(size)
+  check(p, 'gc_allocator.alloc: allocation fail')
+  return p
+end
+
+function gc_allocator.alloc0(size: usize): pointer
+  check(size > 0_u, 'gc_allocator.alloc0: size must be greater than 0')
+  local p: pointer = gc:alloc0(size)
+  check(p, 'gc_allocator.alloc0: allocation fail')
+  return p
+end
+
+function gc_allocator.realloc(p: pointer, size: usize): pointer
+  check(size > 0_u, 'gc_allocator.realloc: size must be greater than 0')
+  p = gc:realloc(p, size)
+  check(size == 0 or p, 'gc_allocator.realloc: allocation fail')
+  return p
+end
+
+function gc_allocator.realloc0(p: pointer, newsize: usize, oldsize: usize): pointer
+  check(newsize > 0_u, 'gc_allocator.realloc0: size must be greater than 0')
+  p = gc:realloc0(p, newsize, oldsize)
+  check(newsize == 0_u or p, 'gc_allocator.realloc0: allocation fail')
+  return p
+end
+
+function gc_allocator.dealloc(p: pointer)
+  gc:dealloc(p)
+end
+
+## implement_allocator_interface(gc_allocator)
diff --git a/lib/allocators/generic_allocator.nelua b/lib/allocators/generic_allocator.nelua
new file mode 100644
index 00000000..0ad1046c
--- /dev/null
+++ b/lib/allocators/generic_allocator.nelua
@@ -0,0 +1,49 @@
+## strict = true
+## unitname = 'nelua'
+
+require 'allocators.allocator_interface'
+
+local function memset(s: pointer, c: cint, n: csize): pointer <cimport'memset',cinclude'<string.h>',nodecl> end
+local function malloc(size: csize): pointer <cimport'malloc',cinclude'<stdlib.h>',nodecl> end
+local function calloc(nmemb: csize, size: csize): pointer <cimport'calloc',cinclude'<stdlib.h>',nodecl> end
+local function realloc(ptr: pointer, size: csize): pointer <cimport'realloc',cinclude'<stdlib.h>',nodecl> end
+local function free(ptr: pointer): void <cimport'free',cinclude'<stdlib.h>',nodecl> end
+
+global generic_allocator = @record{}
+
+function generic_allocator.alloc(size: usize): pointer
+  check(size > 0_u, 'generic_allocator.alloc: size cannot be zero')
+  local p: pointer = malloc(size)
+  check(p, 'generic_allocator.alloc: allocation fail')
+  return p
+end
+
+function generic_allocator.alloc0(size: usize): pointer
+  check(size > 0_u, 'generic_allocator.alloc0: size must be greater than 0')
+  local p: pointer = calloc(size, 1)
+  check(p, 'generic_allocator.alloc0: allocation fail')
+  return p
+end
+
+function generic_allocator.realloc(p: pointer, size: usize): pointer
+  check(size > 0_u, 'generic_allocator.realloc: size must be greater than 0')
+  p = realloc(p, size)
+  check(size == 0 or p, 'generic_allocator.realloc: allocation fail')
+  return p
+end
+
+function generic_allocator.realloc0(p: pointer, newsize: usize, oldsize: usize): pointer
+  check(newsize > 0_u, 'generic_allocator.realloc0: size must be greater than 0')
+  p = realloc(p, newsize)
+  check(newsize == 0_u or p, 'generic_allocator.realloc0: allocation fail')
+  if likely(newsize > oldsize) then
+    memset(&(@byte[0]*)(p)[oldsize], 0, newsize - oldsize)
+  end
+  return p
+end
+
+function generic_allocator.dealloc(p: pointer)
+  free(p)
+end
+
+## implement_allocator_interface(generic_allocator)
diff --git a/lib/gc.nelua b/lib/gc.nelua
index e6a29569..1a403321 100644
--- a/lib/gc.nelua
+++ b/lib/gc.nelua
@@ -1,17 +1,18 @@
 -- Tiny and simple mark and sweep garbage collector
 -- Implementation based on https://github.com/orangeduck/tgc
 
-##[[
-strict = true
-unitname = 'nelua'
-cinclude '<setjmp.h>'
-]]
+## strict = true
+## unitname = 'nelua'
+## cinclude '<setjmp.h>'
 
 require 'memory'
+require 'allocators.generic_allocator'
 
 local jmp_buf <cimport,nodecl> = @record{dummy: usize}
 local function setjmp(env: jmp_buf) <cimport,nodecl> end
 
+local allocator = @generic_allocator
+
 local GCFlags = @enum(usize) {
   MARK = 1,
   ROOT = 2,
@@ -170,7 +171,7 @@ function GC:_rehash(newsize: usize): boolean
   local oldsize: usize = self.nslots
 
   self.nslots = newsize
-  self.items = memory.spanalloc0(@GCItem, self.nslots)
+  self.items = allocator.spanalloc0(@GCItem, self.nslots)
 
   if self.items.size == 0 then
     self.nslots = oldsize
@@ -183,7 +184,7 @@ function GC:_rehash(newsize: usize): boolean
       self:_add_ptr(olditems[i].ptr, olditems[i].size, olditems[i].flags, olditems[i].finalizer)
     end
   end
-  memory.spandealloc(olditems)
+  allocator.spandealloc(olditems)
   return true
 end
 
@@ -238,7 +239,7 @@ function GC:_mark_ptr(ptr: pointer)
   end
 end
 
-function GC:_mark_stack() <noinline>
+function GC:_mark_stack() <noinline, cattribute 'no_sanitize(("address"))'>
   local stk: pointer
   local bot: usize = (@usize)(self.bottom)
   local top: usize = (@usize)(&stk)
@@ -302,7 +303,7 @@ function GC:_sweep() <noinline>
     end
   end
 
-  self.frees = memory.spanrealloc0(self.frees, self.nfrees)
+  self.frees = allocator.spanrealloc0(self.frees, self.nfrees)
   if self.frees.size == 0 then
     return
   end
@@ -353,11 +354,11 @@ function GC:_sweep() <noinline>
       if self.frees[j].finalizer then 
         --self.frees[j].finalizer(self.frees[j].ptr)
       end
-      memory.dealloc(self.frees[j].ptr)
+      allocator.dealloc(self.frees[j].ptr)
     end
   end
 
-  memory.spandealloc(self.frees)
+  allocator.spandealloc(self.frees)
   self.frees = {}
   self.nfrees = 0
 end
@@ -373,8 +374,8 @@ end
 
 function GC:stop()
   self:_sweep()
-  memory.spandealloc(self.items)
-  memory.spandealloc(self.frees)
+  allocator.spandealloc(self.items)
+  allocator.spandealloc(self.frees)
   $self = {}
   self.paused = true
 end
@@ -411,7 +412,7 @@ function GC:add(ptr: pointer, size: usize, flags: usize, finalizer: FinalizerPtr
     return ptr
   else
     self.nitems = self.nitems - 1
-    memory.dealloc(ptr)
+    allocator.dealloc(ptr)
     return nilptr
   end
 end
@@ -422,24 +423,34 @@ function GC:rem(ptr: pointer)
   self.mitems = self.nitems + self.nitems // 2 + 1
 end
 
+function GC:alloc_opt(size: usize, flags: usize, finalizer: FinalizerPtr): pointer
+  local ptr: pointer = allocator.alloc(size)
+  if ptr then
+    ptr = self:add(ptr, size, flags, finalizer)
+  end
+  return ptr
+end
+
 function GC:alloc0_opt(size: usize, flags: usize, finalizer: FinalizerPtr): pointer
-  local ptr: pointer = memory.alloc0(size)
+  local ptr: pointer = allocator.alloc0(size)
   if ptr then
     ptr = self:add(ptr, size, flags, finalizer)
   end
   return ptr
 end
 
+function GC:alloc(size: usize): pointer
+  return self:alloc_opt(size, 0, nilptr)
+end
+
 function GC:alloc0(size: usize): pointer
   return self:alloc0_opt(size, 0, nilptr)
 end
 
-function GC:realloc0(ptr: pointer, size: usize, oldsize: usize): pointer
-  local qtr: pointer = memory.realloc0(ptr, size, oldsize)
-
+function GC:_realloc(qtr: pointer, ptr: pointer, size: usize): pointer <inline>
   if not qtr then
     self:rem(ptr)
-    return qtr
+    return nilptr
   end
 
   if not ptr then
@@ -462,13 +473,23 @@ function GC:realloc0(ptr: pointer, size: usize, oldsize: usize): pointer
   return nilptr
 end
 
+function GC:realloc(ptr: pointer, size: usize): pointer
+  local qtr: pointer = allocator.realloc(ptr, size)
+  return self:_realloc(qtr, ptr, size)
+end
+
+function GC:realloc0(ptr: pointer, size: usize, oldsize: usize): pointer
+  local qtr: pointer = allocator.realloc0(ptr, size, oldsize)
+  return self:_realloc(qtr, ptr, size)
+end
+
 function GC:dealloc(ptr: pointer)
   local p: GCItem* = self:_get_ptr(ptr)
   if p then
     if p.finalizer then
       --p.finalizer(ptr)
     end
-    memory.dealloc(ptr)
+    allocator.dealloc(ptr)
     self:rem(ptr)
   end
 end
diff --git a/lib/memory.nelua b/lib/memory.nelua
index b4ef6124..cba01b85 100644
--- a/lib/memory.nelua
+++ b/lib/memory.nelua
@@ -39,42 +39,7 @@ local function free(ptr: pointer): void <cimport'free',cinclude'<stdlib.h>',node
 global memory = @record{}
 
 --------------------------------------------------------------------------------
--- raw pointer functions
-
-function memory.alloc(size: usize): pointer
-  check(size > 0_u, 'memory.alloc: size cannot be zero')
-  local p: pointer = malloc(size)
-  check(p, 'memory.alloc: allocation fail')
-  return p
-end
-
-function memory.alloc0(size: usize): pointer
-  check(size > 0_u, 'memory.alloc0: size must be greater than 0')
-  local p: pointer = calloc(size, 1)
-  check(p, 'memory.alloc0: allocation fail')
-  return p
-end
-
-function memory.realloc(p: pointer, size: usize): pointer
-  check(size > 0_u, 'memory.realloc: size must be greater than 0')
-  p = realloc(p, size)
-  check(size == 0 or p, 'memory.realloc: allocation fail')
-  return p
-end
-
-function memory.realloc0(p: pointer, newsize: usize, oldsize: usize): pointer
-  check(newsize > 0_u, 'memory.realloc0: size must be greater than 0')
-  p = realloc(p, newsize)
-  check(newsize == 0_u or p, 'memory.realloc0: allocation fail')
-  if likely(newsize > oldsize) then
-    memset(&(@byte[0]*)(p)[oldsize], 0, newsize - oldsize)
-  end
-  return p
-end
-
-function memory.dealloc(p: pointer)
-  free(p)
-end
+-- pointer functions
 
 function memory.copy(dest: pointer, src: pointer, size: usize)
   check(dest and src, 'memory.copy: invalid pointer')
@@ -91,6 +56,11 @@ function memory.set(dest: pointer, x: byte, size: usize)
   memset(dest, x, size)
 end
 
+function memory.zero(dest: pointer, size: usize)
+  check(dest, 'memory.set: invalid pointer')
+  memset(dest, 0, size)
+end
+
 function memory.compare(a: pointer, b: pointer, size: usize): int32
   check(a and b, 'memory.move: invalid pointer')
   return memcmp(a, b, size)
@@ -114,62 +84,6 @@ end
 --------------------------------------------------------------------------------
 -- span functions
 
-function memory.spanalloc(T: type, size: usize)
-  local s: span(T)
-  if likely(size > 0_u) then
-    s.data = (@T*)(malloc(size * #T))
-    check(s.data, 'memory.spanalloc: allocation fail')
-    s.size = size
-  end
-  return s
-end
-
-function memory.spanalloc0(T: type, size: usize)
-  local s: span(T)
-  if likely(size > 0_u) then
-    s.data = (@T*)(calloc(size, #T))
-    check(s.data, 'memory.spanalloc0: allocation fail')
-    s.size = size
-  end
-  return s
-end
-
-function memory.spanrealloc(s: auto, size: usize)
-  local T: type = #[check_span_subtype(s)]#
-  if unlikely(size == 0) then
-    free(s.data)
-    s.data = nilptr
-    s.size = 0
-  else
-    s.data = (@T*)(realloc(s.data, size * #T))
-    check(s.data, 'memory.spanrealloc0: allocation fail')
-    s.size = size
-  end
-  return s
-end
-
-function memory.spanrealloc0(s: auto, size: usize)
-  local T: type = #[check_span_subtype(s)]#
-  if unlikely(size == 0) then
-    free(s.data)
-    s.data = nilptr
-    s.size = 0
-  else
-    s.data = (@T*)(realloc(s.data, size * #T))
-    check(s.data, 'memory.spanrealloc0: allocation fail')
-    if likely(size > s.size) then
-      memset(&s[s.size], 0, (size - s.size) * #T)
-    end
-    s.size = size
-  end
-  return s
-end
-
-function memory.spandealloc(s: auto)
-  ## check_span_subtype(s)
-  free(s.data)
-end
-
 function memory.spancopy(dest: auto, src: auto)
   local T: type = #[check_span_subtype(dest)]#
   ## check_type_match(dest, src)
@@ -196,6 +110,12 @@ function memory.spanset(dest: auto, x: auto)
   end
 end
 
+function memory.spanzero(dest: auto)
+  ## check_span_subtype(dest)
+  local T: type = #[dest.type.subtype]#
+  memset(dest.data, 0, dest.size * #T)
+end
+
 function memory.spancompare(a: auto, b: auto): int32
   ## check_span_subtype(a) check_type_match(a, b)
   local size: usize <noinit>
@@ -248,27 +168,3 @@ function memory.spancontains(s: auto, x: auto): boolean
   end
   return false
 end
-
---------------------------------------------------------------------------------
--- new and delete
-
-function memory.new(T: type, size: auto)
-  ## if not size.type:is_nil() then
-    ## staticassert(size.type:is_integral(), 'memory.new: size must be an integral type')
-    ## if not size.type:is_unsigned() then
-      check(size > 0, 'memory.new: size must be greater than 0')
-    ## end
-    return memory.spanalloc0(T, (@usize)(size))
-  ## else
-    return (@T*)(memory.alloc0(#T))
-  ## end
-end
-
-function memory.delete(s: auto)
-  ## staticassert(s.type:is_pointer() or s.type:is_span(), "memory.delete: invalid type '%s'", s.type)
-  ## if s.type:is_span() then
-    memory.spandealloc(s)
-  ## else -- pointer
-    memory.dealloc(s)
-  ## end
-end
diff --git a/lib/myarraytable.nelua b/lib/myarraytable.nelua
index 6b3c6e97..ab0e829c 100644
--- a/lib/myarraytable.nelua
+++ b/lib/myarraytable.nelua
@@ -4,86 +4,114 @@
 ## strict = true
 ## unitname = 'nelua'
 
-require 'memory'
-
-local T = @byte
-local ArrayTableObj = @record{
-  size: usize,
-  data: span(T)
-}
-
-function ArrayTableObj.create(): ArrayTableObj*
-  return (@ArrayTableObj*)(memory.new(ArrayTableObj))
-end
-
-function ArrayTableObj:reserve(cap: usize)
-  memory.spanrealloc0(self.data, cap + 1)
-  if unlikely(self.data.size == 0) then
-    local t: T
-    self.data[0] = t
+require 'allocators.gc_allocator'
+
+local allocator = @gc_allocator
+
+## local memoize = require 'nelua.utils.memoize'
+## myarraytable = hygienize(memoize(function(T)
+  local T = @#[T]#
+  local ArrayTableImpl <codename #['nelua_ArrayTableImpl_'..T.name]#> = @record {
+    size: usize,
+    data: span(T)
+  }
+  local ArrayTable <codename #['nelua_ArrayTable_'..T.name]#> = @record{
+    impl: ArrayTableImpl*
+  }
+
+  function ArrayTable:_create_impl() <inline>
+    if unlikely(not self.impl) then
+      self.impl = (@ArrayTableImpl*)(allocator.alloc0(#ArrayTableImpl))
+    end
   end
-  self.data.size = cap
-end
-
-function ArrayTableObj:grow()
-  local cap = self.data.size == 0 and 1_usize or (self.data.size << 1)
-  self:reserve(cap)
-end
-
-function ArrayTableObj:resize(n: usize, v: T)
-  local addn = n - self.size
-  if addn > 0 then
-    self:reserve(n)
-    for i=self.size+1,<n do
-      self.data[i+1] = v
+
+  function ArrayTable:_grow()
+    local cap: usize
+    if self.impl.data.size ~= 0 then
+      cap = self.impl.data.size * 2
+    else
+      cap = 2
     end
-    self.size = n
+    self.impl.data = allocator.spanrealloc(self.impl.data, cap)
   end
-end
 
-function ArrayTableObj:push(v: T)
-  self.size = self.size + 1
-  if unlikely(self.size > self.data.size) then
-    self:grow()
+  function ArrayTable:reserve(n: usize)
+    self:_create_impl()
+    local cap = n + 1
+    if self.impl.data.size < cap then
+      self.impl.data = allocator.spanrealloc(self.impl.data, cap)
+    end
   end
-  self.data[self.size] = v
-end
 
-function ArrayTableObj:pop(): T
-  if unlikely(self.size == 0) then
-    error 'ArrayTable_pop: length is 0'
+  function ArrayTable:resize(n: usize, v: T)
+    self:_create_impl()
+    local addn = n - self.impl.size
+    if addn > 0 then
+      self:reserve(n)
+      for i=self.impl.size+1,<n do
+        self.impl.data[i+1] = v
+      end
+      self.impl.size = n
+    end
   end
-  local i = self.size
-  self.size = self.size - 1
-  return self.data[i]
-end
-
-function ArrayTableObj:at(i: usize): T*
-  if unlikely(i > self.size) then
-    if unlikely(i ~= self.size + 1) then
-      error 'ArrayTable_at: index out of range'
+
+  function ArrayTable:push(v: T)
+    self:_create_impl()
+    self.impl.size = self.impl.size + 1
+    if unlikely(self.impl.size + 1 >= self.impl.data.size) then
+      self:_grow()
     end
-    self.size = self.size + 1
-    if unlikely(self.size > self.data.size) then
-      self:grow()
+    self.impl.data[self.impl.size] = v
+  end
+
+  function ArrayTable:pop(): T
+    check(self.impl and self.impl.size > 0, 'arraytable: length is 0')
+    local i = self.impl.size
+    self.impl.size = self.impl.size - 1
+    return self.impl.data[i]
+  end
+
+  function ArrayTable:at(i: usize): T*
+    self:_create_impl()
+    if unlikely(i > self.impl.size) then
+      check(i == self.impl.size + 1, 'arraytable.at: index out of range')
+      self.impl.size = self.impl.size + 1
     end
-  elseif unlikely(i == 0 and self.data.size == 0) then
-    self:grow()
+    if unlikely(self.impl.size + 1 > self.impl.data.size) then
+      self:_grow()
+      memory.zero(&self.impl.data[i], #T)
+    end
+    return &self.impl.data[i]
+  end
+
+  function ArrayTable:get(i: usize): T*
+    check(self.impl and i <= self.impl.size, 'arraytable.get: index out of range')
+    if unlikely(i == 0 and self.impl.data.size == 0) then
+      self:_grow()
+    end
+    return &self.impl.data[i]
   end
-  return &self.data[i]
-end
-
-function ArrayTableObj:get(i: usize): T*
-  if unlikely(i > self.size) then
-    error 'ArrayTable_get: index out of range'
-  elseif unlikely(i == 0 and self.data.size == 0) then
-    self:grow()
+
+  function ArrayTable:length(): usize
+    if unlikely(not self.impl) then
+      return 0
+    end
+    return self.impl.size
   end
-  return &self.data[i]
-end
 
-function ArrayTableObj:length(): usize
-  return self.size
-end
+  function ArrayTable:clear()
+    if likely(self.impl) then
+      self.impl.size = 0
+    end
+  end
+
+  function ArrayTable:reset()
+    if likely(self.impl) then
+      allocator.spandealloc(self.impl.data)
+      allocator.dealloc(self.impl)
+      self.impl = nilptr
+    end
+  end
 
-global myarraytable = @ArrayTableObj*
+  ## return ArrayTable
+## end))
diff --git a/nelua/ppcontext.lua b/nelua/ppcontext.lua
index c1ff470f..72de45f1 100644
--- a/nelua/ppcontext.lua
+++ b/nelua/ppcontext.lua
@@ -65,8 +65,7 @@ function PPContext:tovalue(val, orignode)
     -- inject persistent parsed type
     local pattr = Attr({
       type = typedefs.primtypes.type,
-      value = val,
-      comptime = true
+      value = val
     })
     node.attr:merge(pattr)
     node.pattr = pattr
@@ -74,6 +73,7 @@ function PPContext:tovalue(val, orignode)
     node = aster.String{val}
   elseif traits.is_symbol(val) then
     node = aster.Id{val.name}
+    node.pattr = val
   elseif traits.is_number(val) or traits.is_bignumber(val) then
     local num = bn.new(val)
     local neg = false
diff --git a/tests/memory_test.nelua b/tests/memory_test.nelua
index 1c040a4b..dcdd7104 100644
--- a/tests/memory_test.nelua
+++ b/tests/memory_test.nelua
@@ -1,8 +1,11 @@
 ## strict = true
 
---## cflags '-fsanitize=address -fsanitize=undefined -fsanitize=leak'
+--## cflags '-fsanitize=address -fsanitize=undefined'
 
 require 'memory'
+require 'allocators.generic_allocator'
+
+local allocator = @generic_allocator
 
 -- utilities
 local vec2 = @record{x: integer, y: integer}
@@ -11,76 +14,78 @@ local vec2 = @record{x: integer, y: integer}
 -- raw pointers
 
 do -- alloc and dealloc
-  local p = memory.alloc(4)
+  local p = allocator.alloc(4)
   assert(p)
-  memory.dealloc(p)
+  allocator.dealloc(p)
   p = nilptr
-  memory.dealloc(p)
+  allocator.dealloc(p)
 end
 
 do -- alloc0
-  local p = (@vec2*)(memory.alloc0(#@vec2))
+  local p = (@vec2*)(allocator.alloc0(#@vec2))
   assert(p)
   assert(p.x == 0 and p.y == 0)
-  memory.dealloc(p)
+  allocator.dealloc(p)
 end
 
 do -- realloc
-  local p = memory.alloc(1)
+  local p = allocator.alloc(1)
   assert(p)
-  p = memory.realloc(p, 4)
+  p = allocator.realloc(p, 4)
   assert(p)
-  p = memory.realloc(p, 1)
-  memory.dealloc(p)
+  p = allocator.realloc(p, 1)
+  allocator.dealloc(p)
 end
 
 do -- realloc0
-  local p = (@vec2[0]*)(memory.alloc0(#@vec2))
+  local p = (@vec2[0]*)(allocator.alloc0(#@vec2))
   assert(p)
   assert(p[0].x == 0 and p[0].y == 0)
   p[0] = vec2{x=1, y=2}
-  p = (@vec2[0]*)(memory.realloc0(p, 2*#@vec2, #@vec2))
+  p = (@vec2[0]*)(allocator.realloc0(p, 2*#@vec2, #@vec2))
   assert(p)
   assert(p[0].x == 1 and p[0].y == 2)
   assert(p[1].x == 0 and p[1].y == 0)
-  p = (@vec2[0]*)(memory.realloc0(p, #@vec2, #@vec2))
+  p = (@vec2[0]*)(allocator.realloc0(p, #@vec2, #@vec2))
   assert(p)
   assert(p[0].x == 1 and p[0].y == 2)
-  p = (@vec2[0]*)(memory.realloc0(p, #@vec2, 0))
+  p = (@vec2[0]*)(allocator.realloc0(p, #@vec2, 0))
   assert(p)
   assert(p[0].x == 0 and p[0].y == 0)
-  memory.dealloc(p)
+  allocator.dealloc(p)
 end
 
 do -- copy
-  local pa = (@vec2*)(memory.alloc0(#@vec2))
-  local pb = (@vec2*)(memory.alloc0(#@vec2))
+  local pa = (@vec2*)(allocator.alloc0(#@vec2))
+  local pb = (@vec2*)(allocator.alloc0(#@vec2))
   $pa = {x=1,y=2}
   memory.copy(pb, pa, #@vec2)
   assert(pb.x == 1 and pb.y == 2)
-  memory.dealloc(pa)
-  memory.dealloc(pb)
+  allocator.dealloc(pa)
+  allocator.dealloc(pb)
 end
 
 do -- move
-  local p = (@vec2[0]*)(memory.alloc0(2*#@vec2))
+  local p = (@vec2[0]*)(allocator.alloc0(2*#@vec2))
   local pa, pb = &p[0], &p[1]
   $pa, $pb = {x=1,y=2}, {x=3,y=4}
   memory.move(&((@integer[0]*)(p))[1], p, 3 * #@integer)
   assert(pa.x == 1 and pa.y == 1 and pb.x == 2 and pb.y == 3)
-  memory.dealloc(p)
+  allocator.dealloc(p)
 end
 
-do -- set
-  local p = (@vec2*)(memory.alloc0(#@vec2))
+do -- set and zero
+  local p = (@vec2*)(allocator.alloc0(#@vec2))
   memory.set(p, 0xff, #@vec2)
   assert(p.x == 0xffffffffffffffff and p.y == 0xffffffffffffffff)
-  memory.dealloc(p)
+  memory.zero(p, #@vec2)
+  assert(p.x == 0 and p.y == 0)
+  allocator.dealloc(p)
 end
 
 do -- compare
-  local pa = (@vec2*)(memory.alloc0(#@vec2))
-  local pb = (@vec2*)(memory.alloc0(#@vec2))
+  local pa = (@vec2*)(allocator.alloc0(#@vec2))
+  local pb = (@vec2*)(allocator.alloc0(#@vec2))
   assert(memory.compare(pa, pb, #@vec2) == 0)
   pa.x = 1 pb.x = 2
   assert(memory.compare(pa, pb, #@vec2) == -1)
@@ -88,126 +93,129 @@ do -- compare
   assert(memory.compare(pa, pb, #@vec2) == 1)
   pa.x = 2 pb.x = 2
   assert(memory.compare(pa, pb, #@vec2) == 0)
-  memory.dealloc(pa)
-  memory.dealloc(pb)
+  allocator.dealloc(pa)
+  allocator.dealloc(pb)
 end
 
 do -- equals
-  local pa = (@vec2*)(memory.alloc0(#@vec2))
-  local pb = (@vec2*)(memory.alloc0(#@vec2))
+  local pa = (@vec2*)(allocator.alloc0(#@vec2))
+  local pb = (@vec2*)(allocator.alloc0(#@vec2))
   assert(memory.equals(pa, pb, #@vec2))
   pa.x = 1
   assert(not memory.equals(pa, pb, #@vec2))
-  memory.dealloc(pa)
-  memory.dealloc(pb)
+  allocator.dealloc(pa)
+  allocator.dealloc(pb)
 end
 
 do -- scan
-  local p = (@vec2*)(memory.alloc0(#@vec2))
+  local p = (@vec2*)(allocator.alloc0(#@vec2))
   p.x = 1 p.y = 2
   assert(memory.scan(p, 1, #@vec2) == &p.x)
   assert(memory.scan(p, 2, #@vec2) == &p.y)
   assert(memory.scan(p, 3, #@vec2) == nilptr)
-  memory.dealloc(p)
+  allocator.dealloc(p)
 end
 
 do -- contains
-  local p = (@vec2*)(memory.alloc0(#@vec2))
+  local p = (@vec2*)(allocator.alloc0(#@vec2))
   p.x = 1 p.y = 2
   assert(memory.contains(p, 0, #@vec2))
   assert(memory.contains(p, 1, #@vec2))
   assert(memory.contains(p, 2, #@vec2))
   assert(not memory.contains(p, 3, #@vec2))
-  memory.dealloc(p)
+  allocator.dealloc(p)
 end
 
 --------------------------------------------------------------------------------
 -- span
 
 do -- spanalloc and spandealloc
-  local p = memory.spanalloc(@vec2, 2)
+  local p = allocator.spanalloc(@vec2, 2)
   assert(p.data ~= nilptr and p.size == 2)
-  memory.spandealloc(p)
+  allocator.spandealloc(p)
   p = (@span(vec2)){}
   assert(p.data == nilptr and p.size == 0)
-  memory.spandealloc(p)
-  p = memory.spanalloc(@vec2, 0)
+  allocator.spandealloc(p)
+  p = allocator.spanalloc(@vec2, 0)
   assert(p.data == nilptr and p.size == 0)
 end
 
 do -- spanalloc0
-  local p = memory.spanalloc0(@vec2, 2)
+  local p = allocator.spanalloc0(@vec2, 2)
   assert(p.data ~= nilptr and p.size == 2)
   assert(p[0].x == 0 and p[0].y == 0)
   assert(p[1].x == 0 and p[1].y == 0)
-  memory.spandealloc(p)
-  p = memory.spanalloc0(@vec2, 0)
+  allocator.spandealloc(p)
+  p = allocator.spanalloc0(@vec2, 0)
   assert(p.data == nilptr and p.size == 0)
 end
 
 do -- spanrealloc
-  local p = memory.spanalloc(@vec2, 1)
+  local p = allocator.spanalloc(@vec2, 1)
   assert(p.data ~= nilptr and p.size == 1)
-  p = memory.spanrealloc(p, 4)
+  p = allocator.spanrealloc(p, 4)
   assert(p.data ~= nilptr and p.size == 4)
-  p = memory.spanrealloc(p, 1)
+  p = allocator.spanrealloc(p, 1)
   assert(p.data ~= nilptr and p.size == 1)
-  p = memory.spanrealloc(p, 0)
+  p = allocator.spanrealloc(p, 0)
   assert(p.data == nilptr and p.size == 0)
-  memory.spandealloc(p)
+  allocator.spandealloc(p)
   assert(p.data == nilptr and p.size == 0)
 end
 
 do -- spanrealloc0
-  local p = memory.spanalloc0(@vec2, 1)
+  local p = allocator.spanalloc0(@vec2, 1)
   assert(p.data ~= nilptr and p.size == 1)
   assert(p[0].x == 0 and p[0].y == 0)
   p[0] = vec2{x=1, y=2}
-  p = memory.spanrealloc0(p, 2)
+  p = allocator.spanrealloc0(p, 2)
   assert(p.data ~= nilptr and p.size == 2)
   assert(p[0].x == 1 and p[0].y == 2)
   assert(p[1].x == 0 and p[1].y == 0)
-  p = memory.spanrealloc0(p, 1)
+  p = allocator.spanrealloc0(p, 1)
   assert(p.data ~= nilptr and p.size == 1)
   assert(p[0].x == 1 and p[0].y == 2)
-  p = memory.spanrealloc0(p, 0)
+  p = allocator.spanrealloc0(p, 0)
   assert(p.data == nilptr and p.size == 0)
-  memory.spandealloc(p)
+  allocator.spandealloc(p)
 end
 
 do -- spancopy
-  local pa = memory.spanalloc0(@vec2, 1)
-  local pb = memory.spanalloc0(@vec2, 1)
+  local pa = allocator.spanalloc0(@vec2, 1)
+  local pb = allocator.spanalloc0(@vec2, 1)
   pa[0] = {x=1,y=2}
   memory.spancopy(pb, pa)
   assert(pb[0].x == 1 and pb[0].y == 2)
-  memory.spandealloc(pa)
-  memory.spandealloc(pb)
+  allocator.spandealloc(pa)
+  allocator.spandealloc(pb)
 end
 
 do -- spanmove
-  local p = memory.spanalloc0(@vec2, 2)
+  local p = allocator.spanalloc0(@vec2, 2)
   local pa, pb = &p[0], &p[1]
   $pa, $pb = {x=1,y=2}, {x=3,y=4}
   local sb = (@span(integer))({ data=&(((@integer[0]*)(p.data))[0]), size=3 })
   local db = (@span(integer))({ data=&(((@integer[0]*)(p.data))[1]), size=3 })
   memory.spanmove(db, sb)
   --assert(pa.x == 1 and pa.y == 1 and pb.x == 2 and pb.y == 3)
-  memory.spandealloc(p)
+  allocator.spandealloc(p)
 end
 
-do -- spanset
-  local p = memory.spanalloc0(@vec2, 2)
+do -- spanset and spanzero
+  local p = allocator.spanalloc0(@vec2, 2)
   memory.spanset(p, (@vec2){x=1,y=2})
   assert(p[0].x == 1 and p[0].y == 2)
   assert(p[1].x == 1 and p[1].y == 2)
-  memory.spandealloc(p)
+  memory.spanzero(p)
+  assert(p[0].x == 0 and p[0].y == 0)
+  assert(p[1].x == 0 and p[1].y == 0)
+  allocator.spandealloc(p)
 end
 
 do -- spancompare
-  local pa = memory.spanalloc0(@vec2, 1)
-  local pb = memory.spanalloc0(@vec2, 1)
-  local pc = memory.spanalloc0(@vec2, 0)
+  local pa = allocator.spanalloc0(@vec2, 1)
+  local pb = allocator.spanalloc0(@vec2, 1)
+  local pc = allocator.spanalloc0(@vec2, 0)
   assert(memory.spancompare(pa, pb) == 0)
   pa[0].x = 1 pb[0].x = 2
   assert(memory.spancompare(pa, pb) == -1)
@@ -219,59 +227,59 @@ do -- spancompare
   assert(memory.spancompare(pa, pb) == 0)
   assert(memory.spancompare(pa, pc) == 1)
   assert(memory.spancompare(pc, pa) == -1)
-  memory.spandealloc(pa)
-  memory.spandealloc(pb)
-  memory.spandealloc(pc)
+  allocator.spandealloc(pa)
+  allocator.spandealloc(pb)
+  allocator.spandealloc(pc)
 end
 
 do -- spanequals
-  local pa = memory.spanalloc0(@vec2, 1)
-  local pb = memory.spanalloc0(@vec2, 1)
+  local pa = allocator.spanalloc0(@vec2, 1)
+  local pb = allocator.spanalloc0(@vec2, 1)
   assert(memory.spanequals(pa, pb))
   pa[0].x = 1
   assert(not memory.spanequals(pa, pb))
-  memory.spandealloc(pa)
-  memory.spandealloc(pb)
+  allocator.spandealloc(pa)
+  allocator.spandealloc(pb)
 end
 
 do -- spanscan
-  local p = memory.spanalloc0(@vec2, 2)
+  local p = allocator.spanalloc0(@vec2, 2)
   p[0].x = 1 p[0].y = 2
   p[1].x = 3 p[1].y = 4
   assert(memory.spanscan(p, (@vec2){1,2}) == 0)
   assert(memory.spanscan(p, (@vec2){3,4}) == 1)
   assert(memory.spanscan(p, (@vec2){5,6}) == -1)
-  memory.spandealloc(p)
+  allocator.spandealloc(p)
 end
 
 do -- spancontains
-  local p = memory.spanalloc0(@vec2, 2)
+  local p = allocator.spanalloc0(@vec2, 2)
   p[0].x = 1 p[0].y = 2
   p[1].x = 3 p[1].y = 4
   assert(memory.spancontains(p, (@vec2){1,2}))
   assert(memory.spancontains(p, (@vec2){3,4}))
   assert(not memory.spancontains(p, (@vec2){5,6}))
-  memory.spandealloc(p)
+  allocator.spandealloc(p)
 end
 
 --------------------------------------------------------------------------------
 -- new and delete
 
 do -- new for pointers
-  local v = memory.new(@vec2)
+  local v = allocator.new(@vec2)
   assert(v.x == 0 and v.y == 0)
   v.x = 1 v.y = 2
   assert(v.x == 1 and v.y == 2)
-  memory.delete(v)
+  allocator.delete(v)
 end
 
 do -- new for spans
-  local v = memory.new(@vec2, 2)
+  local v = allocator.new(@vec2, 2)
   assert(v[0].x == 0 and v[0].y == 0)
   assert(v[1].x == 0 and v[1].y == 0)
   v[0].x = 1 v[0].y = 2
   v[1].x = 3 v[1].y = 4
   assert(v[0].x == 1 and v[0].y == 2)
   assert(v[1].x == 3 and v[1].y == 4)
-  memory.delete(v)
+  allocator.delete(v)
 end