bug (GC can collect long identifier during parser) + change (using

a single constant table for all functions in a chunk)

lcode.c

@@ -1,5 +1,5 @@
 /*
-** $Id: lcode.c,v 2.70 2013/06/20 17:37:31 roberto Exp roberto $
+** $Id: lcode.c,v 2.71 2013/06/25 18:57:18 roberto Exp roberto $
 ** Code generator for Lua
 ** See Copyright Notice in lua.h
 */
@@ -307,17 +307,21 @@ static void freeexp (FuncState *fs, expdesc *e) {
 }
 
 
+/*
+** Use scanner's table to cache position of constants in contant list
+** and try to reuse constants
+*/
 static int addk (FuncState *fs, TValue *key, TValue *v) {
   lua_State *L = fs->ls->L;
-  TValue *idx = luaH_set(L, fs->h, key);
   Proto *f = fs->f;
+  TValue *idx = luaH_set(L, fs->ls->h, key);  /* index scanner table */
   int k, oldsize;
-  if (ttisinteger(idx)) {
+  if (ttisinteger(idx)) {  /* is there an index there? */
     k = ivalue(idx);
-    if (luaV_rawequalobj(&f->k[k], v))
-      return k;
-    /* else may be a collision (e.g., between 0.0 and "\0\0\0\0\0\0\0\0");
-       go through and create a new entry for this value */
+    /* correct value? (warning: must distinguish floats from integers!) */
+    if (k < fs->nk && ttype(&f->k[k]) == ttype(v) &&
+                      luaV_rawequalobj(&f->k[k], v))
+      return k;  /* reuse index */
   }
   /* constant not found; create a new entry */
   oldsize = f->sizek;
@@ -377,7 +381,7 @@ static int nilK (FuncState *fs) {
   TValue k, v;
   setnilvalue(&v);
   /* cannot use nil as key; instead use table itself to represent nil */
-  sethvalue(fs->ls->L, &k, fs->h);
+  sethvalue(fs->ls->L, &k, fs->ls->h);
   return addk(fs, &k, &v);
 }
 

llex.c

@@ -1,5 +1,5 @@
 /*
-** $Id: llex.c,v 2.67 2013/06/19 14:27:00 roberto Exp roberto $
+** $Id: llex.c,v 2.68 2013/08/21 20:09:51 roberto Exp roberto $
 ** Lexical Analyzer
 ** See Copyright Notice in lua.h
 */
@@ -119,22 +119,25 @@ l_noret luaX_syntaxerror (LexState *ls, const char *msg) {
 
 
 /*
-** creates a new string and anchors it in function's table so that
-** it will not be collected until the end of the function's compilation
-** (by that time it should be anchored in function's prototype)
+** creates a new string and anchors it in scanner's table so that
+** it will not be collected until the end of the compilation
+** (by that time it should be anchored somewhere)
 */
 TString *luaX_newstring (LexState *ls, const char *str, size_t l) {
   lua_State *L = ls->L;
   TValue *o;  /* entry for `str' */
   TString *ts = luaS_newlstr(L, str, l);  /* create new string */
   setsvalue2s(L, L->top++, ts);  /* temporarily anchor it in stack */
-  o = luaH_set(L, ls->fs->h, L->top - 1);
-  if (ttisnil(o)) {  /* not in use yet? (see 'addK') */
+  o = luaH_set(L, ls->h, L->top - 1);
+  if (ttisnil(o)) {  /* not in use yet? */
     /* boolean value does not need GC barrier;
        table has no metatable, so it does not need to invalidate cache */
     setbvalue(o, 1);  /* t[string] = true */
     luaC_checkGC(L);
   }
+  else {  /* string already present */
+    ts = rawtsvalue(keyfromval(o));  /* re-use value previously stored */
+  }
   L->top--;  /* remove string from stack */
   return ts;
 }

llex.h

@@ -1,5 +1,5 @@
 /*
-** $Id: llex.h,v 1.73 2013/04/16 18:46:28 roberto Exp roberto $
+** $Id: llex.h,v 1.74 2013/04/26 13:07:53 roberto Exp roberto $
 ** Lexical Analyzer
 ** See Copyright Notice in lua.h
 */
@@ -60,6 +60,7 @@ typedef struct LexState {
   struct lua_State *L;
   ZIO *z;  /* input stream */
   Mbuffer *buff;  /* buffer for tokens */
+  Table *h;  /* to avoid collection/reuse strings */
   struct Dyndata *dyd;  /* dynamic structures used by the parser */
   TString *source;  /* current source name */
   TString *envn;  /* environment variable name */

lparser.c

@@ -1,5 +1,5 @@
 /*
-** $Id: lparser.c,v 2.133 2013/04/26 13:07:53 roberto Exp roberto $
+** $Id: lparser.c,v 2.134 2013/08/16 18:55:49 roberto Exp roberto $
 ** Lua Parser
 ** See Copyright Notice in lua.h
 */
@@ -35,6 +35,10 @@
 #define hasmultret(k)		((k) == VCALL || (k) == VVARARG)
 
 
+/* because all strings are unified by the scanner, the parser
+   can use pointer equality for string equality */
+#define eqstr(a,b)	((a) == (b))
+
 
 /*
 ** nodes for block list (list of active blocks)
@@ -57,16 +61,6 @@ static void statement (LexState *ls);
 static void expr (LexState *ls, expdesc *v);
 
 
-static void anchor_token (LexState *ls) {
-  /* last token from outer function must be EOS */
-  lua_assert(ls->fs != NULL || ls->t.token == TK_EOS);
-  if (ls->t.token == TK_NAME || ls->t.token == TK_STRING) {
-    TString *ts = ls->t.seminfo.ts;
-    luaX_newstring(ls, getstr(ts), ts->tsv.len);
-  }
-}
-
-
 /* semantic error */
 static l_noret semerror (LexState *ls, const char *msg) {
   ls->t.token = 0;  /* remove 'near to' from final message */
@@ -222,7 +216,7 @@ static int searchupvalue (FuncState *fs, TString *name) {
   int i;
   Upvaldesc *up = fs->f->upvalues;
   for (i = 0; i < fs->nups; i++) {
-    if (luaS_eqstr(up[i].name, name)) return i;
+    if (eqstr(up[i].name, name)) return i;
   }
   return -1;  /* not found */
 }
@@ -246,7 +240,7 @@ static int newupvalue (FuncState *fs, TString *name, expdesc *v) {
 static int searchvar (FuncState *fs, TString *n) {
   int i;
   for (i = cast_int(fs->nactvar) - 1; i >= 0; i--) {
-    if (luaS_eqstr(n, getlocvar(fs, i)->varname))
+    if (eqstr(n, getlocvar(fs, i)->varname))
       return i;
   }
   return -1;  /* not found */
@@ -342,7 +336,7 @@ static void closegoto (LexState *ls, int g, Labeldesc *label) {
   FuncState *fs = ls->fs;
   Labellist *gl = &ls->dyd->gt;
   Labeldesc *gt = &gl->arr[g];
-  lua_assert(luaS_eqstr(gt->name, label->name));
+  lua_assert(eqstr(gt->name, label->name));
   if (gt->nactvar < label->nactvar) {
     TString *vname = getlocvar(fs, gt->nactvar)->varname;
     const char *msg = luaO_pushfstring(ls->L,
@@ -369,7 +363,7 @@ static int findlabel (LexState *ls, int g) {
   /* check labels in current block for a match */
   for (i = bl->firstlabel; i < dyd->label.n; i++) {
     Labeldesc *lb = &dyd->label.arr[i];
-    if (luaS_eqstr(lb->name, gt->name)) {  /* correct label? */
+    if (eqstr(lb->name, gt->name)) {  /* correct label? */
       if (gt->nactvar > lb->nactvar &&
           (bl->upval || dyd->label.n > bl->firstlabel))
         luaK_patchclose(ls->fs, gt->pc, lb->nactvar);
@@ -403,7 +397,7 @@ static void findgotos (LexState *ls, Labeldesc *lb) {
   Labellist *gl = &ls->dyd->gt;
   int i = ls->fs->bl->firstgoto;
   while (i < gl->n) {
-    if (luaS_eqstr(gl->arr[i].name, lb->name))
+    if (eqstr(gl->arr[i].name, lb->name))
       closegoto(ls, i, lb);
     else
       i++;
@@ -525,7 +519,6 @@ static void codeclosure (LexState *ls, expdesc *v) {
 
 
 static void open_func (LexState *ls, FuncState *fs, BlockCnt *bl) {
-  lua_State *L = ls->L;
   Proto *f;
   fs->prev = ls->fs;  /* linked list of funcstates */
   fs->ls = ls;
@@ -544,10 +537,6 @@ static void open_func (LexState *ls, FuncState *fs, BlockCnt *bl) {
   f = fs->f;
   f->source = ls->source;
   f->maxstacksize = 2;  /* registers 0/1 are always valid */
-  fs->h = luaH_new(L);
-  /* anchor table of constants (to avoid being collected) */
-  sethvalue2s(L, L->top, fs->h);
-  incr_top(L);
   enterblock(fs, bl, 0);
 }
 
@@ -572,9 +561,6 @@ static void close_func (LexState *ls) {
   f->sizeupvalues = fs->nups;
   lua_assert(fs->bl == NULL);
   ls->fs = fs->prev;
-  /* last token read was anchored in defunct function; must re-anchor it */
-  anchor_token(ls);
-  L->top--;  /* pop table of constants */
   luaC_checkGC(L);
 }
 
@@ -1202,7 +1188,7 @@ static void gotostat (LexState *ls, int pc) {
 static void checkrepeated (FuncState *fs, Labellist *ll, TString *label) {
   int i;
   for (i = fs->bl->firstlabel; i < ll->n; i++) {
-    if (luaS_eqstr(label, ll->arr[i].name)) {
+    if (eqstr(label, ll->arr[i].name)) {
       const char *msg = luaO_pushfstring(fs->ls->L,
                           "label " LUA_QS " already defined on line %d",
                           getstr(label), ll->arr[i].line);
@@ -1627,8 +1613,10 @@ Closure *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff,
   LexState lexstate;
   FuncState funcstate;
   Closure *cl = luaF_newLclosure(L, 1);  /* create main closure */
-  /* anchor closure (to avoid being collected) */
-  setclLvalue(L, L->top, cl);
+  setclLvalue(L, L->top, cl);  /* anchor it (to avoid being collected) */
+  incr_top(L);
+  lexstate.h = luaH_new(L);  /* create table for scanner */
+  sethvalue(L, L->top, lexstate.h);  /* anchor it */
   incr_top(L);
   funcstate.f = cl->l.p = luaF_newproto(L);
   funcstate.f->source = luaS_new(L, name);  /* create and anchor TString */
@@ -1641,6 +1629,7 @@ Closure *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff,
   lua_assert(!funcstate.prev && funcstate.nups == 1 && !lexstate.fs);
   /* all scopes should be correctly finished */
   lua_assert(dyd->actvar.n == 0 && dyd->gt.n == 0 && dyd->label.n == 0);
-  return cl;  /* it's on the stack too */
+  L->top--;  /* remove scanner's table */
+  return cl;  /* closure is on the stack, too */
 }
 

lparser.h

@@ -1,5 +1,5 @@
 /*
-** $Id: lparser.h,v 1.70 2012/05/08 13:53:33 roberto Exp roberto $
+** $Id: lparser.h,v 1.71 2013/04/16 18:46:28 roberto Exp roberto $
 ** Lua Parser
 ** See Copyright Notice in lua.h
 */
@@ -97,7 +97,6 @@ struct BlockCnt;  /* defined in lparser.c */
 /* state needed to generate code for a given function */
 typedef struct FuncState {
   Proto *f;  /* current function header */
-  Table *h;  /* table to find (and reuse) elements in `k' */
   struct FuncState *prev;  /* enclosing function */
   struct LexState *ls;  /* lexical state */
   struct BlockCnt *bl;  /* chain of current blocks */

ltable.h

@@ -1,5 +1,5 @@
 /*
-** $Id: ltable.h,v 2.16 2011/08/17 20:26:47 roberto Exp roberto $
+** $Id: ltable.h,v 2.17 2013/04/26 15:39:25 roberto Exp roberto $
 ** Lua tables (hash)
 ** See Copyright Notice in lua.h
 */
@@ -18,6 +18,11 @@
 #define invalidateTMcache(t)	((t)->flags = 0)
 
 
+/* returns the key, given the value of a table entry */
+#define keyfromval(v) \
+  (gkey(cast(Node *, cast(char *, (v)) - offsetof(Node, i_val))))
+
+
 LUAI_FUNC const TValue *luaH_getint (Table *t, lua_Integer key);
 LUAI_FUNC void luaH_setint (lua_State *L, Table *t, lua_Integer key,
                             TValue *value);