src/ltable.js: fix finding table length

There may be array indices higher than the map size.
Use .size as a starting point, not as a known high point

src/ltable.js

@@ -18,6 +18,9 @@ const {
     },
     to_luastring
 } = require('./defs.js');
+const {
+    LUA_MAXINTEGER
+} = require('./luaconf.js');
 const { lua_assert } = require('./llimits.js');
 const ldebug  = require('./ldebug.js');
 const lobject = require('./lobject.js');
@@ -234,7 +237,18 @@ const luaH_setfrom = function(L, t, key, value) {
 */
 const luaH_getn = function(t) {
     let i = 0;
-    let j = t.strong.size + 1; /* use known size of Map to bound search */
+    let j = t.strong.size + 1; /* use known size of Map to kick start search */
+    /* find 'i' and 'j' such that i is present and j is not */
+    while (!(luaH_getint(t, j).ttisnil())) {
+        i = j;
+        if (j > LUA_MAXINTEGER / 2) {  /* overflow? */
+            /* table was built with bad purposes: resort to linear search */
+            i = 1;
+            while (!luaH_getint(t, i).ttisnil()) i++;
+            return i - 1;
+        }
+        j *= 2;
+    }
     /* now do a binary search between them */
     while (j - i > 1) {
         let m = Math.floor((i+j)/2);

test/lapi.test.js

@@ -380,3 +380,47 @@ describe('lua_atnativeerror', () => {
         expect(lua.lua_tojsstring(L, -1)).toBe("runtime error!");
     });
 });
+
+
+describe('lua_len', () => {
+    test('table with two potential boundaries', () => {
+        let L = lauxlib.luaL_newstate();
+        if (!L) throw Error("failed to create lua state");
+
+        {
+            lua.lua_createtable(L);
+            for (let i=3; i<=8; i++) {
+                lua.lua_pushinteger(L, i);
+                lua.lua_seti(L, -2, i);
+            }
+            lua.lua_len(L, -1);
+        }
+
+        // could be 0 or 8
+        let len = lua.lua_tointeger(L, -1);
+        expect(len == 0 || len == 8).toBe(true);
+    });
+
+    test('pathological case', () => {
+        let L = lauxlib.luaL_newstate();
+        if (!L) throw Error("failed to create lua state");
+
+        {
+            lua.lua_createtable(L);
+            let i = 1;
+            for (let j=1; j<=1023; j++) {
+                lua.lua_pushnumber(L, i);
+                lua.lua_pushnumber(L, i);
+                lua.lua_settable(L, -3);
+                i *= 2;
+            }
+            lua.lua_len(L, -1);
+        }
+
+        /* is allowed to be a power of 2 larger than 2
+           however if the result ever changes from 2 then the pathological
+           code path has probably moved */
+        let len = lua.lua_tointeger(L, -1);
+        expect(len).toBe(2);
+    });
+});