From 88dbd0c8dfe5497b7953499e1beedc7cef9969c5 Mon Sep 17 00:00:00 2001 From: NiLuJe Date: Sat, 2 Jan 2021 16:09:23 +0100 Subject: [PATCH 1/5] Minor tweaks to the LuaJIT mcode patch * Better comments in lj_mcode_clear * Follow the same code flow as mcode_allocarea, because it's effectively serving the same purpose * Handle non-contiguous links properly anywhere in the chain --- jni/luajit/koreader-luajit-mcode-debug.patch | 232 ++++++++++++++---- .../koreader-luajit-mcode-reserve-hack.patch | 37 +-- 2 files changed, 211 insertions(+), 58 deletions(-) diff --git a/jni/luajit/koreader-luajit-mcode-debug.patch b/jni/luajit/koreader-luajit-mcode-debug.patch index aad9cd21e..0bdf12ed2 100644 --- a/jni/luajit/koreader-luajit-mcode-debug.patch +++ b/jni/luajit/koreader-luajit-mcode-debug.patch @@ -23,10 +23,17 @@ index aae7b5b..b675e96 100644 #ifdef LUA_USE_ASSERT as->mcp_prev = as->mcp; diff --git a/src/lj_def.h b/src/lj_def.h -index 5b71dfc..1a52020 100644 +index cfe18c4..1a52020 100644 --- a/src/lj_def.h +++ b/src/lj_def.h -@@ -46,6 +46,15 @@ typedef unsigned int uintptr_t; +@@ -39,12 +39,22 @@ typedef int intptr_t; + typedef unsigned int uintptr_t; + #else + #include ++#include + #endif + + /* Needed everywhere. */ #include #include @@ -54,11 +61,36 @@ index 1d0ff54..8ef29a3 100644 lj_trace_flushall(L); } else { if (!(mode & LUAJIT_MODE_ON)) +diff --git a/src/lj_jit.h b/src/lj_jit.h +index 655b84c..1fa6292 100644 +--- a/src/lj_jit.h ++++ b/src/lj_jit.h +@@ -490,6 +490,7 @@ typedef struct jit_State { + BCIns patchins; /* Instruction for pending re-patch. */ + + int mcprot; /* Protection of current mcode area. */ ++ MCode *lastmcarea; /* Base of last unmapped mcode area (i.e., the previous one). */ + MCode *mcarea; /* Base of current mcode area. */ + MCode *mctop; /* Top of current mcode area. */ + MCode *mcbot; /* Bottom of current mcode area. */ diff --git a/src/lj_mcode.c b/src/lj_mcode.c -index 241c86a..198fdd2 100644 +index a5153b2..34c06c0 100644 --- a/src/lj_mcode.c +++ b/src/lj_mcode.c -@@ -109,12 +109,14 @@ static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot, b +@@ -98,19 +98,25 @@ static int mcode_setprot(void *p, size_t sz, DWORD prot) + #define MCPROT_RX (PROT_READ|PROT_EXEC) + #define MCPROT_RWX (PROT_READ|PROT_WRITE|PROT_EXEC) + +-static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot) ++static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot, bool fixed) + { +- void *p = mmap((void *)hint, sz, prot, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); ++ int flags = MAP_PRIVATE | MAP_ANONYMOUS; ++ if (fixed) { ++ flags |= MAP_FIXED; ++ } ++ void *p = mmap((void *)hint, sz, prot, flags, -1, 0); + if (p == MAP_FAILED) { if (!hint) lj_trace_err(J, LJ_TRERR_MCODEAL); p = NULL; } @@ -73,24 +105,51 @@ index 241c86a..198fdd2 100644 munmap(p, sz); } -@@ -214,6 +216,8 @@ static void *mcode_alloc(jit_State *J, size_t sz) +@@ -210,24 +216,48 @@ static void *mcode_alloc(jit_State *J, size_t sz) uintptr_t target = (uintptr_t)(void *)lj_vm_exit_handler & ~(uintptr_t)0xffff; #endif const uintptr_t range = (1u << (LJ_TARGET_JUMPRANGE-1)) - (1u << 21); +- /* First try a contiguous area below the last one. */ +- uintptr_t hint = J->mcarea ? (uintptr_t)J->mcarea - sz : 0; + LJ_LOG("target is @ %p", (void *) target); + LJ_LOG("range is %p", (void *) range); - /* First try a contiguous area below the last one, - * then try the same address as the last area we unmapped - * (this happens after a flush (either explicit or because the mcarea was filled), -@@ -229,6 +233,7 @@ static void *mcode_alloc(jit_State *J, size_t sz) - fixed = true; - } - uintptr_t hint = J->mcarea ? (uintptr_t)J->mcarea - sz : J->lastmcarea ? (uintptr_t)J->lastmcarea : reserve; ++ /* First try a contiguous area below the last one, ++ * then try the same address as the last area we unmapped ++ * (this happens after a flush (either explicit or because the mcarea was filled), ++ * although not in our case, since we patch flushall to clear the region instead of unmapping it now), ++ * and otherwise, use the area we reserved inside the amalgam's address space ++ * (it's currently hard-coded at 1MB, which is twice the default maxmcode). */ ++ uintptr_t reserve = 0; ++ bool fixed = false; ++ /* Only use the reserve for the very first alloc */ ++ if (!J->mcarea && !J->lastmcarea) { ++ reserve = (uintptr_t) &g_lj_mcarea_reserve; ++ /* We'll only use it once, and we *know* it's there, so use MAP_FIXED */ ++ fixed = true; ++ } ++ uintptr_t hint = J->mcarea ? (uintptr_t)J->mcarea - sz : J->lastmcarea ? (uintptr_t)J->lastmcarea : reserve; + LJ_LOG("original hint is @ %p (mcarea: %p / last: %p / reserve: %p)", (void *) hint, (void *) J->mcarea, (void *) J->lastmcarea, (void *) reserve); int i; /* Limit probing iterations, depending on the available pool size. */ for (i = 0; i < LJ_TARGET_JUMPRANGE; i++) { -@@ -249,8 +254,10 @@ static void *mcode_alloc(jit_State *J, size_t sz) + if (mcode_validptr(hint)) { +- void *p = mcode_alloc_at(J, hint, sz, MCPROT_GEN); ++ void *p = mcode_alloc_at(J, hint, sz, MCPROT_GEN, fixed); + + if (mcode_validptr(p) && +- ((uintptr_t)p + sz - target < range || target - (uintptr_t)p < range)) +- return p; +- if (p) mcode_free(J, p, sz); /* Free badly placed area. */ ++ ((uintptr_t)p + sz - target < range || target - (uintptr_t)p < range)) { ++ return p; ++ } ++ if (p) { ++ /* Free badly placed area. */ ++ mcode_free(J, p, sz); ++ /* Ensure the next iteration won't try to use MAP_FIXED */ ++ fixed = false; ++ } + } /* Next try probing 64K-aligned pseudo-random addresses. */ do { hint = lj_prng_u64(&J2G(J)->prng) & ((1u<param[JIT_P_sizemcode] << 10; sz = (sz + LJ_PAGESIZE-1) & ~(size_t)(LJ_PAGESIZE - 1); J->mcarea = (MCode *)mcode_alloc(J, sz); @@ -109,7 +185,7 @@ index 241c86a..198fdd2 100644 J->szmcarea = sz; J->mcprot = MCPROT_GEN; J->mctop = (MCode *)((char *)J->mcarea + J->szmcarea); -@@ -298,6 +306,7 @@ static void mcode_allocarea(jit_State *J) +@@ -275,13 +306,71 @@ static void mcode_allocarea(jit_State *J) void lj_mcode_free(jit_State *J) { MCode *mc = J->mcarea; @@ -117,33 +193,71 @@ index 241c86a..198fdd2 100644 J->mcarea = NULL; J->szallmcarea = 0; while (mc) { -@@ -309,6 +318,7 @@ void lj_mcode_free(jit_State *J) - } - mc = next; - } + MCode *next = ((MCLink *)mc)->next; + mcode_free(J, mc, ((MCLink *)mc)->size); ++ /* Remember the oldest (i.e., highest address) link as lastmcarea */ ++ if (!next) { ++ J->lastmcarea = mc; ++ } ++ mc = next; ++ } + LJ_LOG("lastmcarea is now %p", J->lastmcarea); - } - - /* Clear all MCode areas. */ -@@ -326,7 +336,9 @@ void lj_mcode_clear(jit_State *J) - szallmcarea += size; - mcarea = mc; - prevsize = size; ++} ++ ++/* Clear all MCode areas. */ ++void lj_mcode_clear(jit_State *J) ++{ ++ MCode *mc = J->mcarea; ++ /* Keep track of the previous link in the chain */ ++ MCode *mcarea = J->mcarea; ++ size_t szallmcarea = 0; ++ size_t szmcarea = 0; ++ while (mc) { ++ MCode *next = ((MCLink *)mc)->next; ++ size_t size = ((MCLink *)mc)->size; ++ /* Reset mcarea to the oldest contiguous link */ ++ if ((next && next == mc + size) || (!next && mc == mcarea + szmcarea) || (!next && mc == mcarea)) { ++ /* ^ next link is contiguous ^ last link is contiguous ^ single link in the chain */ ++ mcarea = mc; ++ szmcarea = size; ++ szallmcarea += size; + LJ_LOG("contiguous %zuK link detected @ %p (total: %zuK) (next @ %p)", size / 1024U, mc, szallmcarea / 1024U, next); - } else { ++ } else { ++ mcarea = NULL; + LJ_LOG("non-contiguous %zuK link detected @ %p (next @ %p)!", size / 1024U, mc, next); - mcarea = NULL; - } ++ /* A non-contiguous link anywhere in the chain means we scrap the whole chain, to keep things simple */ ++ break; ++ } mc = next; -@@ -353,6 +365,7 @@ void lj_mcode_clear(jit_State *J) - ((MCLink *)J->mcarea)->size = J->szmcarea; - /* Update the protection cache */ - J->mcprot = MCPROT_GEN; + } ++ ++ /* If we hit a non-contiguous links, fallback to lj_mcode_free */ ++ if (!mcarea) { ++ return lj_mcode_free(J); ++ } ++ ++ /* Ready to recycle the full chain */ ++ /* Rewind to the lowest address (as links are allocated high to low, c.f., mcode_alloc) */ ++ J->mcarea = mcarea + szmcarea - szallmcarea; ++ J->szmcarea = szallmcarea; ++ /* We need write access to clear it */ ++ if (LJ_UNLIKELY(mcode_setprot(J->mcarea, J->szmcarea, MCPROT_GEN))) ++ mcode_protfail(J); ++ /* Update the protection cache */ ++ J->mcprot = MCPROT_GEN; ++ memset(J->mcarea, 0, J->szmcarea); ++ /* Tell the JIT that it once again has the full area available to generate code in, c.f., mcode_allocarea */ ++ J->mctop = (MCode *)((char *)J->mcarea + J->szmcarea); ++ J->mcbot = (MCode *)((char *)J->mcarea + sizeof(MCLink)); ++ /* Update the MCLink data for the newly coalesced area */ ++ ((MCLink *)J->mcarea)->next = NULL; ++ ((MCLink *)J->mcarea)->size = J->szmcarea; ++ J->szallmcarea = J->szmcarea; + LJ_LOG("recycled mcarea @ %p (%zuK)", J->mcarea, J->szmcarea / 1024U); } /* -- MCode transactions -------------------------------------------------- */ -@@ -360,6 +373,7 @@ void lj_mcode_clear(jit_State *J) +@@ -289,6 +378,7 @@ void lj_mcode_free(jit_State *J) /* Reserve the remainder of the current MCode area. */ MCode *lj_mcode_reserve(jit_State *J, MCode **lim) { @@ -151,7 +265,7 @@ index 241c86a..198fdd2 100644 if (!J->mcarea) mcode_allocarea(J); else -@@ -424,6 +438,7 @@ void lj_mcode_limiterr(jit_State *J, size_t need) +@@ -353,6 +443,7 @@ void lj_mcode_limiterr(jit_State *J, size_t need) sizemcode = (size_t)J->param[JIT_P_sizemcode] << 10; sizemcode = (sizemcode + LJ_PAGESIZE-1) & ~(size_t)(LJ_PAGESIZE - 1); maxmcode = (size_t)J->param[JIT_P_maxmcode] << 10; @@ -159,18 +273,34 @@ index 241c86a..198fdd2 100644 if ((size_t)need > sizemcode) lj_trace_err(J, LJ_TRERR_MCODEOV); /* Too long for any area. */ if (J->szallmcarea + sizemcode > maxmcode) +diff --git a/src/lj_mcode.h b/src/lj_mcode.h +index 2f31bf5..27f9c83 100644 +--- a/src/lj_mcode.h ++++ b/src/lj_mcode.h +@@ -17,6 +17,7 @@ LJ_FUNC void lj_mcode_sync(void *start, void *end); + #include "lj_jit.h" + + LJ_FUNC void lj_mcode_free(jit_State *J); ++LJ_FUNC void lj_mcode_clear(jit_State *J); + LJ_FUNC MCode *lj_mcode_reserve(jit_State *J, MCode **lim); + LJ_FUNC void lj_mcode_commit(jit_State *J, MCode *m); + LJ_FUNC void lj_mcode_abort(jit_State *J); diff --git a/src/lj_trace.c b/src/lj_trace.c -index e70756c..bacca91 100644 +index a398089..bacca91 100644 --- a/src/lj_trace.c +++ b/src/lj_trace.c -@@ -298,6 +298,7 @@ int lj_trace_flushall(lua_State *L) +@@ -297,8 +297,9 @@ int lj_trace_flushall(lua_State *L) + J->freetrace = 0; /* Clear penalty cache. */ memset(J->penalty, 0, sizeof(J->penalty)); - /* Clear the whole machine code and invalidate all exit stub groups. */ +- /* Free the whole machine code and invalidate all exit stub groups. */ +- lj_mcode_free(J); ++ /* Clear the whole machine code and invalidate all exit stub groups. */ + LJ_LOG("will clear mcode"); - lj_mcode_clear(J); ++ lj_mcode_clear(J); memset(J->exitstubgroup, 0, sizeof(J->exitstubgroup)); lj_vmevent_send(L, TRACE, + setstrV(L, L->top++, lj_str_newlit(L, "flush")); @@ -361,6 +362,7 @@ void lj_trace_freestate(global_State *g) "trace still allocated"); } @@ -187,11 +317,29 @@ index e70756c..bacca91 100644 lj_trace_flushall(J->L); J->state = LJ_TRACE_IDLE; /* Silently ignored. */ return; -@@ -622,6 +625,7 @@ static int trace_abort(jit_State *J) +@@ -621,8 +624,10 @@ static int trace_abort(jit_State *J) + L->top--; /* Remove error object */ if (e == LJ_TRERR_DOWNREC) return trace_downrec(J); - else if (e == LJ_TRERR_MCODEAL) { +- else if (e == LJ_TRERR_MCODEAL) ++ else if (e == LJ_TRERR_MCODEAL) { + LJ_LOG("LJ_TRERR_MCODEAL -> flush"); lj_trace_flushall(L); - } ++ } return 0; + } + +diff --git a/src/ljamalg.c b/src/ljamalg.c +index 56585e6..9c77132 100644 +--- a/src/ljamalg.c ++++ b/src/ljamalg.c +@@ -18,6 +18,9 @@ + #include "lua.h" + #include "lauxlib.h" + ++/* Android hack: make a page-aligned 1MB hole we can use for the mcode area... */ ++unsigned char g_lj_mcarea_reserve[1U<<20U] __attribute__((aligned(4096))) = { 0 }; ++ + #include "lj_assert.c" + #include "lj_gc.c" + #include "lj_err.c" diff --git a/jni/luajit/koreader-luajit-mcode-reserve-hack.patch b/jni/luajit/koreader-luajit-mcode-reserve-hack.patch index ebb2f62f9..f76673f6a 100644 --- a/jni/luajit/koreader-luajit-mcode-reserve-hack.patch +++ b/jni/luajit/koreader-luajit-mcode-reserve-hack.patch @@ -23,7 +23,7 @@ index 655b84c..1fa6292 100644 MCode *mctop; /* Top of current mcode area. */ MCode *mcbot; /* Bottom of current mcode area. */ diff --git a/src/lj_mcode.c b/src/lj_mcode.c -index a5153b2..241c86a 100644 +index a5153b2..4566158 100644 --- a/src/lj_mcode.c +++ b/src/lj_mcode.c @@ -98,9 +98,13 @@ static int mcode_setprot(void *p, size_t sz, DWORD prot) @@ -103,11 +103,11 @@ index a5153b2..241c86a 100644 #endif } -@@ -280,8 +303,56 @@ void lj_mcode_free(jit_State *J) +@@ -280,8 +303,61 @@ void lj_mcode_free(jit_State *J) while (mc) { MCode *next = ((MCLink *)mc)->next; mcode_free(J, mc, ((MCLink *)mc)->size); -+ /* Remember the oldest link as lastmcarea */ ++ /* Remember the oldest (i.e., highest address) link as lastmcarea */ + if (!next) { + J->lastmcarea = mc; + } @@ -119,44 +119,49 @@ index a5153b2..241c86a 100644 +void lj_mcode_clear(jit_State *J) +{ + MCode *mc = J->mcarea; ++ /* Keep track of the previous link in the chain */ + MCode *mcarea = J->mcarea; + size_t szallmcarea = 0; -+ size_t prevsize = 0; ++ size_t szmcarea = 0; + while (mc) { + MCode *next = ((MCLink *)mc)->next; + size_t size = ((MCLink *)mc)->size; + /* Reset mcarea to the oldest contiguous link */ -+ if ((next && next == mc + size) || (!next && mc == mcarea + prevsize) || (!next && mc == mcarea)) { -+ szallmcarea += size; ++ if ((next && next == mc + size) || (!next && mc == mcarea + szmcarea) || (!next && mc == mcarea)) { ++ /* ^ next link is contiguous ^ last link is contiguous ^ single link in the chain */ + mcarea = mc; -+ prevsize = size; ++ szmcarea = size; ++ szallmcarea += size; + } else { + mcarea = NULL; ++ /* A non-contiguous link anywhere in the chain means we scrap the whole chain, to keep things simple */ ++ break; + } mc = next; } + -+ /* If there were non-contiguous links, fallback to lj_mcode_free */ ++ /* If we hit a non-contiguous links, fallback to lj_mcode_free */ + if (!mcarea) { + return lj_mcode_free(J); + } + -+ /* Recycle the full chain */ -+ J->mcarea = mcarea + prevsize - szallmcarea; ++ /* Ready to recycle the full chain */ ++ /* Rewind to the lowest address (as links are allocated high to low, c.f., mcode_alloc) */ ++ J->mcarea = mcarea + szmcarea - szallmcarea; + J->szmcarea = szallmcarea; -+ J->szallmcarea = szallmcarea; -+ /* Tell the JIT that it once again has the full area available to generate code in, c.f., mcode_allocarea */ -+ J->mctop = (MCode *)((char *)J->mcarea + J->szmcarea); -+ J->mcbot = (MCode *)((char *)J->mcarea + sizeof(MCLink)); + /* We need write access to clear it */ + if (LJ_UNLIKELY(mcode_setprot(J->mcarea, J->szmcarea, MCPROT_GEN))) + mcode_protfail(J); ++ /* Update the protection cache */ ++ J->mcprot = MCPROT_GEN; + memset(J->mcarea, 0, J->szmcarea); ++ /* Tell the JIT that it once again has the full area available to generate code in, c.f., mcode_allocarea */ ++ J->mctop = (MCode *)((char *)J->mcarea + J->szmcarea); ++ J->mcbot = (MCode *)((char *)J->mcarea + sizeof(MCLink)); + /* Update the MCLink data for the newly coalesced area */ + ((MCLink *)J->mcarea)->next = NULL; + ((MCLink *)J->mcarea)->size = J->szmcarea; -+ /* Update the protection cache */ -+ J->mcprot = MCPROT_GEN; ++ J->szallmcarea = J->szmcarea; } /* -- MCode transactions -------------------------------------------------- */ From 4a8487ba128ea1171efd213af61444cc2bd6d2b4 Mon Sep 17 00:00:00 2001 From: NiLuJe Date: Sat, 2 Jan 2021 16:26:05 +0100 Subject: [PATCH 2/5] Fix the debug patch -_-" --- jni/luajit/koreader-luajit-mcode-debug.patch | 239 ++++--------------- 1 file changed, 46 insertions(+), 193 deletions(-) diff --git a/jni/luajit/koreader-luajit-mcode-debug.patch b/jni/luajit/koreader-luajit-mcode-debug.patch index 0bdf12ed2..3cb310b4b 100644 --- a/jni/luajit/koreader-luajit-mcode-debug.patch +++ b/jni/luajit/koreader-luajit-mcode-debug.patch @@ -23,17 +23,10 @@ index aae7b5b..b675e96 100644 #ifdef LUA_USE_ASSERT as->mcp_prev = as->mcp; diff --git a/src/lj_def.h b/src/lj_def.h -index cfe18c4..1a52020 100644 +index 5b71dfc..1a52020 100644 --- a/src/lj_def.h +++ b/src/lj_def.h -@@ -39,12 +39,22 @@ typedef int intptr_t; - typedef unsigned int uintptr_t; - #else - #include -+#include - #endif - - /* Needed everywhere. */ +@@ -46,6 +46,15 @@ typedef unsigned int uintptr_t; #include #include @@ -61,40 +54,15 @@ index 1d0ff54..8ef29a3 100644 lj_trace_flushall(L); } else { if (!(mode & LUAJIT_MODE_ON)) -diff --git a/src/lj_jit.h b/src/lj_jit.h -index 655b84c..1fa6292 100644 ---- a/src/lj_jit.h -+++ b/src/lj_jit.h -@@ -490,6 +490,7 @@ typedef struct jit_State { - BCIns patchins; /* Instruction for pending re-patch. */ - - int mcprot; /* Protection of current mcode area. */ -+ MCode *lastmcarea; /* Base of last unmapped mcode area (i.e., the previous one). */ - MCode *mcarea; /* Base of current mcode area. */ - MCode *mctop; /* Top of current mcode area. */ - MCode *mcbot; /* Bottom of current mcode area. */ diff --git a/src/lj_mcode.c b/src/lj_mcode.c -index a5153b2..34c06c0 100644 +index 4566158..ee3825f 100644 --- a/src/lj_mcode.c +++ b/src/lj_mcode.c -@@ -98,19 +98,25 @@ static int mcode_setprot(void *p, size_t sz, DWORD prot) - #define MCPROT_RX (PROT_READ|PROT_EXEC) - #define MCPROT_RWX (PROT_READ|PROT_WRITE|PROT_EXEC) - --static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot) -+static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot, bool fixed) - { -- void *p = mmap((void *)hint, sz, prot, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); -+ int flags = MAP_PRIVATE | MAP_ANONYMOUS; -+ if (fixed) { -+ flags |= MAP_FIXED; -+ } -+ void *p = mmap((void *)hint, sz, prot, flags, -1, 0); - if (p == MAP_FAILED) { +@@ -109,12 +109,14 @@ static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot, b if (!hint) lj_trace_err(J, LJ_TRERR_MCODEAL); p = NULL; } -+ LJ_LOG("mapped at @ %p (%zuK)", p, sz / 1024U); ++ LJ_LOG("mapped %zuK at @ %p", sz / 1024U, p); return p; } @@ -105,51 +73,24 @@ index a5153b2..34c06c0 100644 munmap(p, sz); } -@@ -210,24 +216,48 @@ static void *mcode_alloc(jit_State *J, size_t sz) +@@ -214,6 +216,8 @@ static void *mcode_alloc(jit_State *J, size_t sz) uintptr_t target = (uintptr_t)(void *)lj_vm_exit_handler & ~(uintptr_t)0xffff; #endif const uintptr_t range = (1u << (LJ_TARGET_JUMPRANGE-1)) - (1u << 21); -- /* First try a contiguous area below the last one. */ -- uintptr_t hint = J->mcarea ? (uintptr_t)J->mcarea - sz : 0; + LJ_LOG("target is @ %p", (void *) target); + LJ_LOG("range is %p", (void *) range); -+ /* First try a contiguous area below the last one, -+ * then try the same address as the last area we unmapped -+ * (this happens after a flush (either explicit or because the mcarea was filled), -+ * although not in our case, since we patch flushall to clear the region instead of unmapping it now), -+ * and otherwise, use the area we reserved inside the amalgam's address space -+ * (it's currently hard-coded at 1MB, which is twice the default maxmcode). */ -+ uintptr_t reserve = 0; -+ bool fixed = false; -+ /* Only use the reserve for the very first alloc */ -+ if (!J->mcarea && !J->lastmcarea) { -+ reserve = (uintptr_t) &g_lj_mcarea_reserve; -+ /* We'll only use it once, and we *know* it's there, so use MAP_FIXED */ -+ fixed = true; -+ } -+ uintptr_t hint = J->mcarea ? (uintptr_t)J->mcarea - sz : J->lastmcarea ? (uintptr_t)J->lastmcarea : reserve; + /* First try a contiguous area below the last one, + * then try the same address as the last area we unmapped + * (this happens after a flush (either explicit or because the mcarea was filled), +@@ -229,6 +233,7 @@ static void *mcode_alloc(jit_State *J, size_t sz) + fixed = true; + } + uintptr_t hint = J->mcarea ? (uintptr_t)J->mcarea - sz : J->lastmcarea ? (uintptr_t)J->lastmcarea : reserve; + LJ_LOG("original hint is @ %p (mcarea: %p / last: %p / reserve: %p)", (void *) hint, (void *) J->mcarea, (void *) J->lastmcarea, (void *) reserve); int i; /* Limit probing iterations, depending on the available pool size. */ for (i = 0; i < LJ_TARGET_JUMPRANGE; i++) { - if (mcode_validptr(hint)) { -- void *p = mcode_alloc_at(J, hint, sz, MCPROT_GEN); -+ void *p = mcode_alloc_at(J, hint, sz, MCPROT_GEN, fixed); - - if (mcode_validptr(p) && -- ((uintptr_t)p + sz - target < range || target - (uintptr_t)p < range)) -- return p; -- if (p) mcode_free(J, p, sz); /* Free badly placed area. */ -+ ((uintptr_t)p + sz - target < range || target - (uintptr_t)p < range)) { -+ return p; -+ } -+ if (p) { -+ /* Free badly placed area. */ -+ mcode_free(J, p, sz); -+ /* Ensure the next iteration won't try to use MAP_FIXED */ -+ fixed = false; -+ } - } +@@ -249,8 +254,10 @@ static void *mcode_alloc(jit_State *J, size_t sz) /* Next try probing 64K-aligned pseudo-random addresses. */ do { hint = lj_prng_u64(&J2G(J)->prng) & ((1u<param[JIT_P_sizemcode] << 10; sz = (sz + LJ_PAGESIZE-1) & ~(size_t)(LJ_PAGESIZE - 1); J->mcarea = (MCode *)mcode_alloc(J, sz); @@ -185,7 +109,7 @@ index a5153b2..34c06c0 100644 J->szmcarea = sz; J->mcprot = MCPROT_GEN; J->mctop = (MCode *)((char *)J->mcarea + J->szmcarea); -@@ -275,13 +306,71 @@ static void mcode_allocarea(jit_State *J) +@@ -298,6 +306,7 @@ static void mcode_allocarea(jit_State *J) void lj_mcode_free(jit_State *J) { MCode *mc = J->mcarea; @@ -193,71 +117,34 @@ index a5153b2..34c06c0 100644 J->mcarea = NULL; J->szallmcarea = 0; while (mc) { - MCode *next = ((MCLink *)mc)->next; - mcode_free(J, mc, ((MCLink *)mc)->size); -+ /* Remember the oldest (i.e., highest address) link as lastmcarea */ -+ if (!next) { -+ J->lastmcarea = mc; -+ } -+ mc = next; -+ } +@@ -309,6 +318,7 @@ void lj_mcode_free(jit_State *J) + } + mc = next; + } + LJ_LOG("lastmcarea is now %p", J->lastmcarea); -+} -+ -+/* Clear all MCode areas. */ -+void lj_mcode_clear(jit_State *J) -+{ -+ MCode *mc = J->mcarea; -+ /* Keep track of the previous link in the chain */ -+ MCode *mcarea = J->mcarea; -+ size_t szallmcarea = 0; -+ size_t szmcarea = 0; -+ while (mc) { -+ MCode *next = ((MCLink *)mc)->next; -+ size_t size = ((MCLink *)mc)->size; -+ /* Reset mcarea to the oldest contiguous link */ -+ if ((next && next == mc + size) || (!next && mc == mcarea + szmcarea) || (!next && mc == mcarea)) { -+ /* ^ next link is contiguous ^ last link is contiguous ^ single link in the chain */ -+ mcarea = mc; -+ szmcarea = size; -+ szallmcarea += size; + } + + /* Clear all MCode areas. */ +@@ -328,8 +338,10 @@ void lj_mcode_clear(jit_State *J) + mcarea = mc; + szmcarea = size; + szallmcarea += size; + LJ_LOG("contiguous %zuK link detected @ %p (total: %zuK) (next @ %p)", size / 1024U, mc, szallmcarea / 1024U, next); -+ } else { -+ mcarea = NULL; + } else { + mcarea = NULL; + LJ_LOG("non-contiguous %zuK link detected @ %p (next @ %p)!", size / 1024U, mc, next); -+ /* A non-contiguous link anywhere in the chain means we scrap the whole chain, to keep things simple */ -+ break; -+ } - mc = next; - } -+ -+ /* If we hit a non-contiguous links, fallback to lj_mcode_free */ -+ if (!mcarea) { -+ return lj_mcode_free(J); -+ } -+ -+ /* Ready to recycle the full chain */ -+ /* Rewind to the lowest address (as links are allocated high to low, c.f., mcode_alloc) */ -+ J->mcarea = mcarea + szmcarea - szallmcarea; -+ J->szmcarea = szallmcarea; -+ /* We need write access to clear it */ -+ if (LJ_UNLIKELY(mcode_setprot(J->mcarea, J->szmcarea, MCPROT_GEN))) -+ mcode_protfail(J); -+ /* Update the protection cache */ -+ J->mcprot = MCPROT_GEN; -+ memset(J->mcarea, 0, J->szmcarea); -+ /* Tell the JIT that it once again has the full area available to generate code in, c.f., mcode_allocarea */ -+ J->mctop = (MCode *)((char *)J->mcarea + J->szmcarea); -+ J->mcbot = (MCode *)((char *)J->mcarea + sizeof(MCLink)); -+ /* Update the MCLink data for the newly coalesced area */ -+ ((MCLink *)J->mcarea)->next = NULL; -+ ((MCLink *)J->mcarea)->size = J->szmcarea; -+ J->szallmcarea = J->szmcarea; -+ LJ_LOG("recycled mcarea @ %p (%zuK)", J->mcarea, J->szmcarea / 1024U); + /* A non-contiguous link anywhere in the chain means we scrap the whole chain, to keep things simple */ + break; + } +@@ -358,6 +370,7 @@ void lj_mcode_clear(jit_State *J) + ((MCLink *)J->mcarea)->next = NULL; + ((MCLink *)J->mcarea)->size = J->szmcarea; + J->szallmcarea = J->szmcarea; ++ LJ_LOG("recycled %zuK mcarea @ %p", J->szmcarea / 1024U, J->mcarea); } /* -- MCode transactions -------------------------------------------------- */ -@@ -289,6 +378,7 @@ void lj_mcode_free(jit_State *J) +@@ -365,6 +378,7 @@ void lj_mcode_clear(jit_State *J) /* Reserve the remainder of the current MCode area. */ MCode *lj_mcode_reserve(jit_State *J, MCode **lim) { @@ -265,7 +152,7 @@ index a5153b2..34c06c0 100644 if (!J->mcarea) mcode_allocarea(J); else -@@ -353,6 +443,7 @@ void lj_mcode_limiterr(jit_State *J, size_t need) +@@ -429,6 +443,7 @@ void lj_mcode_limiterr(jit_State *J, size_t need) sizemcode = (size_t)J->param[JIT_P_sizemcode] << 10; sizemcode = (sizemcode + LJ_PAGESIZE-1) & ~(size_t)(LJ_PAGESIZE - 1); maxmcode = (size_t)J->param[JIT_P_maxmcode] << 10; @@ -273,34 +160,18 @@ index a5153b2..34c06c0 100644 if ((size_t)need > sizemcode) lj_trace_err(J, LJ_TRERR_MCODEOV); /* Too long for any area. */ if (J->szallmcarea + sizemcode > maxmcode) -diff --git a/src/lj_mcode.h b/src/lj_mcode.h -index 2f31bf5..27f9c83 100644 ---- a/src/lj_mcode.h -+++ b/src/lj_mcode.h -@@ -17,6 +17,7 @@ LJ_FUNC void lj_mcode_sync(void *start, void *end); - #include "lj_jit.h" - - LJ_FUNC void lj_mcode_free(jit_State *J); -+LJ_FUNC void lj_mcode_clear(jit_State *J); - LJ_FUNC MCode *lj_mcode_reserve(jit_State *J, MCode **lim); - LJ_FUNC void lj_mcode_commit(jit_State *J, MCode *m); - LJ_FUNC void lj_mcode_abort(jit_State *J); diff --git a/src/lj_trace.c b/src/lj_trace.c -index a398089..bacca91 100644 +index e70756c..bacca91 100644 --- a/src/lj_trace.c +++ b/src/lj_trace.c -@@ -297,8 +297,9 @@ int lj_trace_flushall(lua_State *L) - J->freetrace = 0; +@@ -298,6 +298,7 @@ int lj_trace_flushall(lua_State *L) /* Clear penalty cache. */ memset(J->penalty, 0, sizeof(J->penalty)); -- /* Free the whole machine code and invalidate all exit stub groups. */ -- lj_mcode_free(J); -+ /* Clear the whole machine code and invalidate all exit stub groups. */ + /* Clear the whole machine code and invalidate all exit stub groups. */ + LJ_LOG("will clear mcode"); -+ lj_mcode_clear(J); + lj_mcode_clear(J); memset(J->exitstubgroup, 0, sizeof(J->exitstubgroup)); lj_vmevent_send(L, TRACE, - setstrV(L, L->top++, lj_str_newlit(L, "flush")); @@ -361,6 +362,7 @@ void lj_trace_freestate(global_State *g) "trace still allocated"); } @@ -317,29 +188,11 @@ index a398089..bacca91 100644 lj_trace_flushall(J->L); J->state = LJ_TRACE_IDLE; /* Silently ignored. */ return; -@@ -621,8 +624,10 @@ static int trace_abort(jit_State *J) - L->top--; /* Remove error object */ +@@ -622,6 +625,7 @@ static int trace_abort(jit_State *J) if (e == LJ_TRERR_DOWNREC) return trace_downrec(J); -- else if (e == LJ_TRERR_MCODEAL) -+ else if (e == LJ_TRERR_MCODEAL) { + else if (e == LJ_TRERR_MCODEAL) { + LJ_LOG("LJ_TRERR_MCODEAL -> flush"); lj_trace_flushall(L); -+ } + } return 0; - } - -diff --git a/src/ljamalg.c b/src/ljamalg.c -index 56585e6..9c77132 100644 ---- a/src/ljamalg.c -+++ b/src/ljamalg.c -@@ -18,6 +18,9 @@ - #include "lua.h" - #include "lauxlib.h" - -+/* Android hack: make a page-aligned 1MB hole we can use for the mcode area... */ -+unsigned char g_lj_mcarea_reserve[1U<<20U] __attribute__((aligned(4096))) = { 0 }; -+ - #include "lj_assert.c" - #include "lj_gc.c" - #include "lj_err.c" From 151e5d5df1d574dafc14582007d020f578189d3d Mon Sep 17 00:00:00 2001 From: NiLuJe Date: Sat, 2 Jan 2021 16:28:30 +0100 Subject: [PATCH 3/5] Comment typo --- jni/luajit/koreader-luajit-mcode-debug.patch | 2 +- jni/luajit/koreader-luajit-mcode-reserve-hack.patch | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jni/luajit/koreader-luajit-mcode-debug.patch b/jni/luajit/koreader-luajit-mcode-debug.patch index 3cb310b4b..378929592 100644 --- a/jni/luajit/koreader-luajit-mcode-debug.patch +++ b/jni/luajit/koreader-luajit-mcode-debug.patch @@ -55,7 +55,7 @@ index 1d0ff54..8ef29a3 100644 } else { if (!(mode & LUAJIT_MODE_ON)) diff --git a/src/lj_mcode.c b/src/lj_mcode.c -index 4566158..ee3825f 100644 +index 9cb7a21..66b93de 100644 --- a/src/lj_mcode.c +++ b/src/lj_mcode.c @@ -109,12 +109,14 @@ static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot, b diff --git a/jni/luajit/koreader-luajit-mcode-reserve-hack.patch b/jni/luajit/koreader-luajit-mcode-reserve-hack.patch index f76673f6a..235f21380 100644 --- a/jni/luajit/koreader-luajit-mcode-reserve-hack.patch +++ b/jni/luajit/koreader-luajit-mcode-reserve-hack.patch @@ -23,7 +23,7 @@ index 655b84c..1fa6292 100644 MCode *mctop; /* Top of current mcode area. */ MCode *mcbot; /* Bottom of current mcode area. */ diff --git a/src/lj_mcode.c b/src/lj_mcode.c -index a5153b2..4566158 100644 +index a5153b2..9cb7a21 100644 --- a/src/lj_mcode.c +++ b/src/lj_mcode.c @@ -98,9 +98,13 @@ static int mcode_setprot(void *p, size_t sz, DWORD prot) @@ -140,7 +140,7 @@ index a5153b2..4566158 100644 mc = next; } + -+ /* If we hit a non-contiguous links, fallback to lj_mcode_free */ ++ /* If we hit a non-contiguous link, fallback to lj_mcode_free */ + if (!mcarea) { + return lj_mcode_free(J); + } From 0702541f7812e3b366b5fc5da6c1b26fbe176e2d Mon Sep 17 00:00:00 2001 From: NiLuJe Date: Sat, 2 Jan 2021 16:40:44 +0100 Subject: [PATCH 4/5] Forgot a div there --- jni/luajit/koreader-luajit-mcode-debug.patch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jni/luajit/koreader-luajit-mcode-debug.patch b/jni/luajit/koreader-luajit-mcode-debug.patch index 378929592..20f4f8144 100644 --- a/jni/luajit/koreader-luajit-mcode-debug.patch +++ b/jni/luajit/koreader-luajit-mcode-debug.patch @@ -55,7 +55,7 @@ index 1d0ff54..8ef29a3 100644 } else { if (!(mode & LUAJIT_MODE_ON)) diff --git a/src/lj_mcode.c b/src/lj_mcode.c -index 9cb7a21..66b93de 100644 +index 9cb7a21..87e026a 100644 --- a/src/lj_mcode.c +++ b/src/lj_mcode.c @@ -109,12 +109,14 @@ static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot, b @@ -105,7 +105,7 @@ index 9cb7a21..66b93de 100644 size_t sz = (size_t)J->param[JIT_P_sizemcode] << 10; sz = (sz + LJ_PAGESIZE-1) & ~(size_t)(LJ_PAGESIZE - 1); J->mcarea = (MCode *)mcode_alloc(J, sz); -+ LJ_LOG("new %zuK mcarea @ %p (oldarea @ %p)", sz, J->mcarea, oldarea); ++ LJ_LOG("new %zuK mcarea @ %p (oldarea @ %p)", sz, J->mcarea / 1024U, oldarea); J->szmcarea = sz; J->mcprot = MCPROT_GEN; J->mctop = (MCode *)((char *)J->mcarea + J->szmcarea); From 52487947f4ee1adc11a5fe7733a06f37dcb29d8f Mon Sep 17 00:00:00 2001 From: NiLuJe Date: Sat, 2 Jan 2021 16:41:29 +0100 Subject: [PATCH 5/5] In the right place would be better... --- jni/luajit/koreader-luajit-mcode-debug.patch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jni/luajit/koreader-luajit-mcode-debug.patch b/jni/luajit/koreader-luajit-mcode-debug.patch index 20f4f8144..d7eed3322 100644 --- a/jni/luajit/koreader-luajit-mcode-debug.patch +++ b/jni/luajit/koreader-luajit-mcode-debug.patch @@ -55,7 +55,7 @@ index 1d0ff54..8ef29a3 100644 } else { if (!(mode & LUAJIT_MODE_ON)) diff --git a/src/lj_mcode.c b/src/lj_mcode.c -index 9cb7a21..87e026a 100644 +index 9cb7a21..4b7b649 100644 --- a/src/lj_mcode.c +++ b/src/lj_mcode.c @@ -109,12 +109,14 @@ static void *mcode_alloc_at(jit_State *J, uintptr_t hint, size_t sz, int prot, b @@ -105,7 +105,7 @@ index 9cb7a21..87e026a 100644 size_t sz = (size_t)J->param[JIT_P_sizemcode] << 10; sz = (sz + LJ_PAGESIZE-1) & ~(size_t)(LJ_PAGESIZE - 1); J->mcarea = (MCode *)mcode_alloc(J, sz); -+ LJ_LOG("new %zuK mcarea @ %p (oldarea @ %p)", sz, J->mcarea / 1024U, oldarea); ++ LJ_LOG("new %zuK mcarea @ %p (oldarea @ %p)", sz / 1024U, J->mcarea, oldarea); J->szmcarea = sz; J->mcprot = MCPROT_GEN; J->mctop = (MCode *)((char *)J->mcarea + J->szmcarea);