From 9da06535092d6d9dec442641a26c64bce5574322 Mon Sep 17 00:00:00 2001 From: Mike Pall Date: Sun, 24 Jun 2018 14:08:59 +0200 Subject: [PATCH 47/72] ARM64: Fix exit stub patching. Contributed by Javier Guerra Giraldez. --- src/lj_asm_arm64.h | 64 +++++++++++++++++++++++++------------------ src/lj_emit_arm64.h | 18 ++++++------ src/lj_target_arm64.h | 7 +++-- 3 files changed, 51 insertions(+), 38 deletions(-) diff --git a/src/lj_asm_arm64.h b/src/lj_asm_arm64.h index cbb186d..baafa21 100644 --- a/src/lj_asm_arm64.h +++ b/src/lj_asm_arm64.h @@ -56,11 +56,11 @@ static void asm_exitstub_setup(ASMState *as, ExitNo nexits) asm_mclimit(as); /* 1: str lr,[sp]; bl ->vm_exit_handler; movz w0,traceno; bl <1; bl <1; ... */ for (i = nexits-1; (int32_t)i >= 0; i--) - *--mxp = A64I_LE(A64I_BL|((-3-i)&0x03ffffffu)); - *--mxp = A64I_LE(A64I_MOVZw|A64F_U16(as->T->traceno)); + *--mxp = A64I_LE(A64I_BL | A64F_S26(-3-i)); + *--mxp = A64I_LE(A64I_MOVZw | A64F_U16(as->T->traceno)); mxp--; - *mxp = A64I_LE(A64I_BL|(((MCode *)(void *)lj_vm_exit_handler-mxp)&0x03ffffffu)); - *--mxp = A64I_LE(A64I_STRx|A64F_D(RID_LR)|A64F_N(RID_SP)); + *mxp = A64I_LE(A64I_BL | A64F_S26(((MCode *)(void *)lj_vm_exit_handler-mxp))); + *--mxp = A64I_LE(A64I_STRx | A64F_D(RID_LR) | A64F_N(RID_SP)); as->mctop = mxp; } @@ -77,7 +77,7 @@ static void asm_guardcc(ASMState *as, A64CC cc) MCode *p = as->mcp; if (LJ_UNLIKELY(p == as->invmcp)) { as->loopinv = 1; - *p = A64I_B | ((target-p) & 0x03ffffffu); + *p = A64I_B | A64F_S26(target-p); emit_cond_branch(as, cc^1, p-1); return; } @@ -91,7 +91,7 @@ static void asm_guardtnb(ASMState *as, A64Ins ai, Reg r, uint32_t bit) MCode *p = as->mcp; if (LJ_UNLIKELY(p == as->invmcp)) { as->loopinv = 1; - *p = A64I_B | ((target-p) & 0x03ffffffu); + *p = A64I_B | A64F_S26(target-p); emit_tnb(as, ai^0x01000000u, r, bit, p-1); return; } @@ -105,7 +105,7 @@ static void asm_guardcnb(ASMState *as, A64Ins ai, Reg r) MCode *p = as->mcp; if (LJ_UNLIKELY(p == as->invmcp)) { as->loopinv = 1; - *p = A64I_B | ((target-p) & 0x03ffffffu); + *p = A64I_B | A64F_S26(target-p); emit_cnb(as, ai^0x01000000u, r, p-1); return; } @@ -1850,7 +1850,7 @@ static void asm_loop_fixup(ASMState *as) p[-2] |= ((uint32_t)delta & mask) << 5; } else { ptrdiff_t delta = target - (p - 1); - p[-1] = A64I_B | ((uint32_t)(delta) & 0x03ffffffu); + p[-1] = A64I_B | A64F_S26(delta); } } @@ -1919,7 +1919,7 @@ static void asm_tail_fixup(ASMState *as, TraceNo lnk) } /* Patch exit branch. */ target = lnk ? traceref(as->J, lnk)->mcode : (MCode *)lj_vm_exit_interp; - p[-1] = A64I_B | (((target-p)+1)&0x03ffffffu); + p[-1] = A64I_B | A64F_S26((target-p)+1); } /* Prepare tail of code. */ @@ -1982,40 +1982,50 @@ void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target) { MCode *p = T->mcode; MCode *pe = (MCode *)((char *)p + T->szmcode); - MCode *cstart = NULL, *cend = p; + MCode *cstart = NULL; MCode *mcarea = lj_mcode_patch(J, p, 0); MCode *px = exitstub_trace_addr(T, exitno); + /* Note: this assumes a trace exit is only ever patched once. */ for (; p < pe; p++) { /* Look for exitstub branch, replace with branch to target. */ + ptrdiff_t delta = target - p; MCode ins = A64I_LE(*p); if ((ins & 0xff000000u) == 0x54000000u && ((ins ^ ((px-p)<<5)) & 0x00ffffe0u) == 0) { - /* Patch bcc exitstub. */ - *p = A64I_LE((ins & 0xff00001fu) | (((target-p)<<5) & 0x00ffffe0u)); - cend = p+1; - if (!cstart) cstart = p; + /* Patch bcc, if within range. */ + if (A64F_S_OK(delta, 19)) { + *p = A64I_LE((ins & 0xff00001fu) | A64F_S19(delta)); + if (!cstart) cstart = p; + } } else if ((ins & 0xfc000000u) == 0x14000000u && ((ins ^ (px-p)) & 0x03ffffffu) == 0) { - /* Patch b exitstub. */ - *p = A64I_LE((ins & 0xfc000000u) | ((target-p) & 0x03ffffffu)); - cend = p+1; + /* Patch b. */ + lua_assert(A64F_S_OK(delta, 26)); + *p = A64I_LE((ins & 0xfc000000u) | A64F_S26(delta)); if (!cstart) cstart = p; } else if ((ins & 0x7e000000u) == 0x34000000u && ((ins ^ ((px-p)<<5)) & 0x00ffffe0u) == 0) { - /* Patch cbz/cbnz exitstub. */ - *p = A64I_LE((ins & 0xff00001f) | (((target-p)<<5) & 0x00ffffe0u)); - cend = p+1; - if (!cstart) cstart = p; + /* Patch cbz/cbnz, if within range. */ + if (A64F_S_OK(delta, 19)) { + *p = A64I_LE((ins & 0xff00001fu) | A64F_S19(delta)); + if (!cstart) cstart = p; + } } else if ((ins & 0x7e000000u) == 0x36000000u && ((ins ^ ((px-p)<<5)) & 0x0007ffe0u) == 0) { - /* Patch tbz/tbnz exitstub. */ - *p = A64I_LE((ins & 0xfff8001fu) | (((target-p)<<5) & 0x0007ffe0u)); - cend = p+1; - if (!cstart) cstart = p; + /* Patch tbz/tbnz, if within range. */ + if (A64F_S_OK(delta, 14)) { + *p = A64I_LE((ins & 0xfff8001fu) | A64F_S14(delta)); + if (!cstart) cstart = p; + } } } - lua_assert(cstart != NULL); - lj_mcode_sync(cstart, cend); + { /* Always patch long-range branch in exit stub itself. */ + ptrdiff_t delta = target - px; + lua_assert(A64F_S_OK(delta, 26)); + *px = A64I_B | A64F_S26(delta); + if (!cstart) cstart = px; + } + lj_mcode_sync(cstart, px+1); lj_mcode_patch(J, mcarea, 1); } diff --git a/src/lj_emit_arm64.h b/src/lj_emit_arm64.h index 6da4c7d..1001b1d 100644 --- a/src/lj_emit_arm64.h +++ b/src/lj_emit_arm64.h @@ -241,7 +241,7 @@ static void emit_loadk(ASMState *as, Reg rd, uint64_t u64, int is64) #define mcpofs(as, k) \ ((intptr_t)((uintptr_t)(k) - (uintptr_t)(as->mcp - 1))) #define checkmcpofs(as, k) \ - ((((mcpofs(as, k)>>2) + 0x00040000) >> 19) == 0) + (A64F_S_OK(mcpofs(as, k)>>2, 19)) static Reg ra_allock(ASMState *as, intptr_t k, RegSet allow); @@ -312,7 +312,7 @@ static void emit_cond_branch(ASMState *as, A64CC cond, MCode *target) { MCode *p = --as->mcp; ptrdiff_t delta = target - p; - lua_assert(((delta + 0x40000) >> 19) == 0); + lua_assert(A64F_S_OK(delta, 19)); *p = A64I_BCC | A64F_S19(delta) | cond; } @@ -320,24 +320,24 @@ static void emit_branch(ASMState *as, A64Ins ai, MCode *target) { MCode *p = --as->mcp; ptrdiff_t delta = target - p; - lua_assert(((delta + 0x02000000) >> 26) == 0); - *p = ai | ((uint32_t)delta & 0x03ffffffu); + lua_assert(A64F_S_OK(delta, 26)); + *p = ai | A64F_S26(delta); } static void emit_tnb(ASMState *as, A64Ins ai, Reg r, uint32_t bit, MCode *target) { MCode *p = --as->mcp; ptrdiff_t delta = target - p; - lua_assert(bit < 63 && ((delta + 0x2000) >> 14) == 0); + lua_assert(bit < 63 && A64F_S_OK(delta, 14)); if (bit > 31) ai |= A64I_X; - *p = ai | A64F_BIT(bit & 31) | A64F_S14((uint32_t)delta & 0x3fffu) | r; + *p = ai | A64F_BIT(bit & 31) | A64F_S14(delta) | r; } static void emit_cnb(ASMState *as, A64Ins ai, Reg r, MCode *target) { MCode *p = --as->mcp; ptrdiff_t delta = target - p; - lua_assert(((delta + 0x40000) >> 19) == 0); + lua_assert(A64F_S_OK(delta, 19)); *p = ai | A64F_S19(delta) | r; } @@ -347,8 +347,8 @@ static void emit_call(ASMState *as, void *target) { MCode *p = --as->mcp; ptrdiff_t delta = (char *)target - (char *)p; - if ((((delta>>2) + 0x02000000) >> 26) == 0) { - *p = A64I_BL | ((uint32_t)(delta>>2) & 0x03ffffffu); + if (A64F_S_OK(delta>>2, 26)) { + *p = A64I_BL | A64F_S26(delta>>2); } else { /* Target out of range: need indirect call. But don't use R0-R7. */ Reg r = ra_allock(as, i64ptr(target), RSET_RANGE(RID_X8, RID_MAX_GPR)-RSET_FIXED); diff --git a/src/lj_target_arm64.h b/src/lj_target_arm64.h index 520023a..a207a2b 100644 --- a/src/lj_target_arm64.h +++ b/src/lj_target_arm64.h @@ -132,9 +132,9 @@ static LJ_AINLINE uint32_t *exitstub_trace_addr_(uint32_t *p, uint32_t exitno) #define A64F_IMMR(x) ((x) << 16) #define A64F_U16(x) ((x) << 5) #define A64F_U12(x) ((x) << 10) -#define A64F_S26(x) (x) +#define A64F_S26(x) (((uint32_t)(x) & 0x03ffffffu)) #define A64F_S19(x) (((uint32_t)(x) & 0x7ffffu) << 5) -#define A64F_S14(x) ((x) << 5) +#define A64F_S14(x) (((uint32_t)(x) & 0x3fffu) << 5) #define A64F_S9(x) ((x) << 12) #define A64F_BIT(x) ((x) << 19) #define A64F_SH(sh, x) (((sh) << 22) | ((x) << 10)) @@ -145,6 +145,9 @@ static LJ_AINLINE uint32_t *exitstub_trace_addr_(uint32_t *p, uint32_t exitno) #define A64F_LSL16(x) (((x) / 16) << 21) #define A64F_BSH(sh) ((sh) << 10) +/* Check for valid field range. */ +#define A64F_S_OK(x, b) ((((x) + (1 << (b-1))) >> (b)) == 0) + typedef enum A64Ins { A64I_S = 0x20000000, A64I_X = 0x80000000, -- 2.20.1