RFC: PHP JIT/arm64 port

Currently PHP JIT only supports x86 and x86_64 CPUs on POSIX platforms
and Windows.[1] With the prevalence of PHP language and the notable
growth of ARM-based servers market, we believe JIT/arm64 would be in
urgent need in the near future.

As an initial effort to enable PHP JIT/arm64, we (ARM) have supported
the basic functionality, and (partially) implemented the compilation for
several opcodes. Currently a number of simple JIT test cases from PHP
test framework can be passed on ARM-based machine. There are still a lot
of missing parts, such as hot loops, class/object/array operations,
exception handling, etc, and we will continue working on them.

In this patch, we would like to share our work with you. Any feedback
would be greatly appreciated, and please let we know if anyone wants to
contribute to this port.

Main updates:
1. JIT backend for AArch64
A new alternative, i.e. AArch64, was added while building PHP JIT. See
the updates in the following files. Note that we adopt capstone[2] for
disassembly on AArch64.

  build/Makefile.global
  ext/opcache/config.m4
  ext/opcache/config.w32
  ext/opcache/jit/Makefile.frag
  ext/opcache/jit/zend_jit.c
  ext/opcache/jit/zend_jit_vm_helpers.c
  ext/opcache/jit/zend_jit_disasm_arm64.c
  ext/opcache/jit/zend_jit_gdb.c
  ext/opcache/jit/zend_jit_perf_dump.c

2. DynASM library
PHP JIT uses DynASM[3] (developed for LuaJIT project) to generate native
code on the fly. We added two useful but missing features, global label
reference and dynamic register names, into DynASM/arm64. See the updates
in files:

  ext/opcache/jit/dynasm/dasm_arm64.h
  ext/opcache/jit/dynasm/dasm_arm64.lua

Note that these two features are available on DynASM/x86.

3. compilation for opcodes on AArch64
Our main work falls in the following files.

  ext/opcache/jit/zend_jit_arm64.h
  ext/opcache/jit/zend_jit_arm64.dasc
  ext/opcache/jit/zend_jit_internal.h
  Zend/zend_vm_opcodes.h

* AArch64 registers and calling conventions are defined.

* Instruction cache must be flushed for the JIT-ed code on AArch64. See
macro JIT_CACHE_FLUSH in file 'zend_jit_internal.h'.

* We have (partially) implemented the compilation for several opcodes,
mainly for the function-based JIT (with opcache.jit=1203). Currently,
test cases involving internal function call (e.g. var_dump), additions
with integers/floating-point numbers, integer overflows and simple
exception, can be supported now. See our newly added test cases under
directory 'ext/opcache/tests/jit/arm64/'.

* Trace counter stubs are implemented for tracing JIT (with
opcache.jit=1255). See zend_jit_hybrid_trace_counter_stub() and
zend_jit_hybrid_hot_trace_stub() in file 'zend_jit_arm64.dasc'. Hot
functions can be recognized and compiled successfully. See the test case
'hot_func_002.phpt'.

How to build and test:
Our local test environment is an ARM-based server with Ubuntu 20.04 and
GCC-10. We follow the building commands as shown in the readme file [4].
Note that library capstone should be installed in advance.

We suggest running the JIT test cases using the following command. In
our local test, 59 out of all 128 cases can be passed currently.

  $ make test TESTS='-d opcache.jit=1203 ext/opcache/tests/jit/'

[1] https://wiki.php.net/rfc/jit
[2] https://www.capstone-engine.org/
[3] https://luajit.org/dynasm.html
[4] https://github.com/php/php-src

Zend/zend_vm_opcodes.h

@@ -35,7 +35,8 @@
 #endif
 
 #if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__)
-# if ((defined(i386) && !defined(__PIC__)) || defined(__x86_64__) || defined(_M_X64))
+# if ((defined(i386) && !defined(__PIC__)) || defined(__x86_64__) || \
+       defined(_M_X64) || defined(__aarch64__))
 #  define ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 16
 # endif
 #endif

build/Makefile.global

@@ -125,6 +125,8 @@ distclean: clean
 	rm -f scripts/man1/phpize.1 scripts/php-config scripts/man1/php-config.1 sapi/cli/php.1 sapi/cgi/php-cgi.1 sapi/phpdbg/phpdbg.1 ext/phar/phar.1 ext/phar/phar.phar.1
 	rm -f sapi/fpm/php-fpm.conf sapi/fpm/init.d.php-fpm sapi/fpm/php-fpm.service sapi/fpm/php-fpm.8 sapi/fpm/status.html
 	rm -f ext/phar/phar.phar ext/phar/phar.php
+	rm -f ext/opcache/jit/zend_jit_x86.c
+	rm -f ext/opcache/jit/zend_jit_arm64.c
 	if test "$(srcdir)" != "$(builddir)"; then \
 	  rm -f ext/phar/phar/phar.inc; \
 	fi

ext/opcache/config.m4

@@ -29,7 +29,7 @@ if test "$PHP_OPCACHE" != "no"; then
 
   if test "$PHP_OPCACHE_JIT" = "yes"; then
     case $host_cpu in
-      x86*)
+      x86*|aarch64)
         ;;
       *)
         AC_MSG_WARN([JIT not supported by host architecture])
@@ -59,18 +59,37 @@ if test "$PHP_OPCACHE" != "no"; then
       case $host_alias in
         *x86_64-*-darwin*)
           DASM_FLAGS="-D X64APPLE=1 -D X64=1"
+          DASM_ARCH="x86"
         ;;
         *x86_64*)
           DASM_FLAGS="-D X64=1"
+          DASM_ARCH="x86"
+        ;;
+        *aarch64*)
+          DASM_FLAGS="-D ARM64=1"
+          DASM_ARCH="arm64"
         ;;
       esac
+    else
+      DASM_ARCH="x86"
     fi
 
     if test "$PHP_THREAD_SAFETY" = "yes"; then
       DASM_FLAGS="$DASM_FLAGS -D ZTS=1"
     fi
 
+    if test "$DASM_ARCH" = "arm64"; then
+      PKG_CHECK_MODULES([CAPSTONE], [capstone >= 3.0.0],
+                        [have_capstone="yes"], [have_capstone="no"])
+      if test "$have_capstone" = "yes"; then
+        AC_DEFINE(HAVE_CAPSTONE, 1, [ ])
+        PHP_EVAL_LIBLINE($CAPSTONE_LIBS, OPCACHE_SHARED_LIBADD)
+        PHP_EVAL_INCLINE($CAPSTONE_CFLAGS)
+      fi
+    fi
+
     PHP_SUBST(DASM_FLAGS)
+    PHP_SUBST(DASM_ARCH)
 
     AC_MSG_CHECKING(for opagent in default path)
     for i in /usr/local /usr; do

ext/opcache/config.w32

@@ -25,6 +25,7 @@ if (PHP_OPCACHE != "no") {
 				dasm_flags += " -D ZTS=1";
 			}
 			DEFINE("DASM_FLAGS", dasm_flags);
+			DEFINE("DASM_ARCH", "x86");
 
 			AC_DEFINE('HAVE_JIT', 1, 'Define to enable JIT');
 			/* XXX read this dynamically */

ext/opcache/jit/Makefile.frag

@@ -2,13 +2,13 @@
 $(builddir)/minilua: $(srcdir)/jit/dynasm/minilua.c
 	$(CC) $(srcdir)/jit/dynasm/minilua.c -lm -o $@
 
-$(builddir)/jit/zend_jit_x86.c: $(srcdir)/jit/zend_jit_x86.dasc $(srcdir)/jit/dynasm/*.lua $(builddir)/minilua
-	$(builddir)/minilua $(srcdir)/jit/dynasm/dynasm.lua  $(DASM_FLAGS) -o $@ $(srcdir)/jit/zend_jit_x86.dasc
+$(builddir)/jit/zend_jit_$(DASM_ARCH).c: $(srcdir)/jit/zend_jit_$(DASM_ARCH).dasc $(srcdir)/jit/dynasm/*.lua $(builddir)/minilua
+	$(builddir)/minilua $(srcdir)/jit/dynasm/dynasm.lua  $(DASM_FLAGS) -o $@ $(srcdir)/jit/zend_jit_$(DASM_ARCH).dasc
 
 $(builddir)/jit/zend_jit.lo: \
-	$(builddir)/jit/zend_jit_x86.c \
+	$(builddir)/jit/zend_jit_$(DASM_ARCH).c \
 	$(srcdir)/jit/zend_jit_helpers.c \
-	$(srcdir)/jit/zend_jit_disasm_x86.c \
+	$(srcdir)/jit/zend_jit_disasm_$(DASM_ARCH).c \
 	$(srcdir)/jit/zend_jit_gdb.c \
 	$(srcdir)/jit/zend_jit_perf_dump.c \
 	$(srcdir)/jit/zend_jit_oprofile.c \

ext/opcache/jit/dynasm/dasm_arm64.h

@@ -23,6 +23,7 @@ enum {
   /* The following actions also have an argument. */
   DASM_REL_PC, DASM_LABEL_PC,
   DASM_IMM, DASM_IMM6, DASM_IMM12, DASM_IMM13W, DASM_IMM13X, DASM_IMML,
+  DASM_VREG,
   DASM__MAX
 };
 
@@ -39,6 +40,7 @@ enum {
 #define DASM_S_RANGE_LG		0x13000000
 #define DASM_S_RANGE_PC		0x14000000
 #define DASM_S_RANGE_REL	0x15000000
+#define DASM_S_RANGE_VREG	0x16000000
 #define DASM_S_UNDEF_LG		0x21000000
 #define DASM_S_UNDEF_PC		0x22000000
 
@@ -312,13 +314,17 @@ void dasm_put(Dst_DECL, int start, ...)
 	}
       case DASM_IMML: {
 #ifdef DASM_CHECKS
-	int scale = (p[-2] >> 30);
+	int scale = (ins & 0x3);
 	CK((!(n & ((1<<scale)-1)) && (unsigned int)(n>>scale) < 4096) ||
 	   (unsigned int)(n+256) < 512, RANGE_I);
 #endif
 	b[pos++] = n;
 	break;
 	}
+      case DASM_VREG:
+	CK(n < 32, RANGE_VREG);
+	b[pos++] = n;
+	break;
       }
     }
   }
@@ -348,7 +354,7 @@ int dasm_link(Dst_DECL, size_t *szp)
 
   { /* Handle globals not defined in this translation unit. */
     int idx;
-    for (idx = 20; idx*sizeof(int) < D->lgsize; idx++) {
+    for (idx = 10; idx*sizeof(int) < D->lgsize; idx++) {
       int n = D->lglabels[idx];
       /* Undefined label: Collapse rel chain and replace with marker (< 0). */
       while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = -idx; }
@@ -377,6 +383,7 @@ int dasm_link(Dst_DECL, size_t *szp)
 	case DASM_IMM: case DASM_IMM6: case DASM_IMM12: case DASM_IMM13W:
 	case DASM_IMML: pos++; break;
 	case DASM_IMM13X: pos += 2; break;
+	case DASM_VREG: pos++; break;
 	}
       }
       stop: (void)0;
@@ -426,6 +433,10 @@ int dasm_encode(Dst_DECL, void *buffer)
 	  ins &= 255; while ((((char *)cp - base) & ins)) *cp++ = 0xe1a00000;
 	  break;
 	case DASM_REL_LG:
+	  if (n < 0) {  /* Global label reference */
+	    n = (int)((ptrdiff_t)D->globals[-n] - (ptrdiff_t)cp + 4);
+	    goto patchrel;
+	  }
 	  CK(n >= 0, UNDEF_LG);
 	case DASM_REL_PC:
 	  CK(n >= 0, UNDEF_PC);
@@ -467,11 +478,14 @@ int dasm_encode(Dst_DECL, void *buffer)
 	  cp[-1] |= (dasm_imm13(n, *b++) << 10);
 	  break;
 	case DASM_IMML: {
-	  int scale = (p[-2] >> 30);
+	  int scale = (ins & 0x3);
 	  cp[-1] |= (!(n & ((1<<scale)-1)) && (unsigned int)(n>>scale) < 4096) ?
 	    ((n << (10-scale)) | 0x01000000) : ((n & 511) << 12);
 	  break;
 	  }
+	case DASM_VREG:
+	  cp[-1] |= (n & 0x1f) << (ins & 0x1f);
+	  break;
 	default: *cp++ = ins; break;
 	}
       }

ext/opcache/jit/dynasm/dasm_arm64.lua

@@ -40,6 +40,7 @@ local action_names = {
   "STOP", "SECTION", "ESC", "REL_EXT",
   "ALIGN", "REL_LG", "LABEL_LG",
   "REL_PC", "LABEL_PC", "IMM", "IMM6", "IMM12", "IMM13W", "IMM13X", "IMML",
+  "VREG"
 }
 
 -- Maximum number of section buffer positions for dasm_put().
@@ -246,7 +247,7 @@ local map_cond = {
 
 local parse_reg_type
 
-local function parse_reg(expr)
+local function parse_reg(expr, shift)
   if not expr then werror("expected register name") end
   local tname, ovreg = match(expr, "^([%w_]+):(@?%l%d+)$")
   local tp = map_type[tname or expr]
@@ -266,18 +267,29 @@ local function parse_reg(expr)
       elseif parse_reg_type ~= rt then
 	werror("register size mismatch")
       end
-      return r, tp
+      return shl(r, shift or 0), tp
     end
   end
+  -- Allow Rx(...) for dynamic register names
+  local vrt, vreg = match(expr, "^R([xwqdshb])(%b())$")
+  if vreg then
+    if not parse_reg_type then
+      parse_reg_type = vrt
+    elseif parse_reg_type ~= vrt then
+      werror("register size mismatch")
+    end
+    if shift then waction("VREG", shift, vreg) end
+    return 0
+  end
   werror("bad register name `"..expr.."'")
 end
 
 local function parse_reg_base(expr)
   if expr == "sp" then return 0x3e0 end
-  local base, tp = parse_reg(expr)
+  local base, tp = parse_reg(expr, 5)
   if parse_reg_type ~= "x" then werror("bad register type") end
   parse_reg_type = false
-  return shl(base, 5), tp
+  return base, tp
 end
 
 local parse_ctx = {}
@@ -403,7 +415,7 @@ local function parse_imm_load(imm, scale)
     end
     werror("out of range immediate `"..imm.."'")
   else
-    waction("IMML", 0, imm)
+    waction("IMML", scale, imm)
     return 0
   end
 end
@@ -470,7 +482,7 @@ local function parse_load(params, nparams, n, op)
       if reg and tailr ~= "" then
 	local base, tp = parse_reg_base(reg)
 	if tp then
-	  waction("IMML", 0, format(tp.ctypefmt, tailr))
+	  waction("IMML", shr(op, 30), format(tp.ctypefmt, tailr))
 	  return op + base
 	end
       end
@@ -494,7 +506,7 @@ local function parse_load(params, nparams, n, op)
 	op = op + parse_imm_load(imm, scale)
       else
 	local p2b, p3b, p3s = match(p2a, "^,%s*([^,%s]*)%s*,?%s*(%S*)%s*(.*)$")
-	op = op + shl(parse_reg(p2b), 16) + 0x00200800
+	op = op + parse_reg(p2b, 16) + 0x00200800
 	if parse_reg_type ~= "x" and parse_reg_type ~= "w" then
 	  werror("bad index register type")
 	end
@@ -891,15 +903,15 @@ local function parse_template(params, template, nparams, pos)
   for p in gmatch(sub(template, 9), ".") do
     local q = params[n]
     if p == "D" then
-      op = op + parse_reg(q); n = n + 1
+      op = op + parse_reg(q, 0); n = n + 1
     elseif p == "N" then
-      op = op + shl(parse_reg(q), 5); n = n + 1
+      op = op + parse_reg(q, 5); n = n + 1
     elseif p == "M" then
-      op = op + shl(parse_reg(q), 16); n = n + 1
+      op = op + parse_reg(q, 16); n = n + 1
     elseif p == "A" then
-      op = op + shl(parse_reg(q), 10); n = n + 1
+      op = op + parse_reg(q, 10); n = n + 1
     elseif p == "m" then
-      op = op + shl(parse_reg(params[n-1]), 16)
+      op = op + parse_reg(params[n-1], 16)
 
     elseif p == "p" then
       if q == "sp" then params[n] = "@x31" end

ext/opcache/jit/zend_jit.c

@@ -39,7 +39,14 @@
 #include "Optimizer/zend_call_graph.h"
 #include "Optimizer/zend_dump.h"
 
+#if defined(__x86_64__) || defined(i386)
 #include "jit/zend_jit_x86.h"
+#elif defined (__aarch64__)
+#include "jit/zend_jit_arm64.h"
+#else
+#error "JIT not supported on this platform"
+#endif
+
 #include "jit/zend_jit_internal.h"
 
 #ifdef ZTS
@@ -204,9 +211,18 @@ static bool zend_long_is_power_of_two(zend_long x)
 #define OP2_RANGE()      OP_RANGE(ssa_op, op2)
 #define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1)
 
+#if defined(__x86_64__) || defined(i386)
 #include "dynasm/dasm_x86.h"
+#elif defined(__aarch64__)
+#include "dynasm/dasm_arm64.h"
+#endif
+
 #include "jit/zend_jit_helpers.c"
+#if defined(__x86_64__) || defined(i386)
 #include "jit/zend_jit_disasm_x86.c"
+#elif defined(__aarch64__)
+#include "jit/zend_jit_disasm_arm64.c"
+#endif
 #ifndef _WIN32
 #include "jit/zend_jit_gdb.c"
 #include "jit/zend_jit_perf_dump.c"
@@ -216,7 +232,11 @@ static bool zend_long_is_power_of_two(zend_long x)
 #endif
 #include "jit/zend_jit_vtune.c"
 
+#if defined(__x86_64__) || defined(i386)
 #include "jit/zend_jit_x86.c"
+#elif defined(__aarch64__)
+#include "jit/zend_jit_arm64.c"
+#endif
 
 #if _WIN32
 # include <Windows.h>
@@ -298,15 +318,32 @@ static void handle_dasm_error(int ret) {
 		case DASM_S_RANGE_PC:
 			fprintf(stderr, "DASM_S_RANGE_PC %d\n", ret & 0xffffffu);
 			break;
+#ifdef DASM_S_RANGE_VREG
 		case DASM_S_RANGE_VREG:
 			fprintf(stderr, "DASM_S_RANGE_VREG\n");
 			break;
+#endif
+#ifdef DASM_S_UNDEF_L
 		case DASM_S_UNDEF_L:
 			fprintf(stderr, "DASM_S_UNDEF_L\n");
 			break;
+#endif
+#ifdef DASM_S_UNDEF_LG
+		case DASM_S_UNDEF_LG:
+			fprintf(stderr, "DASM_S_UNDEF_LG\n");
+			break;
+#endif
+#ifdef DASM_S_RANGE_REL
+		case DASM_S_RANGE_REL:
+			fprintf(stderr, "DASM_S_RANGE_REL\n");
+			break;
+#endif
 		case DASM_S_UNDEF_PC:
 			fprintf(stderr, "DASM_S_UNDEF_PC\n");
 			break;
+		default:
+			fprintf(stderr, "DASM_S_%0x\n", ret & 0xff000000u);
+			break;
 	}
 	ZEND_UNREACHABLE();
 }
@@ -391,6 +428,9 @@ static void *dasm_link_and_encode(dasm_State             **dasm_state,
 	entry = *dasm_ptr;
 	*dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT));
 
+	/* flush the hardware I-cache */
+	JIT_CACHE_FLUSH(entry, entry + size);
+
 	if (trace_num) {
 		zend_jit_trace_add_code(entry, size);
 	}