[JIT] Basic expression support for floating point operators

This doesn't work as of yet because we don't allocate floating point
registers yet. But we have to start somewhere.

src/jit/expr.c

@@ -114,11 +114,21 @@ static MVMint32 MVM_jit_expr_add_load(MVMThreadContext *tc, MVMJitExprTree *tree
     return MVM_jit_expr_apply_template_adhoc(tc, tree, "ns..", MVM_JIT_LOAD, 1, addr, MVM_JIT_REG_SZ);
 }
 
+static MVMint32 MVM_jit_expr_add_load_num(MVMThreadContext *tc, MVMJitExprTree *tree,
+                                      MVMint32 addr) {
+    return MVM_jit_expr_apply_template_adhoc(tc, tree, "ns..", MVM_JIT_LOAD_NUM, 1, addr, MVM_JIT_REG_SZ);
+}
+
 static MVMint32 MVM_jit_expr_add_store(MVMThreadContext *tc, MVMJitExprTree *tree,
                                        MVMint32 addr, MVMint32 val, MVMint32 sz) {
     return MVM_jit_expr_apply_template_adhoc(tc, tree, "ns...", MVM_JIT_STORE, 1, addr, val, sz);
 }
 
+static MVMint32 MVM_jit_expr_add_store_num(MVMThreadContext *tc, MVMJitExprTree *tree,
+                                       MVMint32 addr, MVMint32 val, MVMint32 sz) {
+    return MVM_jit_expr_apply_template_adhoc(tc, tree, "ns...", MVM_JIT_STORE_NUM, 1, addr, val, sz);
+}
+
 static MVMint32 MVM_jit_expr_add_cast(MVMThreadContext *tc, MVMJitExprTree *tree,
                                       MVMint32 cast_mode, MVMint32 node, MVMint32 to_size, MVMint32 from_size) {
     return MVM_jit_expr_apply_template_adhoc(tc, tree, "ns....", cast_mode, 1, node, to_size, from_size);
@@ -176,7 +186,6 @@ static MVMint32 MVM_jit_expr_add_const_ptr(MVMThreadContext *tc, MVMJitExprTree
         tree->constants[t].p = const_ptr;
         return t;
     }
-
 }
 
 static MVMint32 MVM_jit_expr_add_const(MVMThreadContext *tc, MVMJitExprTree *tree,
@@ -206,12 +215,12 @@ static MVMint32 MVM_jit_expr_add_const(MVMThreadContext *tc, MVMJitExprTree *tre
         size = MVM_JIT_INT_SZ;
         break;
     case MVM_operand_num32:
-        /* possible endianess issue here */
-        constant = opr.lit_i32;
+        operator = MVM_JIT_CONST_NUM;
+        constant = MVM_jit_expr_add_const_n64(tc, tree, (MVMnum64)opr.lit_n32);
         size = sizeof(MVMnum32);
         break;
     case MVM_operand_num64:
-        operator = MVM_JIT_CONST_LARGE;
+        operator = MVM_JIT_CONST_NUM;
         constant = MVM_jit_expr_add_const_n64(tc, tree, opr.lit_n64);
         size = MVM_JIT_NUM_SZ;
         break;
@@ -252,6 +261,20 @@ static MVMint32 bindlex_needs_write_barrier(MVMThreadContext *tc, MVMJitGraph *j
     return lexical_type == MVM_reg_obj || lexical_type == MVM_reg_str;
 }
 
+static MVMint32 store_value(MVMThreadContext *tc, MVMJitExprTree *tree,
+                            MVMint32 addr, MVMint32 val, MVMint32 sz) {
+    if (MVM_JIT_EXPR_IS_NUM(tree, val)) {
+        return MVM_jit_expr_add_store_num(tc, tree, addr, val, sz);
+    }
+    return MVM_jit_expr_add_store(tc, tree, addr, val, sz);
+}
+
+static MVMint32 load_value(MVMThreadContext *tc, MVMJitExprTree *tree, MVMint32 addr, MVMuint8 opr_type) {
+    if (opr_type == MVM_operand_num32 || opr_type == MVM_operand_num64) {
+        return MVM_jit_expr_add_load_num(tc, tree, addr);
+    }
+    return MVM_jit_expr_add_load(tc, tree, addr);
+}
 
 static MVMint32 ins_has_single_input_output_operand(MVMSpeshIns *ins) {
     switch (ins->info->opcode) {
@@ -266,19 +289,23 @@ static MVMint32 ins_has_single_input_output_operand(MVMSpeshIns *ins) {
     return 0;
 }
 
-void MVM_jit_expr_load_operands(MVMThreadContext *tc, MVMJitExprTree *tree, MVMSpeshIns *ins,
+
+
+void MVM_jit_expr_load_operands(MVMThreadContext *tc, MVMJitExprTree *tree,
+                                MVMSpeshGraph *sg, MVMSpeshIns *ins,
                                 struct ValueDefinition *values, MVMint32 *operands) {
     MVMint32 i;
     for (i = 0; i < ins->info->num_operands; i++) {
         MVMSpeshOperand opr = ins->operands[i];
-        MVMint8    opr_kind = ins->info->operands[i];
+        MVMuint8   opr_kind = ins->info->operands[i];
+        MVMuint8   opr_type = MVM_spesh_get_opr_type(tc, sg, ins, i);
         switch(opr_kind & MVM_operand_rw_mask) {
         case MVM_operand_read_reg:
             if (values[opr.reg.orig].node >= 0) {
                 operands[i] = values[opr.reg.orig].node;
             } else {
                 MVMint32 addr = MVM_jit_expr_add_regaddr(tc, tree, opr.reg.orig);
-                operands[i] = MVM_jit_expr_add_load(tc, tree, addr);
+                operands[i] = load_value(tc, tree, addr, opr_type);
                 values[opr.reg.orig].node = operands[i];
                 values[opr.reg.orig].addr = addr;
                 values[opr.reg.orig].root = -1; /* load is not part of a root */
@@ -294,7 +321,7 @@ void MVM_jit_expr_load_operands(MVMThreadContext *tc, MVMJitExprTree *tree, MVMS
         case MVM_operand_read_lex:
         {
             MVMint32 addr = MVM_jit_expr_add_lexaddr(tc, tree, opr.lex.outers, opr.lex.idx);
-            operands[i] = MVM_jit_expr_add_load(tc, tree, addr);
+            operands[i] = load_value(tc, tree, addr, opr_type);
             break;
         }
         case MVM_operand_write_lex:
@@ -303,9 +330,7 @@ void MVM_jit_expr_load_operands(MVMThreadContext *tc, MVMJitExprTree *tree, MVMS
         default:
             continue;
         }
-        if (operands[i] >= tree->nodes_num || operands[i] < 0) {
-            MVM_oops(tc, "JIT: something is wrong with operand loading");
-        }
+        assert(operands[i] <= MVM_VECTOR_ELEMS(tree->nodes) && operands[i] > 0);
     }
 
     /* A HACK.
@@ -420,6 +445,7 @@ static void analyze_node(MVMThreadContext *tc, MVMJitTreeTraverser *traverser,
     switch (tree->nodes[node]) {
     case MVM_JIT_CONST:
     case MVM_JIT_CONST_LARGE:
+    case MVM_JIT_CONST_NUM:
         /* node size is given */
         node_size        = args[1];
         break;
@@ -430,6 +456,7 @@ static void analyze_node(MVMThreadContext *tc, MVMJitTreeTraverser *traverser,
         node_size = MVM_JIT_EXPR_INFO(tree, links[0])->size;
         break;
     case MVM_JIT_LOAD:
+    case MVM_JIT_LOAD_NUM:
         node_size = args[0];
         break;
     case MVM_JIT_SCAST:
@@ -561,10 +588,7 @@ static void active_values_flush(MVMThreadContext *tc, MVMJitExprTree *tree,
     MVMint32 i;
     for (i = 0; i < num_values; i++) {
         if (values[i].root >= 0) {
-            tree->roots[values[i].root] = MVM_jit_expr_add_store(
-                tc, tree, values[i].addr,
-                values[i].node, MVM_JIT_REG_SZ
-            );
+            tree->roots[values[i].root] = store_value(tc, tree, values[i].addr, values[i].node, MVM_JIT_REG_SZ);
         }
         if (values[i].node >= 0) {
             memset(values + i, -1, sizeof(struct ValueDefinition));
@@ -573,7 +597,7 @@ static void active_values_flush(MVMThreadContext *tc, MVMJitExprTree *tree,
 }
 
 static MVMint32 tree_is_empty(MVMThreadContext *tc, MVMJitExprTree *tree) {
-    return tree->nodes_num == 0;
+    return MVM_VECTOR_ELEMS(tree->nodes) == 0;
 }
 
 MVMJitExprTree * MVM_jit_expr_tree_build(MVMThreadContext *tc, MVMJitGraph *jg, MVMSpeshIterator *iter) {
@@ -620,7 +644,7 @@ MVMJitExprTree * MVM_jit_expr_tree_build(MVMThreadContext *tc, MVMJitGraph *jg,
         MVMuint16 opcode = ins->info->opcode;
         MVMSpeshAnn *ann;
         const MVMJitExprTemplate *template;
-        MVMint32 before_label = -1, after_label = -1, store_directly = 0, root = 0;
+        MVMint32 before_label = -1, after_label = -1, root = 0;
 
         struct ValueDefinition *defined_value = NULL;
 
@@ -721,26 +745,14 @@ MVMJitExprTree * MVM_jit_expr_tree_build(MVMThreadContext *tc, MVMJitGraph *jg,
             MVM_spesh_graph_add_comment(tc, jg->sg, iter->ins, "start of exprjit tree");
         }
 
-        MVM_jit_expr_load_operands(tc, tree, ins, values, operands);
+        MVM_jit_expr_load_operands(tc, tree, sg, ins, values, operands);
         root = MVM_jit_expr_apply_template(tc, tree, template, operands);
 
         /* mark operand types */
         for (i = 0; i < ins->info->num_operands; i++) {
-            MVMuint8 opr_kind = ins->info->operands[i];
-            MVMuint8 opr_type = opr_kind & MVM_operand_type_mask;
             MVMSpeshOperand opr = ins->operands[i];
-            if (opr_type == MVM_operand_type_var) {
-                switch (opr_kind & MVM_operand_rw_mask) {
-                case MVM_operand_read_reg:
-                case MVM_operand_write_reg:
-                    opr_type = MVM_spesh_get_reg_type(tc, sg, opr.reg.orig) << 3; /* shift up 3 to match operand type */
-                    break;
-                case MVM_operand_read_lex:
-                case MVM_operand_write_lex:
-                    opr_type = MVM_spesh_get_lex_type(tc, sg, opr.lex.outers, opr.lex.idx) << 3;
-                    break;
-                }
-            }
+            MVMuint8 opr_kind = ins->info->operands[i];
+            MVMuint8 opr_type = MVM_spesh_get_opr_type(tc, sg, ins, i);
             switch(opr_kind & MVM_operand_rw_mask) {
             case MVM_operand_read_reg:
             case MVM_operand_read_lex:
@@ -771,20 +783,22 @@ MVMJitExprTree * MVM_jit_expr_tree_build(MVMThreadContext *tc, MVMJitGraph *jg,
                     BAIL(i != 0, "Write lex operand %d", i);
                     MVM_JIT_EXPR_INFO(tree, root)->type = opr_type >> 3;
                     /* insert the store to lexicals directly, do not record as value */
-                    root = MVM_jit_expr_add_store(tc, tree, operands[i], root, MVM_JIT_REG_SZ);
+                    root = store_value(tc, tree, operands[i], root, MVM_JIT_REG_SZ);
                 }
                 break;
             }
             assert(MVM_JIT_EXPR_INFO(tree, operands[i])->type >= 0);
         }
 
         if (ins->info->jittivity & (MVM_JIT_INFO_THROWISH | MVM_JIT_INFO_INVOKISH)) {
-            /* NB: we should make this a template-level flag; should be possible
-             * to replace an invokish version with a non-invokish version (but
-             * perhaps best if that is opt-in so people don't accidentally
-             * forget to set it). */
+            /* TODO: make this a template level flag.
+             * Currently it is impossible to replace an invokish version with a
+             * non-invokish version, which should be possible in a specialization */
             active_values_flush(tc, tree, values, sg->num_locals);
-            store_directly = 1;
+            if (defined_value != NULL) {
+                root = store_value(tc, tree, defined_value->addr, root, MVM_JIT_REG_SZ);
+                defined_value = NULL;
+            }
         }
 
         /* Add root to tree to ensure source evaluation order, wrapped with
@@ -794,16 +808,6 @@ MVMJitExprTree * MVM_jit_expr_tree_build(MVMThreadContext *tc, MVMJitGraph *jg,
             MVM_VECTOR_PUSH(tree->roots, MVM_jit_expr_add_label(tc, tree, before_label));
         }
 
-        /* NB: GUARD only wraps void nodes. Currently, we replace any
-         * value-yielding node with it's STORE (and thereby make sure it is
-         * flushed directly) */
-        if (store_directly && defined_value != NULL) {
-            /* If we're wrapping this template and it defines a value, we
-             * had maybe better flush it directly */
-            root = MVM_jit_expr_add_store(tc, tree, defined_value->addr, root, MVM_JIT_REG_SZ);
-            defined_value = NULL;
-        }
-
         if (defined_value != NULL) {
             defined_value->root = tree->roots_num;
         }

src/jit/expr.h

@@ -102,6 +102,10 @@ MVM_STATIC_INLINE MVMuint8 MVM_JIT_EXPR_NCHILD(MVMJitExprTree *tree, MVMint32 no
     return MVM_JIT_EXPR_INFO(tree, node)->num_links;
 }
 
+MVM_STATIC_INLINE MVMuint8 MVM_JIT_EXPR_TYPE(MVMJitExprTree *tree, MVMint32 node) {
+    return MVM_JIT_EXPR_INFO(tree, node)->type;
+}
+
 MVM_STATIC_INLINE MVMint32 MVM_JIT_EXPR_FIRST_CHILD(MVMJitExprTree *tree, MVMint32 node) {
     return node + 2;
 }
@@ -115,3 +119,7 @@ MVM_STATIC_INLINE MVMint32 * MVM_JIT_EXPR_ARGS(MVMJitExprTree *tree, MVMint32 no
     return MVM_JIT_EXPR_LINKS(tree, node) + MVM_JIT_EXPR_NCHILD(tree, node);
 }
 
+MVM_STATIC_INLINE MVMint32 MVM_JIT_EXPR_IS_NUM(MVMJitExprTree *tree, MVMint32 node) {
+    MVMuint8 expr_type = MVM_JIT_EXPR_TYPE(tree,node);
+    return expr_type == MVM_reg_num32 || expr_type == MVM_reg_num64;
+}

src/jit/expr_ops.h

@@ -16,7 +16,9 @@
     _(COPY, 1, 0),   \
     /* memory access */ \
     _(LOAD, 1, 1),   \
+    _(LOAD_NUM, 1, 1), \
     _(STORE, 2, 1), \
+    _(STORE_NUM, 2, 1), \
     _(ADDR, 1, 1),  \
     _(IDX, 2, 1),   \
     /* constant up to 4 bytes */ \
@@ -25,6 +27,8 @@
     _(CONST_PTR, 0, 1), \
     /* large constant (8 bytes) */ \
     _(CONST_LARGE, 0, 2), \
+    /* floating point constant */ \
+    _(CONST_NUM, 0, 2), \
     /* integer comparison */ \
     _(LT, 2, 0),     \
     _(LE, 2, 0),     \

src/jit/x64/tile_pattern.tile

@@ -1,5 +1,5 @@
 # -*-whitespace-*-
-# Terminals: reg, mem, flag, void, expr, lab
+# Types: reg, num, flag, void
 #
 
 #
@@ -16,10 +16,13 @@
 (define: (all flag) flag)
 (define: (any flag) flag)
 (define: (if flag reg) reg)
+(define: (if flag num) num)
 (define: (ifv flag void) void)
 (define: (when flag void) void)
 (define: (discard reg) void)
+(define: (discard num) void)
 (define: (do void reg) reg)
+(define: (do void num) num)
 (define: (dov void void) void)
 
 
@@ -31,7 +34,7 @@
 # ALIASING
 #
 (define: (copy reg) reg)
-
+(define: (copy num) num)
 
 #
 # MEMORY TRAFFIC
@@ -43,16 +46,17 @@
 
 (tile: const_large (const_large $value $size) reg 3)
 (tile: const_large (const_ptr $ptr) reg 3)
-
+(tile: const_num   (const_num $num $size) num 3)
 
 (tile: load_reg  (load reg $size) reg 5)
 (tile: load_addr (load (addr reg $ofs) $size) reg 5)
 (tile: load_idx  (load (idx reg reg $scale) $size) reg 5)
+(tile: load_num  (load reg $size) num 5)
 
 (tile: store (store reg reg $size) void 5)
 (tile: store_addr (store (addr reg $ofs) reg $size) void 5)
 (tile: store_idx  (store (idx reg reg $scl) reg $size) void 5)
-
+(tile: store_num  (store_num reg num $size) void 5)
 
 (tile: cast_signed             (scast reg $to_size $from_size) reg 2)
 (tile: cast_unsigned           (ucast reg $to_size $from_size) reg 2)

src/jit/x64/tiles.dasc

@@ -39,6 +39,21 @@ MVM_JIT_TILE_DECL(const_large) {
     | mov64 Rq(out), val;
 }
 
+MVM_JIT_TILE_DECL(const_num) {
+    MVMint8 out = tile->values[0];
+    MVMnum64 val = tree->constants[tile->args[0]].n;
+    unsigned char bytes[sizeof(val)];
+    int i;
+    memcpy(bytes, &val, sizeof(val));
+    |.data;
+    |5:
+    for (i = 0; i < sizeof(bytes); i++) {
+        |.byte bytes[i];
+    }
+    |.code;
+    | movsd xmm(out), qword [<5];
+}
+
 MVM_JIT_TILE_DECL(load_reg) {
     MVMint8 out  = tile->values[0];
     MVMint8 base = tile->values[1];
@@ -111,6 +126,11 @@ MVM_JIT_TILE_DECL(load_idx) {
     }
 }
 
+MVM_JIT_TILE_DECL(load_num) {
+    MVMint8 out = tile->values[0];
+    MVMint8 addr = tile->values[1];
+    | movsd xmm(out), qword [Rq(addr)];
+}
 
 MVM_JIT_TILE_DECL(store) {
     MVMint8 base  = tile->values[1];
@@ -183,6 +203,11 @@ MVM_JIT_TILE_DECL(store_idx) {
     }
 }
 
+MVM_JIT_TILE_DECL(store_num) {
+    MVMint8 addr = tile->values[0];
+    MVMint8 val =  tile->values[1];
+    | movsd qword [Rq(addr)], xmm(val);
+}
 
 MVM_JIT_TILE_DECL(cast_signed) {
     MVMint32 to_size   = tile->args[0];

src/spesh/lookup.c

@@ -43,3 +43,22 @@ MVMint8 MVM_spesh_get_lex_type(MVMThreadContext *tc, MVMSpeshGraph *sg, MVMuint1
         return sf->body.lexical_types[idx];
     }
 }
+
+MVMuint8 MVM_spesh_get_opr_type(MVMThreadContext *tc, MVMSpeshGraph *sg, MVMSpeshIns *ins, MVMint32 i) {
+    MVMSpeshOperand opr = ins->operands[i];
+    MVMuint8 opr_kind = ins->info->operands[i];
+    MVMuint8 opr_type = opr_kind & MVM_operand_type_mask;
+    if (opr_type == MVM_operand_type_var) {
+        switch (opr_kind & MVM_operand_rw_mask) {
+        case MVM_operand_read_reg:
+        case MVM_operand_write_reg:
+            opr_type = MVM_spesh_get_reg_type(tc, sg, opr.reg.orig) << 3; /* shift up 3 to match operand type */
+            break;
+        case MVM_operand_read_lex:
+        case MVM_operand_write_lex:
+            opr_type = MVM_spesh_get_lex_type(tc, sg, opr.lex.outers, opr.lex.idx) << 3;
+            break;
+        }
+    }
+    return opr_type;
+}

src/spesh/lookup.h

@@ -3,3 +3,4 @@ MVMObject * MVM_spesh_try_find_method(MVMThreadContext *tc, MVMObject *obj, MVMS
 MVMint64 MVM_spesh_try_can_method(MVMThreadContext *tc, MVMObject *obj, MVMString *name);
 MVMint8 MVM_spesh_get_reg_type(MVMThreadContext *tc, MVMSpeshGraph *sg, MVMuint16 reg);
 MVMint8 MVM_spesh_get_lex_type(MVMThreadContext *tc, MVMSpeshGraph *sg, MVMuint16 outers, MVMuint16 idx);
+MVMuint8 MVM_spesh_get_opr_type(MVMThreadContext *tc, MVMSpeshGraph *sg, MVMSpeshIns *ins, MVMint32 i);