Merge 5a34daa into 3fb3da0

tests/factorial.data

@@ -0,0 +1,13 @@
+-- asm
+mov %r1, 1
+mov %r2, 20
+call local exponential
+exit
+exponential:
+mul %r1, %r2
+sub %r2, 1
+jne %r2, 0, exponential
+mov %r0, %r1
+exit
+-- result
+0x21C3677C82B40000

vm/ubpf_jit_arm64.c

@@ -1189,6 +1189,10 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg)
                 *errmsg = ubpf_error("Too many LEA calculations.");
                 break;
             }
+            case TooManyLocalCalls: {
+                *errmsg = ubpf_error("Too many local calls.");
+                break;
+            }
             case UnexpectedInstruction: {
                 // errmsg set at time the error was detected because the message requires
                 // information about the unexpected instruction.

vm/ubpf_jit_support.c

@@ -34,9 +34,11 @@ initialize_jit_state_result(struct jit_state *state, struct ubpf_jit_result *com
     state->jumps = calloc(UBPF_MAX_INSTS, sizeof(state->jumps[0]));
     state->loads = calloc(UBPF_MAX_INSTS, sizeof(state->loads[0]));
     state->leas = calloc(UBPF_MAX_INSTS, sizeof(state->leas[0]));
+    state->local_calls = calloc(UBPF_MAX_INSTS, sizeof(state->local_calls[0]));
     state->num_jumps = 0;
     state->num_loads = 0;
     state->num_leas = 0;
+    state->num_local_calls = 0;
     state->jit_status = NoError;
 
     if (!state->pc_locs || !state->jumps || !state->loads || !state->leas) {
@@ -59,6 +61,8 @@ release_jit_state_result(struct jit_state *state, struct ubpf_jit_result *compil
     state->loads = NULL;
     free(state->leas);
     state->leas = NULL;
+    free(state->local_calls);
+    state->local_calls = NULL;
 }
 
 void
@@ -97,4 +101,4 @@ void
 emit_jump_target(struct jit_state* state, uint32_t jump_src)
 {
     fixup_jump_target(state->jumps, state->num_jumps, jump_src, state->offset);
-}
+}

vm/ubpf_jit_support.h

@@ -33,6 +33,7 @@ enum JitProgress {
     TooManyJumps,
     TooManyLoads,
     TooManyLeas,
+    TooManyLocalCalls,
     NotEnoughSpace,
     UnexpectedInstruction,
     UnknownInstruction
@@ -88,9 +89,11 @@ struct jit_state
     struct patchable_relative* jumps;
     struct patchable_relative* loads;
     struct patchable_relative* leas;
+    struct patchable_relative* local_calls;
     int num_jumps;
     int num_loads;
     int num_leas;
+    int num_local_calls;
     uint32_t stack_size;
 };
 

vm/ubpf_jit_x86_64.c

@@ -121,7 +121,7 @@ emit_local_call(struct jit_state* state, uint32_t target_pc)
     emit_alu64_imm32(state, 0x81, 5, RSP, 4 * sizeof(uint64_t));
 #endif
     emit1(state, 0xe8); // e8 is the opcode for a CALL
-    emit_jump_address_reloc(state, target_pc);
+    emit_local_call_address_reloc(state, target_pc);
 #if defined(_WIN32)
     /* Deallocate home register space - 4 registers */
     emit_alu64_imm32(state, 0x81, 0, RSP, 4 * sizeof(uint64_t));
@@ -301,7 +301,6 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg)
         }
 
         struct ebpf_inst inst = ubpf_fetch_instruction(vm, i);
-        state->pc_locs[i] = state->offset;
 
         int dst = map_register(inst.dst);
         int src = map_register(inst.src);
@@ -314,6 +313,8 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg)
             emit_alu64_imm32(state, 0x81, 5, RSP, 8);
         }
 
+        state->pc_locs[i] = state->offset;
+
         switch (inst.opcode) {
         case EBPF_OP_ADD_IMM:
             emit_alu32_imm32(state, 0x81, 0, dst, inst.imm);
@@ -739,6 +740,10 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg)
             *errmsg = ubpf_error("Too many LEA calculations");
             break;
         }
+        case TooManyLocalCalls: {
+            *errmsg = ubpf_error("Too many local calls");
+            break;
+        }
         case UnexpectedInstruction: {
             // errmsg set at time the error was detected because the message requires
             // information about the unexpected instruction.
@@ -938,6 +943,24 @@ resolve_patchable_relatives(struct jit_state* state)
         memcpy(offset_ptr, &rel, sizeof(uint32_t));
     }
 
+    for (i = 0; i < state->num_local_calls; i++) {
+        struct patchable_relative local_call = state->local_calls[i];
+
+        int target_loc;
+        assert(local_call.target_offset == 0);
+        assert(local_call.target_pc != TARGET_PC_EXIT);
+        assert(local_call.target_pc != TARGET_PC_RETPOLINE);
+
+        target_loc = state->pc_locs[local_call.target_pc];
+
+        /* Assumes call offset is at end of instruction */
+        uint32_t rel = target_loc - (local_call.offset_loc + sizeof(uint32_t));
+        rel -= 7; // For the "sub rsp, 8" instruction that is inserted before the call to align the stack.
+
+        uint8_t* offset_ptr = &state->buf[local_call.offset_loc];
+        memcpy(offset_ptr, &rel, sizeof(uint32_t));
+    }
+
     for (i = 0; i < state->num_loads; i++) {
         struct patchable_relative load = state->loads[i];
 

vm/ubpf_jit_x86_64.h

@@ -121,6 +121,19 @@ emit_jump_address_reloc(struct jit_state* state, int32_t target_pc)
     return target_address_offset;
 }
 
+static uint32_t
+emit_local_call_address_reloc(struct jit_state* state, int32_t target_pc)
+{
+    if (state->num_local_calls == UBPF_MAX_INSTS) {
+        state->jit_status = TooManyLocalCalls;
+        return 0;
+    }
+    uint32_t target_address_offset = state->offset;
+    emit_patchable_relative(state->offset, target_pc, 0, state->local_calls, state->num_local_calls++);
+    emit_4byte_offset_placeholder(state);
+    return target_address_offset;
+}
+
 static inline void
 emit_modrm(struct jit_state* state, int mod, int r, int m)
 {