Skip to content
Permalink
Branch: master
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
10096 lines (8864 sloc) 311 KB
/**
* \file
* llvm "Backend" for the mono JIT
*
* Copyright 2009-2011 Novell Inc (http://www.novell.com)
* Copyright 2011 Xamarin Inc (http://www.xamarin.com)
* Licensed under the MIT license. See LICENSE file in the project root for full license information.
*/
#include "config.h"
#include <mono/metadata/debug-helpers.h>
#include <mono/metadata/debug-internals.h>
#include <mono/metadata/mempool-internals.h>
#include <mono/metadata/environment.h>
#include <mono/metadata/object-internals.h>
#include <mono/metadata/abi-details.h>
#include <mono/utils/mono-tls.h>
#include <mono/utils/mono-dl.h>
#include <mono/utils/mono-time.h>
#include <mono/utils/freebsd-dwarf.h>
#ifndef __STDC_LIMIT_MACROS
#define __STDC_LIMIT_MACROS
#endif
#ifndef __STDC_CONSTANT_MACROS
#define __STDC_CONSTANT_MACROS
#endif
#include "llvm-c/BitWriter.h"
#include "llvm-c/Analysis.h"
#include "mini-llvm-cpp.h"
#include "llvm-jit.h"
#include "aot-compiler.h"
#include "mini-llvm.h"
#include "mini-runtime.h"
#include <mono/utils/mono-math.h>
#ifndef DISABLE_JIT
#if defined(__MINGW32__) || defined(_MSC_VER)
#include <stddef.h>
extern void *memset(void *, int, size_t);
void bzero (void *to, size_t count) { memset (to, 0, count); }
#endif
#if LLVM_API_VERSION < 4
#error "The version of the mono llvm repository is too old."
#endif
/*
* Information associated by mono with LLVM modules.
*/
typedef struct {
LLVMModuleRef lmodule;
LLVMValueRef throw_icall, rethrow, match_exc, throw_corlib_exception, resume_eh;
GHashTable *llvm_types;
LLVMValueRef got_var;
const char *got_symbol;
const char *get_method_symbol;
const char *get_unbox_tramp_symbol;
GHashTable *plt_entries;
GHashTable *plt_entries_ji;
GHashTable *method_to_lmethod;
GHashTable *direct_callables;
char **bb_names;
int bb_names_len;
GPtrArray *used;
LLVMTypeRef ptr_type;
GPtrArray *subprogram_mds;
MonoEERef *mono_ee;
LLVMExecutionEngineRef ee;
gboolean external_symbols;
gboolean emit_dwarf;
int max_got_offset;
LLVMValueRef personality;
/* For AOT */
MonoAssembly *assembly;
char *global_prefix;
MonoAotFileInfo aot_info;
const char *jit_got_symbol;
const char *eh_frame_symbol;
LLVMValueRef get_method, get_unbox_tramp;
LLVMValueRef init_method, init_method_gshared_mrgctx, init_method_gshared_this, init_method_gshared_vtable;
LLVMValueRef code_start, code_end;
LLVMValueRef inited_var;
LLVMValueRef unbox_tramp_indexes;
LLVMValueRef unbox_trampolines;
int max_inited_idx, max_method_idx;
gboolean has_jitted_code;
gboolean static_link;
gboolean llvm_only;
gboolean llvm_disable_self_init;
gboolean interp;
GHashTable *idx_to_lmethod;
GHashTable *idx_to_unbox_tramp;
GPtrArray *callsite_list;
LLVMContextRef context;
LLVMValueRef sentinel_exception;
void *di_builder, *cu;
GHashTable *objc_selector_to_var;
GPtrArray *cfgs;
int unbox_tramp_num, unbox_tramp_elemsize;
} MonoLLVMModule;
/*
* Information associated by the backend with mono basic blocks.
*/
typedef struct {
LLVMBasicBlockRef bblock, end_bblock;
LLVMValueRef finally_ind;
gboolean added, invoke_target;
/*
* If this bblock is the start of a finally clause, this is a list of bblocks it
* needs to branch to in ENDFINALLY.
*/
GSList *call_handler_return_bbs;
/*
* If this bblock is the start of a finally clause, this is the bblock that
* CALL_HANDLER needs to branch to.
*/
LLVMBasicBlockRef call_handler_target_bb;
/* The list of switch statements generated by ENDFINALLY instructions */
GSList *endfinally_switch_ins_list;
GSList *phi_nodes;
} BBInfo;
/*
* Structure containing emit state
*/
typedef struct {
MonoMemPool *mempool;
/* Maps method names to the corresponding LLVMValueRef */
GHashTable *emitted_method_decls;
MonoCompile *cfg;
LLVMValueRef lmethod;
MonoLLVMModule *module;
LLVMModuleRef lmodule;
BBInfo *bblocks;
int sindex, default_index, ex_index;
LLVMBuilderRef builder;
LLVMValueRef *values, *addresses;
MonoType **vreg_cli_types;
LLVMCallInfo *linfo;
MonoMethodSignature *sig;
GSList *builders;
GHashTable *region_to_handler;
GHashTable *clause_to_handler;
LLVMBuilderRef alloca_builder;
LLVMValueRef last_alloca;
LLVMValueRef rgctx_arg;
LLVMValueRef this_arg;
LLVMTypeRef *vreg_types;
gboolean *is_vphi;
LLVMTypeRef method_type;
LLVMBasicBlockRef init_bb, inited_bb;
gboolean *is_dead;
gboolean *unreachable;
gboolean llvm_only;
gboolean has_got_access;
gboolean emit_dummy_arg;
int this_arg_pindex, rgctx_arg_pindex;
LLVMValueRef imt_rgctx_loc;
GHashTable *llvm_types;
LLVMValueRef dbg_md;
MonoDebugMethodInfo *minfo;
/* For every clause, the clauses it is nested in */
GSList **nested_in;
LLVMValueRef ex_var;
GHashTable *exc_meta;
GPtrArray *callsite_list;
GPtrArray *phi_values;
GPtrArray *bblock_list;
char *method_name;
GHashTable *jit_callees;
LLVMValueRef long_bb_break_var;
} EmitContext;
typedef struct {
MonoBasicBlock *bb;
MonoInst *phi;
MonoBasicBlock *in_bb;
int sreg;
} PhiNode;
/*
* Instruction metadata
* This is the same as ins_info, but LREG != IREG.
*/
#ifdef MINI_OP
#undef MINI_OP
#endif
#ifdef MINI_OP3
#undef MINI_OP3
#endif
#define MINI_OP(a,b,dest,src1,src2) dest, src1, src2, ' ',
#define MINI_OP3(a,b,dest,src1,src2,src3) dest, src1, src2, src3,
#define NONE ' '
#define IREG 'i'
#define FREG 'f'
#define VREG 'v'
#define XREG 'x'
#define LREG 'l'
/* keep in sync with the enum in mini.h */
const char
mini_llvm_ins_info[] = {
#include "mini-ops.h"
};
#undef MINI_OP
#undef MINI_OP3
#if TARGET_SIZEOF_VOID_P == 4
#define GET_LONG_IMM(ins) ((ins)->inst_l)
#else
#define GET_LONG_IMM(ins) ((ins)->inst_imm)
#endif
#define LLVM_INS_INFO(opcode) (&mini_llvm_ins_info [((opcode) - OP_START - 1) * 4])
#if 0
#define TRACE_FAILURE(msg) do { printf ("%s\n", msg); } while (0)
#else
#define TRACE_FAILURE(msg)
#endif
#ifdef TARGET_X86
#define IS_TARGET_X86 1
#else
#define IS_TARGET_X86 0
#endif
#ifdef TARGET_AMD64
#define IS_TARGET_AMD64 1
#else
#define IS_TARGET_AMD64 0
#endif
#define ctx_ok(ctx) (!(ctx)->cfg->disable_llvm)
static LLVMIntPredicate cond_to_llvm_cond [] = {
LLVMIntEQ,
LLVMIntNE,
LLVMIntSLE,
LLVMIntSGE,
LLVMIntSLT,
LLVMIntSGT,
LLVMIntULE,
LLVMIntUGE,
LLVMIntULT,
LLVMIntUGT,
};
static LLVMRealPredicate fpcond_to_llvm_cond [] = {
LLVMRealOEQ,
LLVMRealUNE,
LLVMRealOLE,
LLVMRealOGE,
LLVMRealOLT,
LLVMRealOGT,
LLVMRealULE,
LLVMRealUGE,
LLVMRealULT,
LLVMRealUGT,
};
static MonoNativeTlsKey current_cfg_tls_id;
static MonoLLVMModule aot_module;
static GHashTable *intrins_id_to_name;
static GHashTable *intrins_name_to_id;
static void init_jit_module (MonoDomain *domain);
static void emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code);
static void emit_default_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder);
static LLVMValueRef emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name);
static void emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name);
static void emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp);
static LLVMValueRef get_intrinsic (EmitContext *ctx, const char *name);
static void decode_llvm_eh_info (EmitContext *ctx, gpointer eh_frame);
static inline void
set_failure (EmitContext *ctx, const char *message)
{
TRACE_FAILURE (reason);
ctx->cfg->exception_message = g_strdup (message);
ctx->cfg->disable_llvm = TRUE;
}
/*
* IntPtrType:
*
* The LLVM type with width == TARGET_SIZEOF_VOID_P
*/
static LLVMTypeRef
IntPtrType (void)
{
return TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type ();
}
static LLVMTypeRef
ObjRefType (void)
{
return TARGET_SIZEOF_VOID_P == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
}
static LLVMTypeRef
ThisType (void)
{
return TARGET_SIZEOF_VOID_P == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
}
/*
* get_vtype_size:
*
* Return the size of the LLVM representation of the vtype T.
*/
static guint32
get_vtype_size (MonoType *t)
{
int size;
size = mono_class_value_size (mono_class_from_mono_type_internal (t), NULL);
/* LLVMArgAsIArgs depends on this since it stores whole words */
while (size < 2 * TARGET_SIZEOF_VOID_P && mono_is_power_of_two (size) == -1)
size ++;
return size;
}
/*
* simd_class_to_llvm_type:
*
* Return the LLVM type corresponding to the Mono.SIMD class KLASS
*/
static LLVMTypeRef
simd_class_to_llvm_type (EmitContext *ctx, MonoClass *klass)
{
const char *klass_name = m_class_get_name (klass);
if (!strcmp (klass_name, "Vector2d")) {
return LLVMVectorType (LLVMDoubleType (), 2);
} else if (!strcmp (klass_name, "Vector2l")) {
return LLVMVectorType (LLVMInt64Type (), 2);
} else if (!strcmp (klass_name, "Vector2ul")) {
return LLVMVectorType (LLVMInt64Type (), 2);
} else if (!strcmp (klass_name, "Vector4i")) {
return LLVMVectorType (LLVMInt32Type (), 4);
} else if (!strcmp (klass_name, "Vector4ui")) {
return LLVMVectorType (LLVMInt32Type (), 4);
} else if (!strcmp (klass_name, "Vector4f")) {
return LLVMVectorType (LLVMFloatType (), 4);
} else if (!strcmp (klass_name, "Vector8s")) {
return LLVMVectorType (LLVMInt16Type (), 8);
} else if (!strcmp (klass_name, "Vector8us")) {
return LLVMVectorType (LLVMInt16Type (), 8);
} else if (!strcmp (klass_name, "Vector16sb")) {
return LLVMVectorType (LLVMInt8Type (), 16);
} else if (!strcmp (klass_name, "Vector16b")) {
return LLVMVectorType (LLVMInt8Type (), 16);
} else if (!strcmp (klass_name, "Vector2")) {
/* System.Numerics */
return LLVMVectorType (LLVMFloatType (), 4);
} else if (!strcmp (klass_name, "Vector3")) {
return LLVMVectorType (LLVMFloatType (), 4);
} else if (!strcmp (klass_name, "Vector4")) {
return LLVMVectorType (LLVMFloatType (), 4);
} else if (!strcmp (klass_name, "Vector`1")) {
MonoType *etype = mono_class_get_generic_class (klass)->context.class_inst->type_argv [0];
switch (etype->type) {
case MONO_TYPE_I1:
case MONO_TYPE_U1:
return LLVMVectorType (LLVMInt8Type (), 16);
case MONO_TYPE_I2:
case MONO_TYPE_U2:
return LLVMVectorType (LLVMInt16Type (), 8);
case MONO_TYPE_I4:
case MONO_TYPE_U4:
return LLVMVectorType (LLVMInt32Type (), 4);
case MONO_TYPE_I8:
case MONO_TYPE_U8:
return LLVMVectorType (LLVMInt64Type (), 2);
case MONO_TYPE_R4:
return LLVMVectorType (LLVMFloatType (), 4);
case MONO_TYPE_R8:
return LLVMVectorType (LLVMDoubleType (), 2);
default:
g_assert_not_reached ();
return NULL;
}
} else {
printf ("%s\n", klass_name);
NOT_IMPLEMENTED;
return NULL;
}
}
/* Return the 128 bit SIMD type corresponding to the mono type TYPE */
static inline G_GNUC_UNUSED LLVMTypeRef
type_to_simd_type (int type)
{
switch (type) {
case MONO_TYPE_I1:
return LLVMVectorType (LLVMInt8Type (), 16);
case MONO_TYPE_I2:
return LLVMVectorType (LLVMInt16Type (), 8);
case MONO_TYPE_I4:
return LLVMVectorType (LLVMInt32Type (), 4);
case MONO_TYPE_I8:
return LLVMVectorType (LLVMInt64Type (), 2);
case MONO_TYPE_R8:
return LLVMVectorType (LLVMDoubleType (), 2);
case MONO_TYPE_R4:
return LLVMVectorType (LLVMFloatType (), 4);
default:
g_assert_not_reached ();
return NULL;
}
}
static LLVMTypeRef
create_llvm_type_for_type (MonoLLVMModule *module, MonoClass *klass)
{
int i, size, nfields, esize;
LLVMTypeRef *eltypes;
char *name;
MonoType *t;
LLVMTypeRef ltype;
t = m_class_get_byval_arg (klass);
if (mini_type_is_hfa (t, &nfields, &esize)) {
/*
* This is needed on arm64 where HFAs are returned in
* registers.
*/
/* SIMD types have size 16 in mono_class_value_size () */
if (m_class_is_simd_type (klass))
nfields = 16/ esize;
size = nfields;
eltypes = g_new (LLVMTypeRef, size);
for (i = 0; i < size; ++i)
eltypes [i] = esize == 4 ? LLVMFloatType () : LLVMDoubleType ();
} else {
size = get_vtype_size (t);
eltypes = g_new (LLVMTypeRef, size);
for (i = 0; i < size; ++i)
eltypes [i] = LLVMInt8Type ();
}
name = mono_type_full_name (m_class_get_byval_arg (klass));
ltype = LLVMStructCreateNamed (module->context, name);
LLVMStructSetBody (ltype, eltypes, size, FALSE);
g_free (eltypes);
g_free (name);
return ltype;
}
/*
* type_to_llvm_type:
*
* Return the LLVM type corresponding to T.
*/
static LLVMTypeRef
type_to_llvm_type (EmitContext *ctx, MonoType *t)
{
if (t->byref)
return ThisType ();
t = mini_get_underlying_type (t);
switch (t->type) {
case MONO_TYPE_VOID:
return LLVMVoidType ();
case MONO_TYPE_I1:
return LLVMInt8Type ();
case MONO_TYPE_I2:
return LLVMInt16Type ();
case MONO_TYPE_I4:
return LLVMInt32Type ();
case MONO_TYPE_U1:
return LLVMInt8Type ();
case MONO_TYPE_U2:
return LLVMInt16Type ();
case MONO_TYPE_U4:
return LLVMInt32Type ();
case MONO_TYPE_I8:
case MONO_TYPE_U8:
return LLVMInt64Type ();
case MONO_TYPE_R4:
return LLVMFloatType ();
case MONO_TYPE_R8:
return LLVMDoubleType ();
case MONO_TYPE_I:
case MONO_TYPE_U:
return IntPtrType ();
case MONO_TYPE_OBJECT:
case MONO_TYPE_PTR:
return ObjRefType ();
case MONO_TYPE_VAR:
case MONO_TYPE_MVAR:
/* Because of generic sharing */
return ObjRefType ();
case MONO_TYPE_GENERICINST:
if (!mono_type_generic_inst_is_valuetype (t))
return ObjRefType ();
/* Fall through */
case MONO_TYPE_VALUETYPE:
case MONO_TYPE_TYPEDBYREF: {
MonoClass *klass;
LLVMTypeRef ltype;
klass = mono_class_from_mono_type_internal (t);
if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
return simd_class_to_llvm_type (ctx, klass);
if (m_class_is_enumtype (klass))
return type_to_llvm_type (ctx, mono_class_enum_basetype_internal (klass));
ltype = (LLVMTypeRef)g_hash_table_lookup (ctx->module->llvm_types, klass);
if (!ltype) {
ltype = create_llvm_type_for_type (ctx->module, klass);
g_hash_table_insert (ctx->module->llvm_types, klass, ltype);
}
return ltype;
}
default:
printf ("X: %d\n", t->type);
ctx->cfg->exception_message = g_strdup_printf ("type %s", mono_type_full_name (t));
ctx->cfg->disable_llvm = TRUE;
return NULL;
}
}
/*
* type_is_unsigned:
*
* Return whenever T is an unsigned int type.
*/
static gboolean
type_is_unsigned (EmitContext *ctx, MonoType *t)
{
t = mini_get_underlying_type (t);
if (t->byref)
return FALSE;
switch (t->type) {
case MONO_TYPE_U1:
case MONO_TYPE_U2:
case MONO_TYPE_CHAR:
case MONO_TYPE_U4:
case MONO_TYPE_U8:
return TRUE;
default:
return FALSE;
}
}
/*
* type_to_llvm_arg_type:
*
* Same as type_to_llvm_type, but treat i8/i16 as i32.
*/
static LLVMTypeRef
type_to_llvm_arg_type (EmitContext *ctx, MonoType *t)
{
LLVMTypeRef ptype = type_to_llvm_type (ctx, t);
if (ctx->cfg->llvm_only)
return ptype;
/*
* This works on all abis except arm64/ios which passes multiple
* arguments in one stack slot.
*/
#ifndef TARGET_ARM64
if (ptype == LLVMInt8Type () || ptype == LLVMInt16Type ()) {
/*
* LLVM generates code which only sets the lower bits, while JITted
* code expects all the bits to be set.
*/
ptype = LLVMInt32Type ();
}
#endif
return ptype;
}
/*
* llvm_type_to_stack_type:
*
* Return the LLVM type which needs to be used when a value of type TYPE is pushed
* on the IL stack.
*/
static G_GNUC_UNUSED LLVMTypeRef
llvm_type_to_stack_type (MonoCompile *cfg, LLVMTypeRef type)
{
if (type == NULL)
return NULL;
if (type == LLVMInt8Type ())
return LLVMInt32Type ();
else if (type == LLVMInt16Type ())
return LLVMInt32Type ();
else if (!cfg->r4fp && type == LLVMFloatType ())
return LLVMDoubleType ();
else
return type;
}
/*
* regtype_to_llvm_type:
*
* Return the LLVM type corresponding to the regtype C used in instruction
* descriptions.
*/
static LLVMTypeRef
regtype_to_llvm_type (char c)
{
switch (c) {
case 'i':
return LLVMInt32Type ();
case 'l':
return LLVMInt64Type ();
case 'f':
return LLVMDoubleType ();
default:
return NULL;
}
}
/*
* op_to_llvm_type:
*
* Return the LLVM type corresponding to the unary/binary opcode OPCODE.
*/
static LLVMTypeRef
op_to_llvm_type (int opcode)
{
switch (opcode) {
case OP_ICONV_TO_I1:
case OP_LCONV_TO_I1:
return LLVMInt8Type ();
case OP_ICONV_TO_U1:
case OP_LCONV_TO_U1:
return LLVMInt8Type ();
case OP_ICONV_TO_I2:
case OP_LCONV_TO_I2:
return LLVMInt16Type ();
case OP_ICONV_TO_U2:
case OP_LCONV_TO_U2:
return LLVMInt16Type ();
case OP_ICONV_TO_I4:
case OP_LCONV_TO_I4:
return LLVMInt32Type ();
case OP_ICONV_TO_U4:
case OP_LCONV_TO_U4:
return LLVMInt32Type ();
case OP_ICONV_TO_I8:
return LLVMInt64Type ();
case OP_ICONV_TO_R4:
return LLVMFloatType ();
case OP_ICONV_TO_R8:
return LLVMDoubleType ();
case OP_ICONV_TO_U8:
return LLVMInt64Type ();
case OP_FCONV_TO_I4:
return LLVMInt32Type ();
case OP_FCONV_TO_I8:
return LLVMInt64Type ();
case OP_FCONV_TO_I1:
case OP_FCONV_TO_U1:
case OP_RCONV_TO_I1:
case OP_RCONV_TO_U1:
return LLVMInt8Type ();
case OP_FCONV_TO_I2:
case OP_FCONV_TO_U2:
case OP_RCONV_TO_I2:
case OP_RCONV_TO_U2:
return LLVMInt16Type ();
case OP_RCONV_TO_U4:
return LLVMInt32Type ();
case OP_FCONV_TO_I:
case OP_FCONV_TO_U:
return TARGET_SIZEOF_VOID_P == 8 ? LLVMInt64Type () : LLVMInt32Type ();
case OP_IADD_OVF:
case OP_IADD_OVF_UN:
case OP_ISUB_OVF:
case OP_ISUB_OVF_UN:
case OP_IMUL_OVF:
case OP_IMUL_OVF_UN:
return LLVMInt32Type ();
case OP_LADD_OVF:
case OP_LADD_OVF_UN:
case OP_LSUB_OVF:
case OP_LSUB_OVF_UN:
case OP_LMUL_OVF:
case OP_LMUL_OVF_UN:
return LLVMInt64Type ();
default:
printf ("%s\n", mono_inst_name (opcode));
g_assert_not_reached ();
return NULL;
}
}
#define CLAUSE_START(clause) ((clause)->try_offset)
#define CLAUSE_END(clause) (((clause))->try_offset + ((clause))->try_len)
/*
* load_store_to_llvm_type:
*
* Return the size/sign/zero extension corresponding to the load/store opcode
* OPCODE.
*/
static LLVMTypeRef
load_store_to_llvm_type (int opcode, int *size, gboolean *sext, gboolean *zext)
{
*sext = FALSE;
*zext = FALSE;
switch (opcode) {
case OP_LOADI1_MEMBASE:
case OP_STOREI1_MEMBASE_REG:
case OP_STOREI1_MEMBASE_IMM:
case OP_ATOMIC_LOAD_I1:
case OP_ATOMIC_STORE_I1:
*size = 1;
*sext = TRUE;
return LLVMInt8Type ();
case OP_LOADU1_MEMBASE:
case OP_LOADU1_MEM:
case OP_ATOMIC_LOAD_U1:
case OP_ATOMIC_STORE_U1:
*size = 1;
*zext = TRUE;
return LLVMInt8Type ();
case OP_LOADI2_MEMBASE:
case OP_STOREI2_MEMBASE_REG:
case OP_STOREI2_MEMBASE_IMM:
case OP_ATOMIC_LOAD_I2:
case OP_ATOMIC_STORE_I2:
*size = 2;
*sext = TRUE;
return LLVMInt16Type ();
case OP_LOADU2_MEMBASE:
case OP_LOADU2_MEM:
case OP_ATOMIC_LOAD_U2:
case OP_ATOMIC_STORE_U2:
*size = 2;
*zext = TRUE;
return LLVMInt16Type ();
case OP_LOADI4_MEMBASE:
case OP_LOADU4_MEMBASE:
case OP_LOADI4_MEM:
case OP_LOADU4_MEM:
case OP_STOREI4_MEMBASE_REG:
case OP_STOREI4_MEMBASE_IMM:
case OP_ATOMIC_LOAD_I4:
case OP_ATOMIC_STORE_I4:
case OP_ATOMIC_LOAD_U4:
case OP_ATOMIC_STORE_U4:
*size = 4;
return LLVMInt32Type ();
case OP_LOADI8_MEMBASE:
case OP_LOADI8_MEM:
case OP_STOREI8_MEMBASE_REG:
case OP_STOREI8_MEMBASE_IMM:
case OP_ATOMIC_LOAD_I8:
case OP_ATOMIC_STORE_I8:
case OP_ATOMIC_LOAD_U8:
case OP_ATOMIC_STORE_U8:
*size = 8;
return LLVMInt64Type ();
case OP_LOADR4_MEMBASE:
case OP_STORER4_MEMBASE_REG:
case OP_ATOMIC_LOAD_R4:
case OP_ATOMIC_STORE_R4:
*size = 4;
return LLVMFloatType ();
case OP_LOADR8_MEMBASE:
case OP_STORER8_MEMBASE_REG:
case OP_ATOMIC_LOAD_R8:
case OP_ATOMIC_STORE_R8:
*size = 8;
return LLVMDoubleType ();
case OP_LOAD_MEMBASE:
case OP_LOAD_MEM:
case OP_STORE_MEMBASE_REG:
case OP_STORE_MEMBASE_IMM:
*size = TARGET_SIZEOF_VOID_P;
return IntPtrType ();
default:
g_assert_not_reached ();
return NULL;
}
}
/*
* ovf_op_to_intrins:
*
* Return the LLVM intrinsics corresponding to the overflow opcode OPCODE.
*/
static const char*
ovf_op_to_intrins (int opcode)
{
switch (opcode) {
case OP_IADD_OVF:
return "llvm.sadd.with.overflow.i32";
case OP_IADD_OVF_UN:
return "llvm.uadd.with.overflow.i32";
case OP_ISUB_OVF:
return "llvm.ssub.with.overflow.i32";
case OP_ISUB_OVF_UN:
return "llvm.usub.with.overflow.i32";
case OP_IMUL_OVF:
return "llvm.smul.with.overflow.i32";
case OP_IMUL_OVF_UN:
return "llvm.umul.with.overflow.i32";
case OP_LADD_OVF:
return "llvm.sadd.with.overflow.i64";
case OP_LADD_OVF_UN:
return "llvm.uadd.with.overflow.i64";
case OP_LSUB_OVF:
return "llvm.ssub.with.overflow.i64";
case OP_LSUB_OVF_UN:
return "llvm.usub.with.overflow.i64";
case OP_LMUL_OVF:
return "llvm.smul.with.overflow.i64";
case OP_LMUL_OVF_UN:
return "llvm.umul.with.overflow.i64";
default:
g_assert_not_reached ();
return NULL;
}
}
static const char*
simd_op_to_intrins (int opcode)
{
switch (opcode) {
#if defined(TARGET_X86) || defined(TARGET_AMD64)
case OP_MINPD:
return "llvm.x86.sse2.min.pd";
case OP_MINPS:
return "llvm.x86.sse.min.ps";
case OP_MAXPD:
return "llvm.x86.sse2.max.pd";
case OP_MAXPS:
return "llvm.x86.sse.max.ps";
case OP_HADDPD:
return "llvm.x86.sse3.hadd.pd";
case OP_HADDPS:
return "llvm.x86.sse3.hadd.ps";
case OP_HSUBPD:
return "llvm.x86.sse3.hsub.pd";
case OP_HSUBPS:
return "llvm.x86.sse3.hsub.ps";
case OP_ADDSUBPS:
return "llvm.x86.sse3.addsub.ps";
case OP_ADDSUBPD:
return "llvm.x86.sse3.addsub.pd";
case OP_EXTRACT_MASK:
return "llvm.x86.sse2.pmovmskb.128";
case OP_PSHRW:
case OP_PSHRW_REG:
return "llvm.x86.sse2.psrli.w";
case OP_PSHRD:
case OP_PSHRD_REG:
return "llvm.x86.sse2.psrli.d";
case OP_PSHRQ:
case OP_PSHRQ_REG:
return "llvm.x86.sse2.psrli.q";
case OP_PSHLW:
case OP_PSHLW_REG:
return "llvm.x86.sse2.pslli.w";
case OP_PSHLD:
case OP_PSHLD_REG:
return "llvm.x86.sse2.pslli.d";
case OP_PSHLQ:
case OP_PSHLQ_REG:
return "llvm.x86.sse2.pslli.q";
case OP_PSARW:
case OP_PSARW_REG:
return "llvm.x86.sse2.psrai.w";
case OP_PSARD:
case OP_PSARD_REG:
return "llvm.x86.sse2.psrai.d";
case OP_PADDB_SAT:
return "llvm.x86.sse2.padds.b";
case OP_PADDW_SAT:
return "llvm.x86.sse2.padds.w";
case OP_PSUBB_SAT:
return "llvm.x86.sse2.psubs.b";
case OP_PSUBW_SAT:
return "llvm.x86.sse2.psubs.w";
case OP_PADDB_SAT_UN:
return "llvm.x86.sse2.paddus.b";
case OP_PADDW_SAT_UN:
return "llvm.x86.sse2.paddus.w";
case OP_PSUBB_SAT_UN:
return "llvm.x86.sse2.psubus.b";
case OP_PSUBW_SAT_UN:
return "llvm.x86.sse2.psubus.w";
case OP_PAVGB_UN:
return "llvm.x86.sse2.pavg.b";
case OP_PAVGW_UN:
return "llvm.x86.sse2.pavg.w";
case OP_SQRTPS:
return "llvm.x86.sse.sqrt.ps";
case OP_SQRTPD:
return "llvm.x86.sse2.sqrt.pd";
case OP_RSQRTPS:
return "llvm.x86.sse.rsqrt.ps";
case OP_RCPPS:
return "llvm.x86.sse.rcp.ps";
case OP_CVTDQ2PD:
return "llvm.x86.sse2.cvtdq2pd";
case OP_CVTDQ2PS:
return "llvm.x86.sse2.cvtdq2ps";
case OP_CVTPD2DQ:
return "llvm.x86.sse2.cvtpd2dq";
case OP_CVTPS2DQ:
return "llvm.x86.sse2.cvtps2dq";
case OP_CVTPD2PS:
return "llvm.x86.sse2.cvtpd2ps";
case OP_CVTPS2PD:
return "llvm.x86.sse2.cvtps2pd";
case OP_CVTTPD2DQ:
return "llvm.x86.sse2.cvttpd2dq";
case OP_CVTTPS2DQ:
return "llvm.x86.sse2.cvttps2dq";
case OP_PACKW:
return "llvm.x86.sse2.packsswb.128";
case OP_PACKD:
return "llvm.x86.sse2.packssdw.128";
case OP_PACKW_UN:
return "llvm.x86.sse2.packuswb.128";
case OP_PACKD_UN:
return "llvm.x86.sse41.packusdw";
case OP_PMULW_HIGH:
return "llvm.x86.sse2.pmulh.w";
case OP_PMULW_HIGH_UN:
return "llvm.x86.sse2.pmulhu.w";
case OP_DPPS:
return "llvm.x86.sse41.dpps";
#endif
default:
g_assert_not_reached ();
return NULL;
}
}
static LLVMTypeRef
simd_op_to_llvm_type (int opcode)
{
#if defined(TARGET_X86) || defined(TARGET_AMD64)
switch (opcode) {
case OP_EXTRACT_R8:
case OP_EXPAND_R8:
return type_to_simd_type (MONO_TYPE_R8);
case OP_EXTRACT_I8:
case OP_EXPAND_I8:
return type_to_simd_type (MONO_TYPE_I8);
case OP_EXTRACT_I4:
case OP_EXPAND_I4:
return type_to_simd_type (MONO_TYPE_I4);
case OP_EXTRACT_I2:
case OP_EXTRACT_U2:
case OP_EXTRACTX_U2:
case OP_EXPAND_I2:
return type_to_simd_type (MONO_TYPE_I2);
case OP_EXTRACT_I1:
case OP_EXTRACT_U1:
case OP_EXPAND_I1:
return type_to_simd_type (MONO_TYPE_I1);
case OP_EXPAND_R4:
return type_to_simd_type (MONO_TYPE_R4);
case OP_CVTDQ2PD:
case OP_CVTDQ2PS:
return type_to_simd_type (MONO_TYPE_I4);
case OP_CVTPD2DQ:
case OP_CVTPD2PS:
case OP_CVTTPD2DQ:
return type_to_simd_type (MONO_TYPE_R8);
case OP_CVTPS2DQ:
case OP_CVTPS2PD:
case OP_CVTTPS2DQ:
return type_to_simd_type (MONO_TYPE_R4);
case OP_EXTRACT_MASK:
return type_to_simd_type (MONO_TYPE_I1);
case OP_SQRTPS:
case OP_RSQRTPS:
case OP_RCPPS:
case OP_DUPPS_LOW:
case OP_DUPPS_HIGH:
return type_to_simd_type (MONO_TYPE_R4);
case OP_SQRTPD:
case OP_DUPPD:
return type_to_simd_type (MONO_TYPE_R8);
default:
g_assert_not_reached ();
return NULL;
}
#else
return NULL;
#endif
}
static void
set_preserveall_cc (LLVMValueRef func)
{
/*
* xcode10 (watchOS) and ARM/ARM64 doesn't seem to support preserveall, it fails with:
* fatal error: error in backend: Unsupported calling convention
*/
#if !defined(TARGET_WATCHOS) && !defined(TARGET_ARM) && !defined(TARGET_ARM64)
mono_llvm_set_preserveall_cc (func);
#endif
}
static void
set_call_preserveall_cc (LLVMValueRef func)
{
#if !defined(TARGET_WATCHOS) && !defined(TARGET_ARM) && !defined(TARGET_ARM64)
mono_llvm_set_call_preserveall_cc (func);
#endif
}
/*
* get_bb:
*
* Return the LLVM basic block corresponding to BB.
*/
static LLVMBasicBlockRef
get_bb (EmitContext *ctx, MonoBasicBlock *bb)
{
char bb_name_buf [128];
char *bb_name;
if (ctx->bblocks [bb->block_num].bblock == NULL) {
if (bb->flags & BB_EXCEPTION_HANDLER) {
int clause_index = (mono_get_block_region_notry (ctx->cfg, bb->region) >> 8) - 1;
sprintf (bb_name_buf, "EH_CLAUSE%d_BB%d", clause_index, bb->block_num);
bb_name = bb_name_buf;
} else if (bb->block_num < 256) {
if (!ctx->module->bb_names) {
ctx->module->bb_names_len = 256;
ctx->module->bb_names = g_new0 (char*, ctx->module->bb_names_len);
}
if (!ctx->module->bb_names [bb->block_num]) {
char *n;
n = g_strdup_printf ("BB%d", bb->block_num);
mono_memory_barrier ();
ctx->module->bb_names [bb->block_num] = n;
}
bb_name = ctx->module->bb_names [bb->block_num];
} else {
sprintf (bb_name_buf, "BB%d", bb->block_num);
bb_name = bb_name_buf;
}
ctx->bblocks [bb->block_num].bblock = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
ctx->bblocks [bb->block_num].end_bblock = ctx->bblocks [bb->block_num].bblock;
}
return ctx->bblocks [bb->block_num].bblock;
}
/*
* get_end_bb:
*
* Return the last LLVM bblock corresponding to BB.
* This might not be equal to the bb returned by get_bb () since we need to generate
* multiple LLVM bblocks for a mono bblock to handle throwing exceptions.
*/
static LLVMBasicBlockRef
get_end_bb (EmitContext *ctx, MonoBasicBlock *bb)
{
get_bb (ctx, bb);
return ctx->bblocks [bb->block_num].end_bblock;
}
static LLVMBasicBlockRef
gen_bb (EmitContext *ctx, const char *prefix)
{
char bb_name [128];
sprintf (bb_name, "%s%d", prefix, ++ ctx->ex_index);
return LLVMAppendBasicBlock (ctx->lmethod, bb_name);
}
/*
* resolve_patch:
*
* Return the target of the patch identified by TYPE and TARGET.
*/
static gpointer
resolve_patch (MonoCompile *cfg, MonoJumpInfoType type, gconstpointer target)
{
MonoJumpInfo ji;
ERROR_DECL (error);
gpointer res;
memset (&ji, 0, sizeof (ji));
ji.type = type;
ji.data.target = target;
res = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE, error);
mono_error_assert_ok (error);
return res;
}
/*
* convert_full:
*
* Emit code to convert the LLVM value V to DTYPE.
*/
static LLVMValueRef
convert_full (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype, gboolean is_unsigned)
{
LLVMTypeRef stype = LLVMTypeOf (v);
if (stype != dtype) {
gboolean ext = FALSE;
/* Extend */
if (dtype == LLVMInt64Type () && (stype == LLVMInt32Type () || stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
ext = TRUE;
else if (dtype == LLVMInt32Type () && (stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
ext = TRUE;
else if (dtype == LLVMInt16Type () && (stype == LLVMInt8Type ()))
ext = TRUE;
if (ext)
return is_unsigned ? LLVMBuildZExt (ctx->builder, v, dtype, "") : LLVMBuildSExt (ctx->builder, v, dtype, "");
if (dtype == LLVMDoubleType () && stype == LLVMFloatType ())
return LLVMBuildFPExt (ctx->builder, v, dtype, "");
/* Trunc */
if (stype == LLVMInt64Type () && (dtype == LLVMInt32Type () || dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
return LLVMBuildTrunc (ctx->builder, v, dtype, "");
if (stype == LLVMInt32Type () && (dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
return LLVMBuildTrunc (ctx->builder, v, dtype, "");
if (stype == LLVMInt16Type () && dtype == LLVMInt8Type ())
return LLVMBuildTrunc (ctx->builder, v, dtype, "");
if (stype == LLVMDoubleType () && dtype == LLVMFloatType ())
return LLVMBuildFPTrunc (ctx->builder, v, dtype, "");
if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind && LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
return LLVMBuildBitCast (ctx->builder, v, dtype, "");
if (LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
return LLVMBuildIntToPtr (ctx->builder, v, dtype, "");
if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind)
return LLVMBuildPtrToInt (ctx->builder, v, dtype, "");
if (mono_arch_is_soft_float ()) {
if (stype == LLVMInt32Type () && dtype == LLVMFloatType ())
return LLVMBuildBitCast (ctx->builder, v, dtype, "");
if (stype == LLVMInt32Type () && dtype == LLVMDoubleType ())
return LLVMBuildBitCast (ctx->builder, LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), ""), dtype, "");
}
if (LLVMGetTypeKind (stype) == LLVMVectorTypeKind && LLVMGetTypeKind (dtype) == LLVMVectorTypeKind)
return LLVMBuildBitCast (ctx->builder, v, dtype, "");
LLVMDumpValue (v);
LLVMDumpValue (LLVMConstNull (dtype));
g_assert_not_reached ();
return NULL;
} else {
return v;
}
}
static LLVMValueRef
convert (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype)
{
return convert_full (ctx, v, dtype, FALSE);
}
/*
* emit_volatile_load:
*
* If vreg is volatile, emit a load from its address.
*/
static LLVMValueRef
emit_volatile_load (EmitContext *ctx, int vreg)
{
MonoType *t;
LLVMValueRef v;
// On arm64, we pass the rgctx in a callee saved
// register on arm64 (x15), and llvm might keep the value in that register
// even through the register is marked as 'reserved' inside llvm.
v = mono_llvm_build_load (ctx->builder, ctx->addresses [vreg], "", TRUE);
t = ctx->vreg_cli_types [vreg];
if (t && !t->byref) {
/*
* Might have to zero extend since llvm doesn't have
* unsigned types.
*/
if (t->type == MONO_TYPE_U1 || t->type == MONO_TYPE_U2 || t->type == MONO_TYPE_CHAR || t->type == MONO_TYPE_BOOLEAN)
v = LLVMBuildZExt (ctx->builder, v, LLVMInt32Type (), "");
else if (t->type == MONO_TYPE_I1 || t->type == MONO_TYPE_I2)
v = LLVMBuildSExt (ctx->builder, v, LLVMInt32Type (), "");
else if (t->type == MONO_TYPE_U8)
v = LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), "");
}
return v;
}
/*
* emit_volatile_store:
*
* If VREG is volatile, emit a store from its value to its address.
*/
static void
emit_volatile_store (EmitContext *ctx, int vreg)
{
MonoInst *var = get_vreg_to_inst (ctx->cfg, vreg);
if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
g_assert (ctx->addresses [vreg]);
LLVMBuildStore (ctx->builder, convert (ctx, ctx->values [vreg], type_to_llvm_type (ctx, var->inst_vtype)), ctx->addresses [vreg]);
}
}
static LLVMTypeRef
sig_to_llvm_sig_no_cinfo (EmitContext *ctx, MonoMethodSignature *sig)
{
LLVMTypeRef ret_type;
LLVMTypeRef *param_types = NULL;
LLVMTypeRef res;
int i, pindex;
MonoType *rtype;
ret_type = type_to_llvm_type (ctx, sig->ret);
if (!ctx_ok (ctx))
return NULL;
rtype = mini_get_underlying_type (sig->ret);
param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
pindex = 0;
if (sig->hasthis)
param_types [pindex ++] = ThisType ();
for (i = 0; i < sig->param_count; ++i)
param_types [pindex ++] = type_to_llvm_arg_type (ctx, sig->params [i]);
if (!ctx_ok (ctx)) {
g_free (param_types);
return NULL;
}
res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
g_free (param_types);
return res;
}
/*
* sig_to_llvm_sig_full:
*
* Return the LLVM signature corresponding to the mono signature SIG using the
* calling convention information in CINFO. Fill out the parameter mapping information in CINFO.
*/
static LLVMTypeRef
sig_to_llvm_sig_full (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo)
{
LLVMTypeRef ret_type;
LLVMTypeRef *param_types = NULL;
LLVMTypeRef res;
int i, j, pindex, vret_arg_pindex = 0;
gboolean vretaddr = FALSE;
MonoType *rtype;
if (!cinfo)
return sig_to_llvm_sig_no_cinfo (ctx, sig);
ret_type = type_to_llvm_type (ctx, sig->ret);
if (!ctx_ok (ctx))
return NULL;
rtype = mini_get_underlying_type (sig->ret);
switch (cinfo->ret.storage) {
case LLVMArgVtypeInReg:
/* LLVM models this by returning an aggregate value */
if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgNone) {
LLVMTypeRef members [2];
members [0] = IntPtrType ();
ret_type = LLVMStructType (members, 1, FALSE);
} else if (cinfo->ret.pair_storage [0] == LLVMArgNone && cinfo->ret.pair_storage [1] == LLVMArgNone) {
/* Empty struct */
ret_type = LLVMVoidType ();
} else if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgInIReg) {
LLVMTypeRef members [2];
members [0] = IntPtrType ();
members [1] = IntPtrType ();
ret_type = LLVMStructType (members, 2, FALSE);
} else {
g_assert_not_reached ();
}
break;
case LLVMArgVtypeByVal:
/* Vtype returned normally by val */
break;
case LLVMArgVtypeAsScalar: {
int size = mono_class_value_size (mono_class_from_mono_type_internal (rtype), NULL);
/* LLVM models this by returning an int */
if (size < TARGET_SIZEOF_VOID_P) {
g_assert (cinfo->ret.nslots == 1);
ret_type = LLVMIntType (size * 8);
} else {
g_assert (cinfo->ret.nslots == 1 || cinfo->ret.nslots == 2);
ret_type = LLVMIntType (cinfo->ret.nslots * sizeof (target_mgreg_t) * 8);
}
break;
}
case LLVMArgAsIArgs:
ret_type = LLVMArrayType (IntPtrType (), cinfo->ret.nslots);
break;
case LLVMArgFpStruct: {
/* Vtype returned as a fp struct */
LLVMTypeRef members [16];
/* Have to create our own structure since we don't map fp structures to LLVM fp structures yet */
for (i = 0; i < cinfo->ret.nslots; ++i)
members [i] = cinfo->ret.esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
ret_type = LLVMStructType (members, cinfo->ret.nslots, FALSE);
break;
}
case LLVMArgVtypeByRef:
/* Vtype returned using a hidden argument */
ret_type = LLVMVoidType ();
break;
case LLVMArgVtypeRetAddr:
case LLVMArgGsharedvtFixed:
case LLVMArgGsharedvtFixedVtype:
case LLVMArgGsharedvtVariable:
vretaddr = TRUE;
ret_type = LLVMVoidType ();
break;
default:
break;
}
param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
pindex = 0;
if (cinfo->ret.storage == LLVMArgVtypeByRef) {
/*
* Has to be the first argument because of the sret argument attribute
* FIXME: This might conflict with passing 'this' as the first argument, but
* this is only used on arm64 which has a dedicated struct return register.
*/
cinfo->vret_arg_pindex = pindex;
param_types [pindex] = type_to_llvm_arg_type (ctx, sig->ret);
if (!ctx_ok (ctx)) {
g_free (param_types);
return NULL;
}
param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
pindex ++;
}
if (!ctx->llvm_only && cinfo->rgctx_arg) {
cinfo->rgctx_arg_pindex = pindex;
param_types [pindex] = ctx->module->ptr_type;
pindex ++;
}
if (cinfo->imt_arg) {
cinfo->imt_arg_pindex = pindex;
param_types [pindex] = ctx->module->ptr_type;
pindex ++;
}
if (vretaddr) {
/* Compute the index in the LLVM signature where the vret arg needs to be passed */
vret_arg_pindex = pindex;
if (cinfo->vret_arg_index == 1) {
/* Add the slots consumed by the first argument */
LLVMArgInfo *ainfo = &cinfo->args [0];
switch (ainfo->storage) {
case LLVMArgVtypeInReg:
for (j = 0; j < 2; ++j) {
if (ainfo->pair_storage [j] == LLVMArgInIReg)
vret_arg_pindex ++;
}
break;
default:
vret_arg_pindex ++;
}
}
cinfo->vret_arg_pindex = vret_arg_pindex;
}
if (vretaddr && vret_arg_pindex == pindex)
param_types [pindex ++] = IntPtrType ();
if (sig->hasthis) {
cinfo->this_arg_pindex = pindex;
param_types [pindex ++] = ThisType ();
cinfo->args [0].pindex = cinfo->this_arg_pindex;
}
if (vretaddr && vret_arg_pindex == pindex)
param_types [pindex ++] = IntPtrType ();
for (i = 0; i < sig->param_count; ++i) {
LLVMArgInfo *ainfo = &cinfo->args [i + sig->hasthis];
if (vretaddr && vret_arg_pindex == pindex)
param_types [pindex ++] = IntPtrType ();
ainfo->pindex = pindex;
switch (ainfo->storage) {
case LLVMArgVtypeInReg:
for (j = 0; j < 2; ++j) {
switch (ainfo->pair_storage [j]) {
case LLVMArgInIReg:
param_types [pindex ++] = LLVMIntType (TARGET_SIZEOF_VOID_P * 8);
break;
case LLVMArgNone:
break;
default:
g_assert_not_reached ();
}
}
break;
case LLVMArgVtypeByVal:
param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
if (!ctx_ok (ctx))
break;
param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
pindex ++;
break;
case LLVMArgAsIArgs:
if (ainfo->esize == 8)
param_types [pindex] = LLVMArrayType (LLVMInt64Type (), ainfo->nslots);
else
param_types [pindex] = LLVMArrayType (IntPtrType (), ainfo->nslots);
pindex ++;
break;
case LLVMArgVtypeByRef:
param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
if (!ctx_ok (ctx))
break;
param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
pindex ++;
break;
case LLVMArgAsFpArgs: {
int j;
/* Emit dummy fp arguments if needed so the rest is passed on the stack */
for (j = 0; j < ainfo->ndummy_fpargs; ++j)
param_types [pindex ++] = LLVMDoubleType ();
for (j = 0; j < ainfo->nslots; ++j)
param_types [pindex ++] = ainfo->esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
break;
}
case LLVMArgVtypeAsScalar:
g_assert_not_reached ();
break;
case LLVMArgGsharedvtFixed:
case LLVMArgGsharedvtFixedVtype:
param_types [pindex ++] = LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0);
break;
case LLVMArgGsharedvtVariable:
param_types [pindex ++] = LLVMPointerType (IntPtrType (), 0);
break;
default:
param_types [pindex ++] = type_to_llvm_arg_type (ctx, ainfo->type);
break;
}
}
if (!ctx_ok (ctx)) {
g_free (param_types);
return NULL;
}
if (vretaddr && vret_arg_pindex == pindex)
param_types [pindex ++] = IntPtrType ();
if (ctx->llvm_only && cinfo->rgctx_arg) {
/* Pass the rgctx as the last argument */
cinfo->rgctx_arg_pindex = pindex;
param_types [pindex] = ctx->module->ptr_type;
pindex ++;
} else if (ctx->llvm_only && cinfo->dummy_arg) {
/* Pass a dummy arg last */
cinfo->dummy_arg_pindex = pindex;
param_types [pindex] = ctx->module->ptr_type;
pindex ++;
}
res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
g_free (param_types);
return res;
}
static LLVMTypeRef
sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig)
{
return sig_to_llvm_sig_full (ctx, sig, NULL);
}
/*
* LLVMFunctionType1:
*
* Create an LLVM function type from the arguments.
*/
static G_GNUC_UNUSED LLVMTypeRef
LLVMFunctionType0 (LLVMTypeRef ReturnType,
int IsVarArg)
{
return LLVMFunctionType (ReturnType, NULL, 0, IsVarArg);
}
/*
* LLVMFunctionType1:
*
* Create an LLVM function type from the arguments.
*/
static G_GNUC_UNUSED LLVMTypeRef
LLVMFunctionType1 (LLVMTypeRef ReturnType,
LLVMTypeRef ParamType1,
int IsVarArg)
{
LLVMTypeRef param_types [1];
param_types [0] = ParamType1;
return LLVMFunctionType (ReturnType, param_types, 1, IsVarArg);
}
/*
* LLVMFunctionType2:
*
* Create an LLVM function type from the arguments.
*/
static G_GNUC_UNUSED LLVMTypeRef
LLVMFunctionType2 (LLVMTypeRef ReturnType,
LLVMTypeRef ParamType1,
LLVMTypeRef ParamType2,
int IsVarArg)
{
LLVMTypeRef param_types [2];
param_types [0] = ParamType1;
param_types [1] = ParamType2;
return LLVMFunctionType (ReturnType, param_types, 2, IsVarArg);
}
/*
* LLVMFunctionType3:
*
* Create an LLVM function type from the arguments.
*/
static G_GNUC_UNUSED LLVMTypeRef
LLVMFunctionType3 (LLVMTypeRef ReturnType,
LLVMTypeRef ParamType1,
LLVMTypeRef ParamType2,
LLVMTypeRef ParamType3,
int IsVarArg)
{
LLVMTypeRef param_types [3];
param_types [0] = ParamType1;
param_types [1] = ParamType2;
param_types [2] = ParamType3;
return LLVMFunctionType (ReturnType, param_types, 3, IsVarArg);
}
static G_GNUC_UNUSED LLVMTypeRef
LLVMFunctionType5 (LLVMTypeRef ReturnType,
LLVMTypeRef ParamType1,
LLVMTypeRef ParamType2,
LLVMTypeRef ParamType3,
LLVMTypeRef ParamType4,
LLVMTypeRef ParamType5,
int IsVarArg)
{
LLVMTypeRef param_types [5];
param_types [0] = ParamType1;
param_types [1] = ParamType2;
param_types [2] = ParamType3;
param_types [3] = ParamType4;
param_types [4] = ParamType5;
return LLVMFunctionType (ReturnType, param_types, 5, IsVarArg);
}
/*
* create_builder:
*
* Create an LLVM builder and remember it so it can be freed later.
*/
static LLVMBuilderRef
create_builder (EmitContext *ctx)
{
LLVMBuilderRef builder = LLVMCreateBuilder ();
ctx->builders = g_slist_prepend_mempool (ctx->cfg->mempool, ctx->builders, builder);
emit_default_dbg_loc (ctx, builder);
return builder;
}
static char*
get_aotconst_name (MonoJumpInfoType type, gconstpointer data, int got_offset)
{
char *name;
switch (type) {
case MONO_PATCH_INFO_JIT_ICALL:
name = g_strdup_printf ("jit_icall_%s", data);
break;
case MONO_PATCH_INFO_RGCTX_SLOT_INDEX: {
MonoJumpInfoRgctxEntry *entry = (MonoJumpInfoRgctxEntry*)data;
name = g_strdup_printf ("RGCTX_SLOT_INDEX_%s", mono_rgctx_info_type_to_str (entry->info_type));
break;
}
default:
name = g_strdup_printf ("%s_%d", mono_ji_type_to_string (type), got_offset);
break;
}
return name;
}
static LLVMValueRef
get_aotconst_typed (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data, LLVMTypeRef llvm_type)
{
MonoCompile *cfg;
guint32 got_offset;
LLVMValueRef indexes [2];
LLVMValueRef got_entry_addr, load;
LLVMBuilderRef builder = ctx->builder;
char *name = NULL;
cfg = ctx->cfg;
MonoJumpInfo tmp_ji;
tmp_ji.type = type;
tmp_ji.data.target = data;
MonoJumpInfo *ji = mono_aot_patch_info_dup (&tmp_ji);
ji->next = cfg->patch_info;
cfg->patch_info = ji;
got_offset = mono_aot_get_got_offset (cfg->patch_info);
ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
/*
* If the got slot is shared, it means its initialized when the aot image is loaded, so we don't need to
* explicitly initialize it.
*/
if (!mono_aot_is_shared_got_offset (got_offset)) {
//mono_print_ji (ji);
//printf ("\n");
ctx->cfg->got_access_count ++;
}
indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
name = get_aotconst_name (type, data, got_offset);
if (llvm_type) {
load = LLVMBuildLoad (builder, got_entry_addr, "");
load = convert (ctx, load, llvm_type);
LLVMSetValueName (load, name ? name : "");
} else {
load = LLVMBuildLoad (builder, got_entry_addr, name ? name : "");
}
g_free (name);
//set_invariant_load_flag (load);
return load;
}
static LLVMValueRef
get_aotconst (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data)
{
return get_aotconst_typed (ctx, type, data, NULL);
}
typedef struct {
MonoJumpInfo *ji;
MonoMethod *method;
LLVMValueRef load;
LLVMTypeRef type;
} CallSite;
static LLVMValueRef
get_callee_llvmonly (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
{
LLVMValueRef callee;
char *callee_name = NULL;
if (ctx->module->static_link && ctx->module->assembly->image != mono_get_corlib () && type == MONO_PATCH_INFO_JIT_ICALL) {
MonoJitICallInfo *info = mono_find_jit_icall_by_name ((const char*)data);
g_assert (info);
if (info->func != info->wrapper) {
type = MONO_PATCH_INFO_METHOD;
data = mono_icall_get_wrapper_method (info);
callee_name = mono_aot_get_mangled_method_name ((MonoMethod*)data);
}
}
if (!callee_name)
callee_name = mono_aot_get_direct_call_symbol (type, data);
if (callee_name) {
/* Directly callable */
// FIXME: Locking
callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->direct_callables, callee_name);
if (!callee) {
callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
LLVMSetVisibility (callee, LLVMHiddenVisibility);
g_hash_table_insert (ctx->module->direct_callables, (char*)callee_name, callee);
} else {
/* LLVMTypeRef's are uniqued */
if (LLVMGetElementType (LLVMTypeOf (callee)) != llvm_sig)
return LLVMConstBitCast (callee, LLVMPointerType (llvm_sig, 0));
g_free (callee_name);
}
return callee;
}
/*
* Change references to jit icalls to the icall wrappers when in corlib, so
* they can be called directly.
*/
if (ctx->module->assembly->image == mono_get_corlib () && type == MONO_PATCH_INFO_JIT_ICALL) {
MonoJitICallInfo *info = mono_find_jit_icall_by_name ((const char*)data);
g_assert (info);
if (info->func != info->wrapper) {
type = MONO_PATCH_INFO_METHOD;
data = mono_icall_get_wrapper_method (info);
}
}
/*
* Instead of emitting an indirect call through a got slot, emit a placeholder, and
* replace it with a direct call or an indirect call in mono_llvm_fixup_aot_module ()
* after all methods have been emitted.
*/
if (type == MONO_PATCH_INFO_METHOD) {
MonoMethod *method = (MonoMethod*)data;
if (m_class_get_image (method->klass)->assembly == ctx->module->assembly) {
MonoJumpInfo tmp_ji;
tmp_ji.type = type;
tmp_ji.data.target = data;
MonoJumpInfo *ji = mono_aot_patch_info_dup (&tmp_ji);
ji->next = ctx->cfg->patch_info;
ctx->cfg->patch_info = ji;
LLVMTypeRef llvm_type = LLVMPointerType (llvm_sig, 0);
ctx->cfg->got_access_count ++;
CallSite *info = g_new0 (CallSite, 1);
info->method = method;
info->ji = ji;
info->type = llvm_type;
/*
* Emit a dummy load to represent the callee, and either replace it with
* a reference to the llvm method for the callee, or from a load from the
* GOT.
*/
LLVMValueRef indexes [2];
LLVMValueRef got_entry_addr, load;
LLVMBuilderRef builder = ctx->builder;
indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
indexes [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
load = LLVMBuildLoad (builder, got_entry_addr, "");
load = convert (ctx, load, llvm_type);
info->load = load;
g_ptr_array_add (ctx->callsite_list, info);
return load;
}
}
/*
* Calls are made through the GOT.
*/
callee = get_aotconst_typed (ctx, type, data, LLVMPointerType (llvm_sig, 0));
return callee;
}
/*
* get_callee:
*
* Return an llvm value representing the callee given by the arguments.
*/
static LLVMValueRef
get_callee (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
{
LLVMValueRef callee;
char *callee_name;
MonoJumpInfo *ji = NULL;
if (ctx->llvm_only)
return get_callee_llvmonly (ctx, llvm_sig, type, data);
callee_name = mono_aot_get_plt_symbol (type, data);
if (!callee_name)
return NULL;
if (ctx->cfg->compile_aot)
/* Add a patch so referenced wrappers can be compiled in full aot mode */
mono_add_patch_info (ctx->cfg, 0, type, data);
// FIXME: Locking
callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->plt_entries, callee_name);
if (!callee) {
callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
LLVMSetVisibility (callee, LLVMHiddenVisibility);
g_hash_table_insert (ctx->module->plt_entries, (char*)callee_name, callee);
}
if (ctx->cfg->compile_aot) {
ji = g_new0 (MonoJumpInfo, 1);
ji->type = type;
ji->data.target = data;
g_hash_table_insert (ctx->module->plt_entries_ji, ji, callee);
}
return callee;
}
static LLVMValueRef
emit_jit_callee (EmitContext *ctx, const char *name, LLVMTypeRef llvm_sig, gpointer target)
{
#if LLVM_API_VERSION > 100
LLVMValueRef tramp_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (llvm_sig, 0), name);
LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (llvm_sig, 0)));
LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
LLVMValueRef callee = LLVMBuildLoad (ctx->builder, tramp_var, "");
return callee;
#else
LLVMValueRef callee = LLVMAddFunction (ctx->lmodule, "", llvm_sig);
LLVMAddGlobalMapping (ctx->module->ee, callee, target);
return callee;
#endif
}
static int
get_handler_clause (MonoCompile *cfg, MonoBasicBlock *bb)
{
MonoMethodHeader *header = cfg->header;
MonoExceptionClause *clause;
int i;
/* Directly */
if (bb->region != -1 && MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))
return (bb->region >> 8) - 1;
/* Indirectly */
for (i = 0; i < header->num_clauses; ++i) {
clause = &header->clauses [i];
if (MONO_OFFSET_IN_CLAUSE (clause, bb->real_offset) && clause->flags == MONO_EXCEPTION_CLAUSE_NONE)
return i;
}
return -1;
}
static MonoExceptionClause *
get_most_deep_clause (MonoCompile *cfg, EmitContext *ctx, MonoBasicBlock *bb)
{
if (bb == cfg->bb_init)
return NULL;
// Since they're sorted by nesting we just need
// the first one that the bb is a member of
for (int i = 0; i < cfg->header->num_clauses; i++) {
MonoExceptionClause *curr = &cfg->header->clauses [i];
if (MONO_OFFSET_IN_CLAUSE (curr, bb->real_offset))
return curr;
}
return NULL;
}
static void
set_metadata_flag (LLVMValueRef v, const char *flag_name)
{
LLVMValueRef md_arg;
int md_kind;
md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
md_arg = LLVMMDString ("mono", 4);
LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
}
static void
set_invariant_load_flag (LLVMValueRef v)
{
LLVMValueRef md_arg;
int md_kind;
const char *flag_name;
// FIXME: Cache this
flag_name = "invariant.load";
md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
md_arg = LLVMMDString ("<index>", strlen ("<index>"));
LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
}
/*
* emit_call:
*
* Emit an LLVM call or invoke instruction depending on whenever the call is inside
* a try region.
*/
static LLVMValueRef
emit_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, LLVMValueRef callee, LLVMValueRef *args, int pindex)
{
MonoCompile *cfg = ctx->cfg;
LLVMValueRef lcall = NULL;
LLVMBuilderRef builder = *builder_ref;
MonoExceptionClause *clause;
if (ctx->llvm_only) {
clause = get_most_deep_clause (cfg, ctx, bb);
if (clause) {
g_assert (clause->flags == MONO_EXCEPTION_CLAUSE_NONE || clause->flags == MONO_EXCEPTION_CLAUSE_FINALLY || clause->flags == MONO_EXCEPTION_CLAUSE_FAULT);
/*
* Have to use an invoke instead of a call, branching to the
* handler bblock of the clause containing this bblock.
*/
intptr_t key = CLAUSE_END(clause);
LLVMBasicBlockRef lpad_bb = (LLVMBasicBlockRef)g_hash_table_lookup (ctx->exc_meta, (gconstpointer)key);
// FIXME: Find the one that has the lowest end bound for the right start address
// FIXME: Finally + nesting
if (lpad_bb) {
LLVMBasicBlockRef noex_bb = gen_bb (ctx, "CALL_NOEX_BB");
/* Use an invoke */
lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, lpad_bb, "");
builder = ctx->builder = create_builder (ctx);
LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
ctx->bblocks [bb->block_num].end_bblock = noex_bb;
}
}
} else {
int clause_index = get_handler_clause (cfg, bb);
if (clause_index != -1) {
MonoMethodHeader *header = cfg->header;
MonoExceptionClause *ec = &header->clauses [clause_index];
MonoBasicBlock *tblock;
LLVMBasicBlockRef ex_bb, noex_bb;
/*
* Have to use an invoke instead of a call, branching to the
* handler bblock of the clause containing this bblock.
*/
g_assert (ec->flags == MONO_EXCEPTION_CLAUSE_NONE || ec->flags == MONO_EXCEPTION_CLAUSE_FINALLY || ec->flags == MONO_EXCEPTION_CLAUSE_FAULT);
tblock = cfg->cil_offset_to_bb [ec->handler_offset];
g_assert (tblock);
ctx->bblocks [tblock->block_num].invoke_target = TRUE;
ex_bb = get_bb (ctx, tblock);
noex_bb = gen_bb (ctx, "NOEX_BB");
/* Use an invoke */
lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, ex_bb, "");
builder = ctx->builder = create_builder (ctx);
LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
ctx->bblocks [bb->block_num].end_bblock = noex_bb;
}
}
if (!lcall) {
lcall = LLVMBuildCall (builder, callee, args, pindex, "");
ctx->builder = builder;
}
if (builder_ref)
*builder_ref = ctx->builder;
return lcall;
}
static LLVMValueRef
emit_load_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, LLVMValueRef base, const char *name, gboolean is_faulting, BarrierKind barrier)
{
const char *intrins_name;
LLVMValueRef args [16], res;
LLVMTypeRef addr_type;
gboolean use_intrinsics = TRUE;
#if LLVM_API_VERSION > 100
if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only) {
/* The llvm.mono.load/store intrinsics are not supported by this llvm version, emit an explicit null check instead */
LLVMValueRef cmp;
cmp = LLVMBuildICmp (*builder_ref, LLVMIntEQ, base, LLVMConstNull (LLVMTypeOf (base)), "");
emit_cond_system_exception (ctx, bb, "NullReferenceException", cmp);
*builder_ref = ctx->builder;
use_intrinsics = FALSE;
}
#endif
if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only && use_intrinsics) {
LLVMAtomicOrdering ordering;
switch (barrier) {
case LLVM_BARRIER_NONE:
ordering = LLVMAtomicOrderingNotAtomic;
break;
case LLVM_BARRIER_ACQ:
ordering = LLVMAtomicOrderingAcquire;
break;
case LLVM_BARRIER_SEQ:
ordering = LLVMAtomicOrderingSequentiallyConsistent;
break;
default:
g_assert_not_reached ();
break;
}
/*
* We handle loads which can fault by calling a mono specific intrinsic
* using an invoke, so they are handled properly inside try blocks.
* We can't use this outside clauses, since LLVM optimizes intrinsics which
* are marked with IntrReadArgMem.
*/
switch (size) {
case 1:
intrins_name = "llvm.mono.load.i8.p0i8";
break;
case 2:
intrins_name = "llvm.mono.load.i16.p0i16";
break;
case 4:
intrins_name = "llvm.mono.load.i32.p0i32";
break;
case 8:
intrins_name = "llvm.mono.load.i64.p0i64";
break;
default:
g_assert_not_reached ();
}
addr_type = LLVMTypeOf (addr);
if (addr_type == LLVMPointerType (LLVMDoubleType (), 0) || addr_type == LLVMPointerType (LLVMFloatType (), 0))
addr = LLVMBuildBitCast (*builder_ref, addr, LLVMPointerType (LLVMIntType (size * 8), 0), "");
args [0] = addr;
args [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
args [2] = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
args [3] = LLVMConstInt (LLVMInt32Type (), ordering, FALSE);
res = emit_call (ctx, bb, builder_ref, get_intrinsic (ctx, intrins_name), args, 4);
if (addr_type == LLVMPointerType (LLVMDoubleType (), 0))
res = LLVMBuildBitCast (*builder_ref, res, LLVMDoubleType (), "");
else if (addr_type == LLVMPointerType (LLVMFloatType (), 0))
res = LLVMBuildBitCast (*builder_ref, res, LLVMFloatType (), "");
return res;
} else {
LLVMValueRef res;
/*
* We emit volatile loads for loads which can fault, because otherwise
* LLVM will generate invalid code when encountering a load from a
* NULL address.
*/
if (barrier != LLVM_BARRIER_NONE)
res = mono_llvm_build_atomic_load (*builder_ref, addr, name, is_faulting, size, barrier);
else
res = mono_llvm_build_load (*builder_ref, addr, name, is_faulting);
/* Mark it with a custom metadata */
/*
if (is_faulting)
set_metadata_flag (res, "mono.faulting.load");
*/
return res;
}
}
static LLVMValueRef
emit_load (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, const char *name, gboolean is_faulting)
{
return emit_load_general (ctx, bb, builder_ref, size, addr, addr, name, is_faulting, LLVM_BARRIER_NONE);
}
static void
emit_store_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, LLVMValueRef base, gboolean is_faulting, BarrierKind barrier)
{
const char *intrins_name;
LLVMValueRef args [16];
gboolean use_intrinsics = TRUE;
#if LLVM_API_VERSION > 100
if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only) {
/* The llvm.mono.load/store intrinsics are not supported by this llvm version, emit an explicit null check instead */
LLVMValueRef cmp = LLVMBuildICmp (*builder_ref, LLVMIntEQ, base, LLVMConstNull (LLVMTypeOf (base)), "");
emit_cond_system_exception (ctx, bb, "NullReferenceException", cmp);
*builder_ref = ctx->builder;
use_intrinsics = FALSE;
}
#endif
if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only && use_intrinsics) {
LLVMAtomicOrdering ordering;
switch (barrier) {
case LLVM_BARRIER_NONE:
ordering = LLVMAtomicOrderingNotAtomic;
break;
case LLVM_BARRIER_REL:
ordering = LLVMAtomicOrderingRelease;
break;
case LLVM_BARRIER_SEQ:
ordering = LLVMAtomicOrderingSequentiallyConsistent;
break;
default:
g_assert_not_reached ();
break;
}
switch (size) {
case 1:
intrins_name = "llvm.mono.store.i8.p0i8";
break;
case 2:
intrins_name = "llvm.mono.store.i16.p0i16";
break;
case 4:
intrins_name = "llvm.mono.store.i32.p0i32";
break;
case 8:
intrins_name = "llvm.mono.store.i64.p0i64";
break;
default:
g_assert_not_reached ();
}
if (LLVMTypeOf (value) == LLVMDoubleType () || LLVMTypeOf (value) == LLVMFloatType ()) {
value = LLVMBuildBitCast (*builder_ref, value, LLVMIntType (size * 8), "");
addr = LLVMBuildBitCast (*builder_ref, addr, LLVMPointerType (LLVMIntType (size * 8), 0), "");
}
args [0] = value;
args [1] = addr;
args [2] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
args [3] = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
args [4] = LLVMConstInt (LLVMInt32Type (), ordering, FALSE);
emit_call (ctx, bb, builder_ref, get_intrinsic (ctx, intrins_name), args, 5);
} else {
if (barrier != LLVM_BARRIER_NONE)
mono_llvm_build_aligned_store (*builder_ref, value, addr, barrier, size);
else
mono_llvm_build_store (*builder_ref, value, addr, is_faulting, barrier);
}
}
static void
emit_store (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, LLVMValueRef base, gboolean is_faulting)
{
emit_store_general (ctx, bb, builder_ref, size, value, addr, base, is_faulting, LLVM_BARRIER_NONE);
}
/*
* emit_cond_system_exception:
*
* Emit code to throw the exception EXC_TYPE if the condition CMP is false.
* Might set the ctx exception.
*/
static void
emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp)
{
LLVMBasicBlockRef ex_bb, ex2_bb = NULL, noex_bb;
LLVMBuilderRef builder;
MonoClass *exc_class;
LLVMValueRef args [2];
LLVMValueRef callee;
gboolean no_pc = FALSE;
if (IS_TARGET_AMD64)
/* Some platforms don't require the pc argument */
no_pc = TRUE;
ex_bb = gen_bb (ctx, "EX_BB");
if (ctx->llvm_only)
ex2_bb = gen_bb (ctx, "EX2_BB");
noex_bb = gen_bb (ctx, "NOEX_BB");
LLVMBuildCondBr (ctx->builder, cmp, ex_bb, noex_bb);
exc_class = mono_class_load_from_name (mono_get_corlib (), "System", exc_type);
/* Emit exception throwing code */
ctx->builder = builder = create_builder (ctx);
LLVMPositionBuilderAtEnd (builder, ex_bb);
if (ctx->cfg->llvm_only) {
LLVMBuildBr (builder, ex2_bb);
ctx->builder = builder = create_builder (ctx);
LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
if (!strcmp (exc_type, "NullReferenceException")) {
static LLVMTypeRef sig;
if (!sig)
sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, "mini_llvmonly_throw_nullref_exception");
emit_call (ctx, bb, &builder, callee, NULL, 0);
} else {
static LLVMTypeRef sig;
if (!sig)
sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, "mono_llvm_throw_corlib_exception");
args [0] = LLVMConstInt (LLVMInt32Type (), m_class_get_type_token (exc_class) - MONO_TOKEN_TYPE_DEF, FALSE);
emit_call (ctx, bb, &builder, callee, args, 1);
}
LLVMBuildUnreachable (builder);
ctx->builder = builder = create_builder (ctx);
LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
ctx->bblocks [bb->block_num].end_bblock = noex_bb;
ctx->ex_index ++;
return;
}
callee = ctx->module->throw_corlib_exception;
if (!callee) {
LLVMTypeRef sig;
const char *icall_name;
if (no_pc)
sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
else
sig = LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), LLVMPointerType (LLVMInt8Type (), 0), FALSE);
icall_name = "llvm_throw_corlib_exception_abs_trampoline";
if (ctx->cfg->compile_aot) {
callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL, icall_name);
} else {
/*
* Differences between the LLVM/non-LLVM throw corlib exception trampoline:
* - On x86, LLVM generated code doesn't push the arguments
* - The trampoline takes the throw address as an arguments, not a pc offset.
*/
gpointer target = resolve_patch (ctx->cfg, MONO_PATCH_INFO_JIT_ICALL, icall_name);
callee = emit_jit_callee (ctx, "llvm_throw_corlib_exception_trampoline", sig, target);
#if LLVM_API_VERSION > 100
/*
* Make sure that ex_bb starts with the invoke, so the block address points to it, and not to the load
* added by emit_jit_callee ().
*/
ex2_bb = gen_bb (ctx, "EX2_BB");
LLVMBuildBr (builder, ex2_bb);
ex_bb = ex2_bb;
ctx->builder = builder = create_builder (ctx);
LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
#else
mono_memory_barrier ();
ctx->module->throw_corlib_exception = callee;
#endif
}
}
args [0] = LLVMConstInt (LLVMInt32Type (), m_class_get_type_token (exc_class) - MONO_TOKEN_TYPE_DEF, FALSE);
/*
* The LLVM mono branch contains changes so a block address can be passed as an
* argument to a call.
*/
if (no_pc) {
emit_call (ctx, bb, &builder, callee, args, 1);
} else {
args [1] = LLVMBlockAddress (ctx->lmethod, ex_bb);
emit_call (ctx, bb, &builder, callee, args, 2);
}
LLVMBuildUnreachable (builder);
ctx->builder = builder = create_builder (ctx);
LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
ctx->bblocks [bb->block_num].end_bblock = noex_bb;
ctx->ex_index ++;
return;
}
/*
* emit_args_to_vtype:
*
* Emit code to store the vtype in the arguments args to the address ADDRESS.
*/
static void
emit_args_to_vtype (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args)
{
int j, size, nslots;
size = mono_class_value_size (mono_class_from_mono_type_internal (t), NULL);
if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (t))) {
address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
}
if (ainfo->storage == LLVMArgAsFpArgs)
nslots = ainfo->nslots;
else
nslots = 2;
for (j = 0; j < nslots; ++j) {
LLVMValueRef index [2], addr, daddr;
int part_size = size > TARGET_SIZEOF_VOID_P ? TARGET_SIZEOF_VOID_P : size;
LLVMTypeRef part_type;
while (part_size != 1 && part_size != 2 && part_size != 4 && part_size < 8)
part_size ++;
if (ainfo->pair_storage [j] == LLVMArgNone)
continue;
switch (ainfo->pair_storage [j]) {
case LLVMArgInIReg: {
part_type = LLVMIntType (part_size * 8);
if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (t))) {
index [0] = LLVMConstInt (LLVMInt32Type (), j * TARGET_SIZEOF_VOID_P, FALSE);
addr = LLVMBuildGEP (builder, address, index, 1, "");
} else {
daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
addr = LLVMBuildGEP (builder, daddr, index, 1, "");
}
LLVMBuildStore (builder, convert (ctx, args [j], part_type), LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (part_type, 0), ""));
break;
}
case LLVMArgInFPReg: {
LLVMTypeRef arg_type;
if (ainfo->esize == 8)
arg_type = LLVMDoubleType ();
else
arg_type = LLVMFloatType ();
index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
addr = LLVMBuildGEP (builder, daddr, index, 1, "");
LLVMBuildStore (builder, args [j], addr);
break;
}
case LLVMArgNone:
break;
default:
g_assert_not_reached ();
}
size -= TARGET_SIZEOF_VOID_P;
}
}
/*
* emit_vtype_to_args:
*
* Emit code to load a vtype at address ADDRESS into scalar arguments. Store the arguments
* into ARGS, and the number of arguments into NARGS.
*/
static void
emit_vtype_to_args (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args, guint32 *nargs)
{
int pindex = 0;
int j, size, nslots;
LLVMTypeRef arg_type;
size = get_vtype_size (t);
if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (t)))
address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
if (ainfo->storage == LLVMArgAsFpArgs)
nslots = ainfo->nslots;
else
nslots = 2;
for (j = 0; j < nslots; ++j) {
LLVMValueRef index [2], addr, daddr;
int partsize = size > TARGET_SIZEOF_VOID_P ? TARGET_SIZEOF_VOID_P : size;
if (ainfo->pair_storage [j] == LLVMArgNone)
continue;
switch (ainfo->pair_storage [j]) {
case LLVMArgInIReg:
if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (t))) {
index [0] = LLVMConstInt (LLVMInt32Type (), j * TARGET_SIZEOF_VOID_P, FALSE);
addr = LLVMBuildGEP (builder, address, index, 1, "");
} else {
daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
addr = LLVMBuildGEP (builder, daddr, index, 1, "");
}
args [pindex ++] = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (LLVMIntType (partsize * 8), 0), ""), ""), IntPtrType ());
break;
case LLVMArgInFPReg:
if (ainfo->esize == 8)
arg_type = LLVMDoubleType ();
else
arg_type = LLVMFloatType ();
daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
addr = LLVMBuildGEP (builder, daddr, index, 1, "");
args [pindex ++] = LLVMBuildLoad (builder, addr, "");
break;
case LLVMArgNone:
break;
default:
g_assert_not_reached ();
}
size -= TARGET_SIZEOF_VOID_P;
}
*nargs = pindex;
}
static LLVMValueRef
build_alloca_llvm_type_name (EmitContext *ctx, LLVMTypeRef t, int align, const char *name)
{
/*
* Have to place all alloca's at the end of the entry bb, since otherwise they would
* get executed every time control reaches them.
*/
LLVMPositionBuilder (ctx->alloca_builder, get_bb (ctx, ctx->cfg->bb_entry), ctx->last_alloca);
ctx->last_alloca = mono_llvm_build_alloca (ctx->alloca_builder, t, NULL, align, name);
return ctx->last_alloca;
}
static LLVMValueRef
build_alloca_llvm_type (EmitContext *ctx, LLVMTypeRef t, int align)
{
return build_alloca_llvm_type_name (ctx, t, align, "");
}
static LLVMValueRef
build_alloca (EmitContext *ctx, MonoType *t)
{
MonoClass *k = mono_class_from_mono_type_internal (t);
int align;
g_assert (!mini_is_gsharedvt_variable_type (t));
if (MONO_CLASS_IS_SIMD (ctx->cfg, k))
align = 16;
else
align = mono_class_min_align (k);
/* Sometimes align is not a power of 2 */
while (mono_is_power_of_two (align) == -1)
align ++;
return build_alloca_llvm_type (ctx, type_to_llvm_type (ctx, t), align);
}
static LLVMValueRef
emit_gsharedvt_ldaddr (EmitContext *ctx, int vreg)
{
/*
* gsharedvt local.
* Compute the address of the local as gsharedvt_locals_var + gsharedvt_info_var->locals_offsets [idx].
*/
MonoCompile *cfg = ctx->cfg;
LLVMBuilderRef builder = ctx->builder;
LLVMValueRef offset, offset_var;
LLVMValueRef info_var = ctx->values [cfg->gsharedvt_info_var->dreg];
LLVMValueRef locals_var = ctx->values [cfg->gsharedvt_locals_var->dreg];
LLVMValueRef ptr;
char *name;
g_assert (info_var);
g_assert (locals_var);
int idx = cfg->gsharedvt_vreg_to_idx [vreg] - 1;
offset = LLVMConstInt (LLVMInt32Type (), MONO_STRUCT_OFFSET (MonoGSharedVtMethodRuntimeInfo, entries) + (idx * TARGET_SIZEOF_VOID_P), FALSE);
ptr = LLVMBuildAdd (builder, convert (ctx, info_var, IntPtrType ()), convert (ctx, offset, IntPtrType ()), "");
name = g_strdup_printf ("gsharedvt_local_%d_offset", vreg);
offset_var = LLVMBuildLoad (builder, convert (ctx, ptr, LLVMPointerType (LLVMInt32Type (), 0)), name);
return LLVMBuildAdd (builder, convert (ctx, locals_var, IntPtrType ()), convert (ctx, offset_var, IntPtrType ()), "");
}
/*
* Put the global into the 'llvm.used' array to prevent it from being optimized away.
*/
static void
mark_as_used (MonoLLVMModule *module, LLVMValueRef global)
{
if (!module->used)
module->used = g_ptr_array_sized_new (16);
g_ptr_array_add (module->used, global);
}
static void
emit_llvm_used (MonoLLVMModule *module)
{
LLVMModuleRef lmodule = module->lmodule;
LLVMTypeRef used_type;
LLVMValueRef used, *used_elem;
int i;
if (!module->used)
return;
used_type = LLVMArrayType (LLVMPointerType (LLVMInt8Type (), 0), module->used->len);
used = LLVMAddGlobal (lmodule, used_type, "llvm.used");
used_elem = g_new0 (LLVMValueRef, module->used->len);
for (i = 0; i < module->used->len; ++i)
used_elem [i] = LLVMConstBitCast ((LLVMValueRef)g_ptr_array_index (module->used, i), LLVMPointerType (LLVMInt8Type (), 0));
LLVMSetInitializer (used, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), used_elem, module->used->len));
LLVMSetLinkage (used, LLVMAppendingLinkage);
LLVMSetSection (used, "llvm.metadata");
}
/*
* emit_get_method:
*
* Emit a function mapping method indexes to their code
*/
static void
emit_get_method (MonoLLVMModule *module)
{
LLVMModuleRef lmodule = module->lmodule;
LLVMValueRef func, switch_ins, m;
LLVMBasicBlockRef entry_bb, fail_bb, bb, code_start_bb, code_end_bb;
LLVMBasicBlockRef *bbs = NULL;
LLVMTypeRef rtype;
LLVMBuilderRef builder = LLVMCreateBuilder ();
LLVMValueRef table = NULL;
char *name;
int i;
gboolean emit_table = FALSE;
#ifdef TARGET_WASM
/*
* Emit a table of functions instead of a switch statement,
* its very efficient on wasm. This might be usable on
* other platforms too.
*/
emit_table = TRUE;
#endif
rtype = LLVMPointerType (LLVMInt8Type (), 0);
if (emit_table) {
LLVMTypeRef table_type;
LLVMValueRef *table_elems;
char *table_name;
int table_len = module->max_method_idx + 1;
table_type = LLVMArrayType (rtype, table_len);
table_name = g_strdup_printf ("%s_method_table", module->assembly->aname.name);
table = LLVMAddGlobal (lmodule, table_type, table_name);
table_elems = g_new0 (LLVMValueRef, table_len);
for (i = 0; i < table_len; ++i) {
m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_lmethod, GINT_TO_POINTER (i));
if (m)
table_elems [i] = LLVMBuildBitCast (builder, m, rtype, "");
else
table_elems [i] = LLVMConstNull (rtype);
}
LLVMSetInitializer (table, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), table_elems, table_len));
}
/*
* Emit a switch statement. Emitting a table of function addresses is smaller/faster,
* but generating code seems safer.
*/
func = LLVMAddFunction (lmodule, module->get_method_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
LLVMSetLinkage (func, LLVMExternalLinkage);
LLVMSetVisibility (func, LLVMHiddenVisibility);
mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
module->get_method = func;
entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
/*
* Return llvm_code_start/llvm_code_end when called with -1/-2.
* Hopefully, the toolchain doesn't reorder these functions. If it does,
* then we will have to find another solution.
*/
name = g_strdup_printf ("BB_CODE_START");
code_start_bb = LLVMAppendBasicBlock (func, name);
g_free (name);
LLVMPositionBuilderAtEnd (builder, code_start_bb);
LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_start, rtype, ""));
name = g_strdup_printf ("BB_CODE_END");
code_end_bb = LLVMAppendBasicBlock (func, name);
g_free (name);
LLVMPositionBuilderAtEnd (builder, code_end_bb);
LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_end, rtype, ""));
if (emit_table) {
/*
* switch (index) {
* case -1: return code_start;
* case -2: return code_end;
* default: return method_table [index];
*/
LLVMBasicBlockRef default_bb = LLVMAppendBasicBlock (func, "DEFAULT");
LLVMPositionBuilderAtEnd (builder, default_bb);
LLVMValueRef base = table;
LLVMValueRef indexes [2];
indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
indexes [1] = LLVMGetParam (func, 0);
LLVMValueRef addr = LLVMBuildGEP (builder, base, indexes, 2, "");
LLVMValueRef res = mono_llvm_build_load (builder, addr, "", FALSE);
LLVMBuildRet (builder, res);
LLVMPositionBuilderAtEnd (builder, entry_bb);
switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), default_bb, 0);
LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -1, FALSE), code_start_bb);
LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -2, FALSE), code_end_bb);
} else {
bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
for (i = 0; i < module->max_method_idx + 1; ++i) {
name = g_strdup_printf ("BB_%d", i);
bb = LLVMAppendBasicBlock (func, name);
g_free (name);
bbs [i] = bb;
LLVMPositionBuilderAtEnd (builder, bb);
m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_lmethod, GINT_TO_POINTER (i));
if (m)
LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
else
LLVMBuildRet (builder, LLVMConstNull (rtype));
}
fail_bb = LLVMAppendBasicBlock (func, "FAIL");
LLVMPositionBuilderAtEnd (builder, fail_bb);
LLVMBuildRet (builder, LLVMConstNull (rtype));
LLVMPositionBuilderAtEnd (builder, entry_bb);
switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -1, FALSE), code_start_bb);
LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -2, FALSE), code_end_bb);
for (i = 0; i < module->max_method_idx + 1; ++i) {
LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
}
}
mark_as_used (module, func);
LLVMDisposeBuilder (builder);
}
/*
* emit_get_unbox_tramp:
*
* Emit a function mapping method indexes to their unbox trampoline
*/
static void
emit_get_unbox_tramp (MonoLLVMModule *module)
{
LLVMModuleRef lmodule = module->lmodule;
LLVMValueRef func, switch_ins, m;
LLVMBasicBlockRef entry_bb, fail_bb, bb;
LLVMBasicBlockRef *bbs;
LLVMTypeRef rtype;
LLVMBuilderRef builder = LLVMCreateBuilder ();
char *name;
int i;
gboolean emit_table = FALSE;
/* Similar to emit_get_method () */
#ifndef TARGET_WATCHOS
emit_table = TRUE;
#endif
rtype = LLVMPointerType (LLVMInt8Type (), 0);
if (emit_table) {
// About 10% of methods have an unbox tramp, so emit a table of indexes for them
// that the runtime can search using a binary search
int len = 0;
for (i = 0; i < module->max_method_idx + 1; ++i) {
m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
if (m)
len ++;
}
LLVMTypeRef table_type, elemtype;
LLVMValueRef *table_elems;
LLVMValueRef table;
char *table_name;
int table_len;
int elemsize;
table_len = len;
elemsize = module->max_method_idx < 65000 ? 2 : 4;
// The index table
elemtype = elemsize == 2 ? LLVMInt16Type () : LLVMInt32Type ();
table_type = LLVMArrayType (elemtype, table_len);
table_name = g_strdup_printf ("%s_unbox_tramp_indexes", module->assembly->aname.name);
table = LLVMAddGlobal (lmodule, table_type, table_name);
table_elems = g_new0 (LLVMValueRef, table_len);
int idx = 0;
for (i = 0; i < module->max_method_idx + 1; ++i) {
m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
if (m)
table_elems [idx ++] = LLVMConstInt (elemtype, i, FALSE);
}
LLVMSetInitializer (table, LLVMConstArray (elemtype, table_elems, table_len));
module->unbox_tramp_indexes = table;
// The trampoline table
elemtype = rtype;
table_type = LLVMArrayType (elemtype, table_len);
table_name = g_strdup_printf ("%s_unbox_trampolines", module->assembly->aname.name);
table = LLVMAddGlobal (lmodule, table_type, table_name);
table_elems = g_new0 (LLVMValueRef, table_len);
idx = 0;
for (i = 0; i < module->max_method_idx + 1; ++i) {
m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
if (m)
table_elems [idx ++] = LLVMBuildBitCast (builder, m, rtype, "");
}
LLVMSetInitializer (table, LLVMConstArray (elemtype, table_elems, table_len));
module->unbox_trampolines = table;
module->unbox_tramp_num = table_len;
module->unbox_tramp_elemsize = elemsize;
return;
}
func = LLVMAddFunction (lmodule, module->get_unbox_tramp_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
LLVMSetLinkage (func, LLVMExternalLinkage);
LLVMSetVisibility (func, LLVMHiddenVisibility);
mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
module->get_unbox_tramp = func;
entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
for (i = 0; i < module->max_method_idx + 1; ++i) {
m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
if (!m)
continue;
name = g_strdup_printf ("BB_%d", i);
bb = LLVMAppendBasicBlock (func, name);
g_free (name);
bbs [i] = bb;
LLVMPositionBuilderAtEnd (builder, bb);
LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
}
fail_bb = LLVMAppendBasicBlock (func, "FAIL");
LLVMPositionBuilderAtEnd (builder, fail_bb);
LLVMBuildRet (builder, LLVMConstNull (rtype));
LLVMPositionBuilderAtEnd (builder, entry_bb);
switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
for (i = 0; i < module->max_method_idx + 1; ++i) {
m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
if (!m)
continue;
LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
}
mark_as_used (module, func);
LLVMDisposeBuilder (builder);
}
/* Add a function to mark the beginning of LLVM code */
static void
emit_llvm_code_start (MonoLLVMModule *module)
{
LLVMModuleRef lmodule = module->lmodule;
LLVMValueRef func;
LLVMBasicBlockRef entry_bb;
LLVMBuilderRef builder;
func = LLVMAddFunction (lmodule, "llvm_code_start", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
LLVMSetLinkage (func, LLVMInternalLinkage);
mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
module->code_start = func;
entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
builder = LLVMCreateBuilder ();
LLVMPositionBuilderAtEnd (builder, entry_bb);
LLVMBuildRetVoid (builder);
LLVMDisposeBuilder (builder);
}
static LLVMValueRef
emit_init_icall_wrapper (MonoLLVMModule *module, MonoAotInitSubtype subtype)
{
LLVMModuleRef lmodule = module->lmodule;
LLVMValueRef func, indexes [2], got_entry_addr, args [16], callee;
LLVMBasicBlockRef entry_bb;
LLVMBuilderRef builder;
LLVMTypeRef sig;
MonoJumpInfo *ji;
int got_offset;
const char *wrapper_name = mono_marshal_get_aot_init_wrapper_name (subtype);
char *name = g_strdup_printf ("%s%s", module->global_prefix, wrapper_name);
const char *icall_name = NULL;
switch (subtype) {
case AOT_INIT_METHOD:
icall_name = "mini_llvm_init_method";
func = LLVMAddFunction (lmodule, name, LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE));
sig = LLVMFunctionType2 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), FALSE);
break;
case AOT_INIT_METHOD_GSHARED_MRGCTX:
icall_name = "mini_llvm_init_gshared_method_mrgctx"; // Deliberate fall-through
case AOT_INIT_METHOD_GSHARED_VTABLE:
/* mrgctx/vtable */
if (!icall_name)
icall_name = "mini_llvm_init_gshared_method_vtable";
func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), IntPtrType (), FALSE));
sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), IntPtrType (), FALSE);
break;
case AOT_INIT_METHOD_GSHARED_THIS:
icall_name = "mini_llvm_init_gshared_method_this";
func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), ObjRefType (), FALSE));
sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), ObjRefType (), FALSE);
break;
default:
g_assert_not_reached ();
}
g_assert (icall_name);
LLVMSetLinkage (func, LLVMInternalLinkage);
mono_llvm_add_func_attr (func, LLVM_ATTR_NO_INLINE);
// FIXME? Using this with mono debug info causes unwind.c to explode when
// parsing some of these registers saved by this call. Can't unwind through it.
// Not an issue with llvmonly because it doesn't use that DWARF
if (module->llvm_only)
set_preserveall_cc (func);
else
LLVMSetFunctionCallConv (func, LLVMMono1CallConv);
entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
builder = LLVMCreateBuilder ();
LLVMPositionBuilderAtEnd (builder, entry_bb);
/* get_aotconst */
ji = g_new0 (MonoJumpInfo, 1);
ji->type = MONO_PATCH_INFO_AOT_MODULE;
ji = mono_aot_patch_info_dup (ji);
got_offset = mono_aot_get_got_offset (ji);
module->max_got_offset = MAX (module->max_got_offset, got_offset);
indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
args [0] = LLVMBuildPtrToInt (builder, LLVMBuildLoad (builder, got_entry_addr, ""), IntPtrType (), "");
args [1] = LLVMGetParam (func, 0);
if (subtype)
args [2] = LLVMGetParam (func, 1);
ji = g_new0 (MonoJumpInfo, 1);
ji->type = MONO_PATCH_INFO_JIT_ICALL;
ji->data.name = icall_name;
ji = mono_aot_patch_info_dup (ji);
got_offset = mono_aot_get_got_offset (ji);
module->max_got_offset = MAX (module->max_got_offset, got_offset);
indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
callee = LLVMBuildLoad (builder, got_entry_addr, "");
callee = LLVMBuildBitCast (builder, callee, LLVMPointerType (sig, 0), "");
LLVMBuildCall (builder, callee, args, LLVMCountParamTypes (sig), "");
// Set the inited flag
indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
indexes [1] = LLVMGetParam (func, 0);
LLVMBuildStore (builder, LLVMConstInt (LLVMInt8Type (), 1, FALSE), LLVMBuildGEP (builder, module->inited_var, indexes, 2, ""));
LLVMBuildRetVoid (builder);
LLVMVerifyFunction(func, LLVMAbortProcessAction);
LLVMDisposeBuilder (builder);
return func;
}
/*
* Emit wrappers around the C icalls used to initialize llvm methods, to
* make the calling code smaller and to enable usage of the llvm
* PreserveAll calling convention.
*/
static void
emit_init_icall_wrappers (MonoLLVMModule *module)
{
module->init_method = emit_init_icall_wrapper (module, AOT_INIT_METHOD);
module->init_method_gshared_mrgctx = emit_init_icall_wrapper (module, AOT_INIT_METHOD_GSHARED_MRGCTX);
module->init_method_gshared_this = emit_init_icall_wrapper (module, AOT_INIT_METHOD_GSHARED_THIS);
module->init_method_gshared_vtable = emit_init_icall_wrapper (module, AOT_INIT_METHOD_GSHARED_VTABLE);
}
static LLVMValueRef
get_init_icall_wrapper (MonoLLVMModule *module, MonoAotInitSubtype subtype)
{
switch (subtype) {
case AOT_INIT_METHOD:
return module->init_method;
case AOT_INIT_METHOD_GSHARED_MRGCTX:
return module->init_method_gshared_mrgctx;
case AOT_INIT_METHOD_GSHARED_THIS:
return module->init_method_gshared_this;
case AOT_INIT_METHOD_GSHARED_VTABLE:
return module->init_method_gshared_vtable;
default:
g_assert_not_reached ();
}
}
static void
emit_gc_safepoint_poll (MonoLLVMModule *module)
{
LLVMModuleRef lmodule = module->lmodule;
LLVMValueRef func, indexes [2], got_entry_addr, flag_addr, val_ptr, callee, val, cmp;
LLVMBasicBlockRef entry_bb, poll_bb, exit_bb;
LLVMBuilderRef builder;
LLVMTypeRef sig;
MonoJumpInfo *ji;
int got_offset;
sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
func = mono_llvm_get_or_insert_gc_safepoint_poll (lmodule);
mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
LLVMSetLinkage (func, LLVMWeakODRLinkage);
// set_preserveall_cc (func);
entry_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.entry");
poll_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.poll");
exit_bb = LLVMAppendBasicBlock (func, "gc.safepoint_poll.exit");
builder = LLVMCreateBuilder ();
/* entry: */
LLVMPositionBuilderAtEnd (builder, entry_bb);
/* get_aotconst */
ji = g_new0 (MonoJumpInfo, 1);
ji->type = MONO_PATCH_INFO_GC_SAFE_POINT_FLAG;
ji = mono_aot_patch_info_dup (ji);
got_offset = mono_aot_get_got_offset (ji);
module->max_got_offset = MAX (module->max_got_offset, got_offset);
indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
flag_addr = LLVMBuildLoad (builder, got_entry_addr, "");
val_ptr = LLVMBuildLoad (builder, flag_addr, "");
val = LLVMBuildPtrToInt (builder, val_ptr, IntPtrType (), "");
cmp = LLVMBuildICmp (builder, LLVMIntEQ, val, LLVMConstNull (LLVMTypeOf (val)), "");
LLVMBuildCondBr (builder, cmp, exit_bb, poll_bb);
/* poll: */
LLVMPositionBuilderAtEnd(builder, poll_bb);
ji = g_new0 (MonoJumpInfo, 1);
ji->type = MONO_PATCH_INFO_JIT_ICALL;
ji->data.name = "mono_threads_state_poll";
ji = mono_aot_patch_info_dup (ji);
got_offset = mono_aot_get_got_offset (ji);
module->max_got_offset = MAX (module->max_got_offset, got_offset);
indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
callee = LLVMBuildLoad (builder, got_entry_addr, "");
callee = LLVMBuildBitCast (builder, callee, LLVMPointerType (sig, 0), "");
LLVMBuildCall (builder, callee, NULL, 0, "");
LLVMBuildBr(builder, exit_bb);
/* exit: */
LLVMPositionBuilderAtEnd(builder, exit_bb);
LLVMBuildRetVoid (builder);
LLVMVerifyFunction(func, LLVMAbortProcessAction);
LLVMDisposeBuilder (builder);
}
static void
emit_llvm_code_end (MonoLLVMModule *module)
{
LLVMModuleRef lmodule = module->lmodule;
LLVMValueRef func;
LLVMBasicBlockRef entry_bb;
LLVMBuilderRef builder;
func = LLVMAddFunction (lmodule, "llvm_code_end", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
LLVMSetLinkage (func, LLVMInternalLinkage);
mono_llvm_add_func_attr (func, LLVM_ATTR_NO_UNWIND);
module->code_end = func;
entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
builder = LLVMCreateBuilder ();
LLVMPositionBuilderAtEnd (builder, entry_bb);
LLVMBuildRetVoid (builder);
LLVMDisposeBuilder (builder);
}
static void
emit_div_check (EmitContext *ctx, LLVMBuilderRef builder, MonoBasicBlock *bb, MonoInst *ins, LLVMValueRef lhs, LLVMValueRef rhs)
{
gboolean need_div_check = ctx->cfg->backend->need_div_check;
if (bb->region)
/* LLVM doesn't know that these can throw an exception since they are not called through an intrinsic */
need_div_check = TRUE;
if (!need_div_check)
return;
switch (ins->opcode) {
case OP_IDIV:
case OP_LDIV:
case OP_IREM:
case OP_LREM:
case OP_IDIV_UN:
case OP_LDIV_UN:
case OP_IREM_UN:
case OP_LREM_UN:
case OP_IDIV_IMM:
case OP_LDIV_IMM:
case OP_IREM_IMM:
case OP_LREM_IMM:
case OP_IDIV_UN_IMM:
case OP_LDIV_UN_IMM:
case OP_IREM_UN_IMM:
case OP_LREM_UN_IMM: {
LLVMValueRef cmp;
gboolean is_signed = (ins->opcode == OP_IDIV || ins->opcode == OP_LDIV || ins->opcode == OP_IREM || ins->opcode == OP_LREM ||
ins->opcode == OP_IDIV_IMM || ins->opcode == OP_LDIV_IMM || ins->opcode == OP_IREM_IMM || ins->opcode == OP_LREM_IMM);
cmp = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), 0, FALSE), "");
emit_cond_system_exception (ctx, bb, "DivideByZeroException", cmp);
if (!ctx_ok (ctx))
break;
builder = ctx->builder;
/* b == -1 && a == 0x80000000 */
if (is_signed) {
LLVMValueRef c = (LLVMTypeOf (lhs) == LLVMInt32Type ()) ? LLVMConstInt (LLVMTypeOf (lhs), 0x80000000, FALSE) : LLVMConstInt (LLVMTypeOf (lhs), 0x8000000000000000LL, FALSE);
LLVMValueRef cond1 = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), -1, FALSE), "");
LLVMValueRef cond2 = LLVMBuildICmp (builder, LLVMIntEQ, lhs, c, "");
cmp = LLVMBuildICmp (builder, LLVMIntEQ, LLVMBuildAnd (builder, cond1, cond2, ""), LLVMConstInt (LLVMInt1Type (), 1, FALSE), "");
emit_cond_system_exception (ctx, bb, "OverflowException", cmp);
if (!ctx_ok (ctx))
break;
builder = ctx->builder;
}
break;
}
default:
break;
}
}
/*
* emit_init_method:
*
* Emit code to initialize the GOT slots used by the method.
*/
static void
emit_init_method (EmitContext *ctx)
{
LLVMValueRef indexes [16], args [16], callee;
LLVMValueRef inited_var, cmp, call;
LLVMBasicBlockRef inited_bb, notinited_bb;
LLVMBuilderRef builder = ctx->builder;
MonoCompile *cfg = ctx->cfg;
ctx->module->max_inited_idx = MAX (ctx->module->max_inited_idx, cfg->method_index);
indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
indexes [1] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, FALSE);
inited_var = LLVMBuildLoad (builder, LLVMBuildGEP (builder, ctx->module->inited_var, indexes, 2, ""), "is_inited");
//WASM doesn't support the "llvm.expect.i8" intrinsic
#ifndef TARGET_WASM
args [0] = inited_var;
args [1] = LLVMConstInt (LLVMInt8Type (), 1, FALSE);
inited_var = LLVMBuildCall (ctx->builder, get_intrinsic (ctx, "llvm.expect.i8"), args, 2, "");
#endif
cmp = LLVMBuildICmp (builder, LLVMIntEQ, inited_var, LLVMConstInt (LLVMTypeOf (inited_var), 0, FALSE), "");
inited_bb = ctx->inited_bb;
notinited_bb = gen_bb (ctx, "NOTINITED_BB");
ctx->cfg->llvmonly_init_cond = LLVMBuildCondBr (ctx->builder, cmp, notinited_bb, inited_bb);
builder = ctx->builder = create_builder (ctx);
LLVMPositionBuilderAtEnd (ctx->builder, notinited_bb);
// FIXME: Cache
if (ctx->rgctx_arg && ((cfg->method->is_inflated && mono_method_get_context (cfg->method)->method_inst) ||
mini_method_is_default_method (cfg->method))) {
args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
callee = ctx->module->init_method_gshared_mrgctx;
call = LLVMBuildCall (builder, callee, args, 2, "");
} else if (ctx->rgctx_arg) {
/* A vtable is passed as the rgctx argument */
args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
callee = ctx->module->init_method_gshared_vtable;
call = LLVMBuildCall (builder, callee, args, 2, "");
} else if (cfg->gshared) {
args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
args [1] = convert (ctx, ctx->this_arg, ObjRefType ());
callee = ctx->module->init_method_gshared_this;
call = LLVMBuildCall (builder, callee, args, 2, "");
} else {
args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
callee = ctx->module->init_method;
call = LLVMBuildCall (builder, callee, args, 1, "");
}
/*
* This enables llvm to keep arguments in their original registers/
* scratch registers, since the call will not clobber them.
*/
if (ctx->llvm_only)
set_call_preserveall_cc (call);
else
LLVMSetInstructionCallConv (call, LLVMMono1CallConv);
LLVMBuildBr (builder, inited_bb);
ctx->bblocks [cfg->bb_entry->block_num].end_bblock = inited_bb;
builder = ctx->builder = create_builder (ctx);
LLVMPositionBuilderAtEnd (ctx->builder, inited_bb);
}
static void
emit_unbox_tramp (EmitContext *ctx, const char *method_name, LLVMTypeRef method_type, LLVMValueRef method, int method_index)
{
/*
* Emit unbox trampoline using a tailcall
*/
LLVMValueRef tramp, call, *args;
LLVMBuilderRef builder;
LLVMBasicBlockRef lbb;
LLVMCallInfo *linfo;
char *tramp_name;
int i, nargs;
tramp_name = g_strdup_printf ("ut_%s", method_name);
tramp = LLVMAddFunction (ctx->module->lmodule, tramp_name, method_type);
LLVMSetLinkage (tramp, LLVMInternalLinkage);
mono_llvm_add_func_attr (tramp, LLVM_ATTR_OPTIMIZE_FOR_SIZE);
//mono_llvm_add_func_attr (tramp, LLVM_ATTR_NO_UNWIND);
linfo = ctx->linfo;
// FIXME: Reduce code duplication with mono_llvm_compile_method () etc.
if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
mono_llvm_add_param_attr (LLVMGetParam (tramp, ctx->rgctx_arg_pindex), LLVM_ATTR_IN_REG);
if (ctx->cfg->vret_addr) {
LLVMSetValueName (LLVMGetParam (tramp, linfo->vret_arg_pindex), "vret");
if (linfo->ret.storage == LLVMArgVtypeByRef) {
mono_llvm_add_param_attr (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVM_ATTR_STRUCT_RET);
mono_llvm_add_param_attr (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVM_ATTR_NO_ALIAS);
}
}
lbb = LLVMAppendBasicBlock (tramp, "");
builder = LLVMCreateBuilder ();
LLVMPositionBuilderAtEnd (builder, lbb);
nargs = LLVMCountParamTypes (method_type);
args = g_new0 (LLVMValueRef, nargs);
for (i = 0; i < nargs; ++i) {
args [i] = LLVMGetParam (tramp, i);
if (i == ctx->this_arg_pindex) {
LLVMTypeRef arg_type = LLVMTypeOf (args [i]);
args [i] = LLVMBuildPtrToInt (builder, args [i], IntPtrType (), "");
args [i] = LLVMBuildAdd (builder, args [i], LLVMConstInt (IntPtrType (), MONO_ABI_SIZEOF (MonoObject), FALSE), "");
args [i] = LLVMBuildIntToPtr (builder, args [i], arg_type, "");
}
}
call = LLVMBuildCall (builder, method, args, nargs, "");
if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
mono_llvm_add_instr_attr (call, 1 + ctx->rgctx_arg_pindex, LLVM_ATTR_IN_REG);
if (linfo->ret.storage == LLVMArgVtypeByRef)
mono_llvm_add_instr_attr (call, 1 + linfo->vret_arg_pindex, LLVM_ATTR_STRUCT_RET);
// FIXME: This causes assertions in clang
//mono_llvm_set_must_tailcall (call);
if (LLVMGetReturnType (method_type) == LLVMVoidType ())
LLVMBuildRetVoid (builder);
else
LLVMBuildRet (builder, call);
g_hash_table_insert (ctx->module->idx_to_unbox_tramp, GINT_TO_POINTER (method_index), tramp);
LLVMDisposeBuilder (builder);
}
/*
* emit_entry_bb:
*
* Emit code to load/convert arguments.
*/
static void
emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder)
{
int i, j, pindex;
MonoCompile *cfg = ctx->cfg;
MonoMethodSignature *sig = ctx->sig;
LLVMCallInfo *linfo = ctx->linfo;
MonoBasicBlock *bb;
char **names;
LLVMBuilderRef old_builder = ctx->builder;
ctx->builder = builder;
ctx->alloca_builder = create_builder (ctx);
/*
* Handle indirect/volatile variables by allocating memory for them
* using 'alloca', and storing their address in a temporary.
*/
for (i = 0; i < cfg->num_varinfo; ++i) {
MonoInst *var = cfg->varinfo [i];
LLVMTypeRef vtype;
if (var->opcode == OP_GSHAREDVT_LOCAL || var->opcode == OP_GSHAREDVT_ARG_REGOFFSET) {
} else if (var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) || (mini_type_is_vtype (var->inst_vtype) && !MONO_CLASS_IS_SIMD (ctx->cfg, var->klass))) {
vtype = type_to_llvm_type (ctx, var->inst_vtype);
if (!ctx_ok (ctx))
return;
/* Could be already created by an OP_VPHI */
if (!ctx->addresses [var->dreg]) {
ctx->addresses [var->dreg] = build_alloca (ctx, var->inst_vtype);
//LLVMSetValueName (ctx->addresses [var->dreg], g_strdup_printf ("vreg_loc_%d", var->dreg));
}
ctx->vreg_cli_types [var->dreg] = var->inst_vtype;
}
}
names = g_new (char *, sig->param_count);
mono_method_get_param_names (cfg->method, (const char **) names);
for (i = 0; i < sig->param_count; ++i) {
LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
int reg = cfg->args [i + sig->hasthis]->dreg;
char *name;
pindex = ainfo->pindex;
switch (ainfo->storage) {
case LLVMArgVtypeInReg:
case LLVMArgAsFpArgs: {
LLVMValueRef args [8];
int j;
pindex += ainfo->ndummy_fpargs;
/* The argument is received as a set of int/fp arguments, store them into the real argument */
memset (args, 0, sizeof (args));
if (ainfo->storage == LLVMArgVtypeInReg) {
args [0] = LLVMGetParam (ctx->lmethod, pindex);
if (ainfo->pair_storage [1] != LLVMArgNone)
args [1] = LLVMGetParam (ctx->lmethod, pindex + 1);
} else {
g_assert (ainfo->nslots <= 8);
for (j = 0; j < ainfo->nslots; ++j)
args [j] = LLVMGetParam (ctx->lmethod, pindex + j);
}
ctx->addresses [reg] = build_alloca (ctx, ainfo->type);
emit_args_to_vtype (ctx, builder, ainfo->type, ctx->addresses [reg], ainfo, args);
if (ainfo->storage == LLVMArgVtypeInReg && MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (ainfo->type))) {
/* Treat these as normal values */
ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
}
break;
}
case LLVMArgVtypeByVal: {
ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type_internal (ainfo->type))) {
/* Treat these as normal values */
ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
}
break;
}
case LLVMArgVtypeByRef: {
/* The argument is passed by ref */
ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
break;
}
case LLVMArgAsIArgs: {
LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
int size;
/* The argument is received as an array of ints, store it into the real argument */
ctx->addresses [reg] = build_alloca (ctx, ainfo->type);
size = mono_class_value_size (mono_class_from_mono_type_internal (ainfo->type), NULL);
if (size == 0) {
} else if (size < TARGET_SIZEOF_VOID_P) {
/* The upper bits of the registers might not be valid */
LLVMValueRef val = LLVMBuildExtractValue (builder, arg, 0, "");
LLVMValueRef dest = convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMIntType (size * 8), 0));
LLVMBuildStore (ctx->builder, LLVMBuildTrunc (builder, val, LLVMIntType (size * 8), ""), dest);
} else {
LLVMBuildStore (ctx->builder, arg, convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMTypeOf (arg), 0)));
}
break;
}
case LLVMArgVtypeAsScalar:
g_assert_not_reached ();
break;
case LLVMArgGsharedvtFixed: {
/* These are non-gsharedvt arguments passed by ref, the rest of the IR treats them as scalars */
LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
if (names [i])
name = g_strdup_printf ("arg_%s", names [i]);
else
name = g_strdup_printf ("arg_%d", i);
ctx->values [reg] = LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), name);
break;
}
case LLVMArgGsharedvtFixedVtype: {
LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
if (names [i])
name = g_strdup_printf ("vtype_arg_%s", names [i]);
else
name = g_strdup_printf ("vtype_arg_%d", i);
/* Non-gsharedvt vtype argument passed by ref, the rest of the IR treats it as a vtype */
g_assert (ctx->addresses [reg]);
LLVMSetValueName (ctx->addresses [reg], name);
LLVMBuildStore (builder, LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), ""), ctx->addresses [reg]);
break;
}
case LLVMArgGsharedvtVariable:
/* The IR treats these as variables with addresses */
ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
break;
default:
ctx->values [reg] = convert_full (ctx, ctx->values [reg], llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, ainfo->type)), type_is_unsigned (ctx, ainfo->type));
break;
}
}
g_free (names);
if (cfg->vret_addr)
emit_volatile_store (ctx, cfg->vret_addr->dreg);
if (sig->hasthis)
emit_volatile_store (ctx, cfg->args [0]->dreg);
for (i = 0; i < sig->param_count; ++i)
if (!mini_type_is_vtype (sig->params [i]))
emit_volatile_store (ctx, cfg->args [i + sig->hasthis]->dreg);
if (sig->hasthis && !cfg->rgctx_var && cfg->gshared) {
LLVMValueRef this_alloc;
/*
* The exception handling code needs the location where the this argument was
* stored for gshared methods. We create a separate alloca to hold it, and mark it
* with the "mono.this" custom metadata to tell llvm that it needs to save its
* location into the LSDA.
*/
this_alloc = mono_llvm_build_alloca (builder, ThisType (), LLVMConstInt (LLVMInt32Type (), 1, FALSE), 0, "");
/* This volatile store will keep the alloca alive */
mono_llvm_build_store (builder, ctx->values [cfg->args [0]->dreg], this_alloc, TRUE, LLVM_BARRIER_NONE);
set_metadata_flag (this_alloc, "mono.this");
}
if (cfg->rgctx_var) {
LLVMValueRef rgctx_alloc, store;
/*
* We handle the rgctx arg similarly to the this pointer.
*/
g_assert (ctx->addresses [cfg->rgctx_var->dreg]);
rgctx_alloc = ctx->addresses [cfg->rgctx_var->dreg];
/* This volatile store will keep the alloca alive */
store = mono_llvm_build_store (builder, convert (ctx, ctx->rgctx_arg, IntPtrType ()), rgctx_alloc, TRUE, LLVM_BARRIER_NONE);
set_metadata_flag (rgctx_alloc, "mono.this");
}
/* Initialize the method if needed */
if (cfg->compile_aot && !ctx->module->llvm_disable_self_init) {
/* Emit a location for the initialization code */
ctx->init_bb = gen_bb (ctx, "INIT_BB");
ctx->inited_bb = gen_bb (ctx, "INITED_BB");
LLVMBuildBr (ctx->builder, ctx->init_bb);
builder = ctx->builder = create_builder (ctx);
LLVMPositionBuilderAtEnd (ctx->builder, ctx->inited_bb);
ctx->bblocks [cfg->bb_entry->block_num].end_bblock = ctx->inited_bb;
}
/* Compute nesting between clauses */
ctx->nested_in = (GSList**)mono_mempool_alloc0 (cfg->mempool, sizeof (GSList*) * cfg->header->num_clauses);
for (i = 0; i < cfg->header->num_clauses; ++i) {
for (j = 0; j < cfg->header->num_clauses; ++j) {
MonoExceptionClause *clause1 = &cfg->header->clauses [i];
MonoExceptionClause *clause2 = &cfg->header->clauses [j];
if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset)
ctx->nested_in [i] = g_slist_prepend_mempool (cfg->mempool, ctx->nested_in [i], GINT_TO_POINTER (j));
}
}
/*
* For finally clauses, create an indicator variable telling OP_ENDFINALLY whenever
* it needs to continue normally, or return back to the exception handling system.
*/
for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
int clause_index;
char name [128];
if (!(bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER)))
continue;
clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
g_hash_table_insert (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)), bb);
g_hash_table_insert (ctx->clause_to_handler, GINT_TO_POINTER (clause_index), bb);
if (bb->in_scount == 0) {
LLVMValueRef val;
sprintf (name, "finally_ind_bb%d", bb->block_num);
val = LLVMBuildAlloca (builder, LLVMInt32Type (), name);
LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), val);
ctx->bblocks [bb->block_num].finally_ind = val;
} else {
/* Create a variable to hold the exception var */
if (!ctx->ex_var)
ctx->ex_var = LLVMBuildAlloca (builder, ObjRefType (), "exvar");
}
/*
* Create a new bblock which CALL_HANDLER/landing pads can branch to, because branching to the
* LLVM bblock containing a landing pad causes problems for the
* LLVM optimizer passes.
*/
sprintf (name, "BB%d_CALL_HANDLER_TARGET", bb->block_num);
ctx->bblocks [bb->block_num].call_handler_target_bb = LLVMAppendBasicBlock (ctx->lmethod, name);
}
ctx->builder = old_builder;
}
static gboolean
needs_extra_arg (EmitContext *ctx, MonoMethod *method)
{
WrapperInfo *info = NULL;
/*
* When targeting wasm, the caller and callee signature has to match exactly. This means
* that every method which can be called indirectly need an extra arg since the caller
* will call it through an ftnptr and will pass an extra arg.
*/
if (!ctx->cfg->llvm_only || !ctx->emit_dummy_arg)
return FALSE;
if (method->wrapper_type)
info = mono_marshal_get_wrapper_info (method);
switch (method->wrapper_type) {
case MONO_WRAPPER_OTHER:
if (info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_IN_SIG || info->subtype == WRAPPER_SUBTYPE_GSHAREDVT_OUT_SIG)
/* Already have an explicit extra arg */
return FALSE;
break;
case MONO_WRAPPER_MANAGED_TO_NATIVE:
if (strstr (method->name, "icall_wrapper"))
/* These are JIT icall wrappers which are only called from JITted code directly */
return FALSE;
/* Normal icalls can be virtual methods which need an extra arg */
break;
case MONO_WRAPPER_RUNTIME_INVOKE:
case MONO_WRAPPER_ALLOC:
case MONO_WRAPPER_CASTCLASS:
case MONO_WRAPPER_WRITE_BARRIER:
return FALSE;
case MONO_WRAPPER_STELEMREF:
if (info->subtype != WRAPPER_SUBTYPE_VIRTUAL_STELEMREF)
return FALSE;
break;
case MONO_WRAPPER_MANAGED_TO_MANAGED:
if (info->subtype == WRAPPER_SUBTYPE_STRING_CTOR)
return FALSE;
break;
default:
break;
}
if (method->string_ctor)
return FALSE;
/* These are called from gsharedvt code with an indirect call which doesn't pass an extra arg */
if (method->klass == mono_get_string_class () && (strstr (method->name, "memcpy") || strstr (method->name, "bzero")))
return FALSE;
return TRUE;
}
static inline gboolean
is_supported_callconv (EmitContext *ctx, MonoCallInst *call)
{
#if defined(TARGET_WIN32) && defined(TARGET_AMD64)
gboolean result = (call->signature->call_convention == MONO_CALL_DEFAULT) ||
(call->signature->call_convention == MONO_CALL_C) ||
(call->signature->call_convention == MONO_CALL_STDCALL);
#else
gboolean result = (call->signature->call_convention == MONO_CALL_DEFAULT) || ((call->signature->call_convention == MONO_CALL_C) && ctx->llvm_only);
#endif
return result;
}
static void
process_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, MonoInst *ins)
{
MonoCompile *cfg = ctx->cfg;
LLVMValueRef *values = ctx->values;
LLVMValueRef *addresses = ctx->addresses;
MonoCallInst *call = (MonoCallInst*)ins;
MonoMethodSignature *sig = call->signature;
LLVMValueRef callee = NULL, lcall;
LLVMValueRef *args;
LLVMCallInfo *cinfo;
GSList *l;
int i, len, nargs;
gboolean vretaddr;
LLVMTypeRef llvm_sig;
gpointer target;
gboolean is_virtual, calli, preserveall;
LLVMBuilderRef builder = *builder_ref;
/* If both imt and rgctx arg are required, only pass the imt arg, the rgctx trampoline will pass the rgctx */
if (call->imt_arg_reg)
call->rgctx_arg_reg = 0;
if (!is_supported_callconv (ctx, call)) {
set_failure (ctx, "non-default callconv");
return;
}
cinfo = call->cinfo;
g_assert (cinfo);
if (call->rgctx_arg_reg)
cinfo->rgctx_arg = TRUE;
if (call->imt_arg_reg)
cinfo->imt_arg = TRUE;
if (call->method && needs_extra_arg (ctx, call->method))
cinfo->dummy_arg = TRUE;
vretaddr = (cinfo->ret.storage == LLVMArgVtypeRetAddr || cinfo->ret.storage == LLVMArgVtypeByRef || cinfo->ret.storage == LLVMArgGsharedvtFixed || cinfo->ret.storage == LLVMArgGsharedvtVariable || cinfo->ret.storage == LLVMArgGsharedvtFixedVtype);
llvm_sig = sig_to_llvm_sig_full (ctx, sig, cinfo);
if (!ctx_ok (ctx))
return;
int const opcode = ins->opcode;
is_virtual = opcode == OP_VOIDCALL_MEMBASE || opcode == OP_CALL_MEMBASE
|| opcode == OP_VCALL_MEMBASE || opcode == OP_LCALL_MEMBASE
|| opcode == OP_FCALL_MEMBASE || opcode == OP_RCALL_MEMBASE
|| opcode == OP_TAILCALL_MEMBASE;
calli = !call->fptr_is_patch && (opcode == OP_VOIDCALL_REG || opcode == OP_CALL_REG
|| opcode == OP_VCALL_REG || opcode == OP_LCALL_REG || opcode == OP_FCALL_REG
|| opcode == OP_RCALL_REG || opcode == OP_TAILCALL_REG);
/* Unused */
preserveall = FALSE;
/* FIXME: Avoid creating duplicate methods */
if (ins->flags & MONO_INST_HAS_METHOD) {
if (is_virtual) {
callee = NULL;
} else {
if (cfg->compile_aot) {
callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_METHOD, call->method);
if (!callee) {
set_failure (ctx, "can't encode patch");
return;
}
} else {
ERROR_DECL (error);
static int tramp_index;
char *name;
name = g_strdup_printf ("tramp_%d", tramp_index);
tramp_index ++;
#if LLVM_API_VERSION > 100
/*
* Use our trampoline infrastructure for lazy compilation instead of llvm's.
* Make all calls through a global. The address of the global will be saved in
* MonoJitDomainInfo.llvm_jit_callees and updated when the method it refers to is
* compiled.
*/
LLVMValueRef tramp_var = (LLVMValueRef)g_hash_table_lookup (ctx->jit_callees, call->method);
if (!tramp_var) {
target =
mono_create_jit_trampoline (mono_domain_get (),
call->method, error);
if (!is_ok (error)) {
set_failure (ctx, mono_error_get_message (error));
mono_error_cleanup (error);
return;
}
tramp_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (llvm_sig, 0), name);
LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (llvm_sig, 0)));
LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
g_hash_table_insert (ctx->jit_callees, call->method, tramp_var);
}
callee = LLVMBuildLoad (builder, tramp_var, "");
#else
target =
mono_create_jit_trampoline (mono_domain_get (),
call->method, error);
if (!is_ok (error)) {
g_free (name);
set_failure (ctx, mono_error_get_message (error));
mono_error_cleanup (error);
return;
}
callee = LLVMAddFunction (ctx->lmodule, name, llvm_sig);
g_free (name);
LLVMAddGlobalMapping (ctx->module->ee, callee, target);
#endif
}
}
if (!cfg->llvm_only && call->method && strstr (m_class_get_name (call->method->klass), "AsyncVoidMethodBuilder")) {
/* LLVM miscompiles async methods */
set_failure (ctx, "#13734");
return;
}
} else if (calli) {
} else {
MonoJitICallInfo *info = mono_find_jit_icall_by_addr (call->fptr);
if (info) {
/*
MonoJumpInfo ji;
memset (&ji, 0, sizeof (ji));
ji.type = MONO_PATCH_INFO_JIT_ICALL_ADDR;
ji.data.target = info->name;
target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE);
*/
if (cfg->compile_aot) {
callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_JIT_ICALL, (char*)info->name);
if (!callee) {
set_failure (ctx, "can't encode patch");
return;
}
} else {
target = (gpointer)mono_icall_get_wrapper (info);
callee = emit_jit_callee (ctx, "", llvm_sig, target);
}
} else {
if (cfg->compile_aot) {
callee = NULL;
if (cfg->abs_patches) {
MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
if (abs_ji) {
callee = get_callee (ctx, llvm_sig, abs_ji->type, abs_ji->data.target);
if (!callee) {
set_failure (ctx, "can't encode patch");
return;
}
}
}
if (!callee) {
set_failure (ctx, "aot");
return;
}
} else {
#if LLVM_API_VERSION > 100
if (cfg->abs_patches) {
MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
if (abs_ji) {
ERROR_DECL (error);
target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE, error);
mono_error_assert_ok (error);
callee = emit_jit_callee (ctx, "", llvm_sig, target);
} else {
g_assert_not_reached ();
}
} else {
g_assert_not_reached ();
}
#else
callee = LLVMAddFunction (ctx->lmodule, "", llvm_sig);
target = NULL;
if (cfg->abs_patches) {
MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
if (abs_ji) {
ERROR_DECL (error);
/*
* FIXME: Some trampolines might have
* their own calling convention on some platforms.
*/
target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE, error);
mono_error_assert_ok (error);
LLVMAddGlobalMapping (ctx->module->ee, callee, target);
}
}
if (!target)
LLVMAddGlobalMapping (ctx->module->ee, callee, (gpointer)call->fptr);
#endif
}
}
}
if (is_virtual) {
int size = TARGET_SIZEOF_VOID_P;
LLVMValueRef index;
g_assert (ins->inst_offset % size == 0);
index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
callee = convert (ctx, LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMPointerType (IntPtrType (), 0), 0)), &index, 1, ""), ""), LLVMPointerType (llvm_sig, 0));
} else if (calli) {
callee = convert (ctx, values [ins->sreg1], LLVMPointerType (llvm_sig, 0));
} else {
if (ins->flags & MONO_INST_HAS_METHOD) {
}
}
/*
* Collect and convert arguments
*/
nargs = (sig->param_count * 16) + sig->hasthis + vretaddr + call->rgctx_reg + call->imt_arg_reg + 1;
len = sizeof (LLVMValueRef) * nargs;
args = g_newa (LLVMValueRef, nargs);
memset (args, 0, len);
l = call->out_ireg_args;
if (call->rgctx_arg_reg) {
g_assert (values [call->rgctx_arg_reg]);
g_assert (cinfo->rgctx_arg_pindex < nargs);
/*
* On ARM, the imt/rgctx argument is passed in a caller save register, but some of our trampolines etc. clobber it, leading to
* problems is LLVM moves the arg assignment earlier. To work around this, save the argument into a stack slot and load
* it using a volatile load.
*/
#ifdef TARGET_ARM
if (!ctx->imt_rgctx_loc)
ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, TARGET_SIZEOF_VOID_P);
LLVMBuildStore (builder, convert (ctx, ctx->values [call->rgctx_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
args [cinfo->rgctx_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE);
#else
args [cinfo->rgctx_arg_pindex] = convert (ctx, values [call->rgctx_arg_reg], ctx->module->ptr_type);
#endif
}
if (call->imt_arg_reg) {
g_assert (!ctx->llvm_only);
g_assert (values [call->imt_arg_reg]);
g_assert (cinfo->imt_arg_pindex < nargs);
#ifdef TARGET_ARM
if (!ctx->imt_rgctx_loc)
ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, TARGET_SIZEOF_VOID_P);
LLVMBuildStore (builder, convert (ctx, ctx->values [call->imt_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
args [cinfo->imt_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE);
#else
args [cinfo->imt_arg_pindex] = convert (ctx, values [call->imt_arg_reg], ctx->module->ptr_type);
#endif
}
switch (cinfo->ret.storage) {
case LLVMArgGsharedvtVariable: {
MonoInst *var = get_vreg_to_inst (cfg, call->inst.dreg);
if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
args [cinfo->vret_arg_pindex] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), IntPtrType ());
} else {
g_assert (addresses [call->inst.dreg]);
args [cinfo->vret_arg_pindex] = convert (ctx, addresses [call->inst.dreg], IntPtrType ());
}
break;
}
default:
if (vretaddr) {
if (!addresses [call->inst.dreg])
addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
g_assert (cinfo->vret_arg_pindex < nargs);
if (cinfo->ret.storage == LLVMArgVtypeByRef)
args [cinfo->vret_arg_pindex] = addresses [call->inst.dreg];
else
args [cinfo->vret_arg_pindex] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), "");
}
break;
}
/*
* Sometimes the same method is called with two different signatures (i.e. with and without 'this'), so
* use the real callee for argument type conversion.
*/
LLVMTypeRef callee_type = LLVMGetElementType (LLVMTypeOf (callee));