Permalink
Switch branches/tags
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
7523 lines (7056 sloc) 312 KB
// This file is a part of Julia. License is MIT: https://julialang.org/license
#include "llvm-version.h"
#include "platform.h"
#include "options.h"
#if defined(_OS_WINDOWS_) && JL_LLVM_VERSION < 70000
// trick llvm into skipping the generation of _chkstk calls
// since it has some codegen issues associated with them:
// (a) assumed to be within 32-bit offset
// (b) bad asm is generated for certain code patterns:
// see https://github.com/JuliaLang/julia/pull/11644#issuecomment-112276813
// also, use ELF because RuntimeDyld COFF I686 support didn't exist
// also, use ELF because RuntimeDyld COFF X86_64 doesn't seem to work (fails to generate function pointers)?
#define FORCE_ELF
#endif
#if defined(_OS_WINDOWS_) || defined(_OS_FREEBSD_)
# define JL_DISABLE_FPO
#endif
#if defined(_CPU_X86_)
#define JL_NEED_FLOATTEMP_VAR 1
#endif
#ifndef __STDC_LIMIT_MACROS
#define __STDC_LIMIT_MACROS
#define __STDC_CONSTANT_MACROS
#endif
#include <setjmp.h>
#include <string>
#include <sstream>
#include <fstream>
#include <map>
#include <array>
#include <vector>
#include <set>
#include <cstdio>
#include <iostream>
#include <functional>
// target machine computation
#include <llvm/CodeGen/TargetSubtargetInfo.h>
#include <llvm/Support/TargetRegistry.h>
#include <llvm/Target/TargetOptions.h>
#include <llvm/Support/Host.h>
#include <llvm/Support/TargetSelect.h>
#include <llvm/Object/SymbolSize.h>
// IR building
#include <llvm/IR/IntrinsicInst.h>
#include <llvm/Object/ObjectFile.h>
#include <llvm/IR/DIBuilder.h>
#include <llvm/AsmParser/Parser.h>
#include <llvm/DebugInfo/DIContext.h>
#include <llvm/IR/DerivedTypes.h>
#include <llvm/IR/Intrinsics.h>
#include <llvm/IR/Attributes.h>
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/MDBuilder.h>
// support
#include <llvm/ADT/SmallBitVector.h>
#include <llvm/ADT/Optional.h>
#include <llvm/Support/raw_ostream.h>
#include <llvm/Support/FormattedStream.h>
#include <llvm/Support/SourceMgr.h> // for llvmcall
#include <llvm/Transforms/Utils/Cloning.h> // for llvmcall inlining
#include <llvm/Transforms/Utils/BasicBlockUtils.h>
#include <llvm/IR/Verifier.h> // for llvmcall validation
#include <llvm/Bitcode/BitcodeWriter.h>
// C API
#include <llvm-c/Types.h>
// for configuration options
#include <llvm/Support/PrettyStackTrace.h>
#include <llvm/Support/CommandLine.h>
#include <llvm/IR/InlineAsm.h>
#if defined(_CPU_ARM_) || defined(_CPU_AARCH64_)
# include <sys/utsname.h>
#endif
#if defined(USE_POLLY)
#include <polly/RegisterPasses.h>
#include <polly/ScopDetection.h>
#endif
#include <llvm/ExecutionEngine/ExecutionEngine.h>
using namespace llvm;
namespace llvm {
extern bool annotateSimdLoop(BasicBlock *latch);
}
#if defined(_OS_WINDOWS_) && !defined(NOMINMAX)
#define NOMINMAX
#endif
#include "julia.h"
#include "julia_internal.h"
#include "jitlayers.h"
#include "codegen_shared.h"
#include "processor.h"
#include "julia_assert.h"
// LLVM version compatibility macros
legacy::PassManager *jl_globalPM;
#define DIFlagZero (DINode::FlagZero)
extern "C" {
#include "builtin_proto.h"
#ifdef HAVE_SSP
extern uintptr_t __stack_chk_guard;
extern void __stack_chk_fail();
#else
JL_DLLEXPORT uintptr_t __stack_chk_guard = (uintptr_t)0xBAD57ACCBAD67ACC; // 0xBADSTACKBADSTACK
JL_DLLEXPORT void __stack_chk_fail()
{
/* put your panic function or similar in here */
fprintf(stderr, "fatal error: stack corruption detected\n");
gc_debug_critical_error();
abort(); // end with abort, since the compiler destroyed the stack upon entry to this function, there's no going back now
}
#endif
#ifdef _OS_WINDOWS_
#if defined(_CPU_X86_64_)
#if defined(_COMPILER_MINGW_)
extern void ___chkstk_ms(void);
#else
extern void __chkstk(void);
#endif
#else
#if defined(_COMPILER_MINGW_)
#undef _alloca
extern void _alloca(void);
#else
extern void _chkstk(void);
#endif
#endif
//void *force_chkstk(void) {
// return alloca(40960);
//}
#endif
}
#if defined(_COMPILER_MICROSOFT_) && !defined(__alignof__)
#define __alignof__ __alignof
#endif
#define DISABLE_FLOAT16
// llvm state
JL_DLLEXPORT LLVMContext jl_LLVMContext;
TargetMachine *jl_TargetMachine;
extern JITEventListener *CreateJuliaJITEventListener();
// for image reloading
bool imaging_mode = false;
Module *shadow_output;
#define jl_Module ctx.f->getParent()
#define jl_builderModule(builder) (builder).GetInsertBlock()->getParent()->getParent()
static DataLayout jl_data_layout("");
// types
static Type *T_jlvalue;
static Type *T_pjlvalue;
static Type *T_prjlvalue;
static Type *T_ppjlvalue;
static Type *T_pprjlvalue;
static Type *jl_array_llvmt;
static Type *jl_parray_llvmt;
static FunctionType *jl_func_sig;
static FunctionType *jl_func_sig_sparams;
static Type *T_pvoidfunc;
static IntegerType *T_int1;
static IntegerType *T_int8;
static IntegerType *T_int16;
static IntegerType *T_int32;
static IntegerType *T_int64;
static IntegerType *T_uint8;
static IntegerType *T_uint16;
static IntegerType *T_uint32;
static IntegerType *T_uint64;
static IntegerType *T_char;
static IntegerType *T_size;
static IntegerType *T_sigatomic;
static Type *T_float16;
static Type *T_float32;
static Type *T_float64;
static Type *T_float128;
static Type *T_pint8;
static Type *T_pint16;
static Type *T_pint32;
static Type *T_pint64;
static Type *T_psize;
static Type *T_pfloat32;
static Type *T_pfloat64;
static Type *T_ppint8;
static Type *T_pppint8;
static Type *T_void;
// type-based alias analysis nodes. Indentation of comments indicates hierarchy.
static MDNode *tbaa_gcframe; // GC frame
// LLVM should have enough info for alias analysis of non-gcframe stack slot
// this is mainly a place holder for `jl_cgval_t::tbaa`
static MDNode *tbaa_stack; // stack slot
static MDNode *tbaa_data; // Any user data that `pointerset/ref` are allowed to alias
static MDNode *tbaa_binding; // jl_binding_t::value
static MDNode *tbaa_value; // jl_value_t, that is not jl_array_t
static MDNode *tbaa_mutab; // mutable type
static MDNode *tbaa_immut; // immutable type
static MDNode *tbaa_ptrarraybuf; // Data in an array of boxed values
static MDNode *tbaa_arraybuf; // Data in an array of POD
static MDNode *tbaa_unionselbyte; // a selector byte in isbits Union struct fields
static MDNode *tbaa_array; // jl_array_t
static MDNode *tbaa_arrayptr; // The pointer inside a jl_array_t
static MDNode *tbaa_arraysize; // A size in a jl_array_t
static MDNode *tbaa_arraylen; // The len in a jl_array_t
static MDNode *tbaa_arrayflags; // The flags in a jl_array_t
static MDNode *tbaa_arrayoffset; // The offset in a jl_array_t
static MDNode *tbaa_arrayselbyte; // a selector byte in a isbits Union jl_array_t
static MDNode *tbaa_const; // Memory that is immutable by the time LLVM can see it
static Attribute Thunk;
// Basic DITypes
static DICompositeType *jl_value_dillvmt;
static DIDerivedType *jl_pvalue_dillvmt;
static DIDerivedType *jl_ppvalue_dillvmt;
static DISubroutineType *jl_di_func_sig;
static DISubroutineType *jl_di_func_null_sig;
// constants
static Constant *V_null;
extern "C" {
JL_DLLEXPORT Type *julia_type_to_llvm(jl_value_t *jt, bool *isboxed=NULL);
}
static bool type_is_ghost(Type *ty)
{
return (ty == T_void || ty->isEmptyTy());
}
// global vars
static GlobalVariable *jlRTLD_DEFAULT_var;
#ifdef _OS_WINDOWS_
static GlobalVariable *jlexe_var;
static GlobalVariable *jldll_var;
#endif //_OS_WINDOWS_
static Function *jltls_states_func;
// important functions
static Function *jlnew_func;
static Function *jlthrow_func;
static Function *jlerror_func;
static Function *jltypeerror_func;
static Function *jlundefvarerror_func;
static Function *jlboundserror_func;
static Function *jluboundserror_func;
static Function *jlvboundserror_func;
static Function *jlboundserrorv_func;
static Function *jlcheckassign_func;
static Function *jldeclareconst_func;
static Function *jlgetbindingorerror_func;
static Function *jlboundp_func;
static Function *jltopeval_func;
static Function *jlcopyast_func;
static Function *jltuple_func;
static Function *jlnsvec_func;
static Function *jlapplygeneric_func;
static Function *jlinvoke_func;
static Function *jlapply2va_func;
static Function *jlgetfield_func;
static Function *jlmethod_func;
static Function *jlgenericfunction_func;
static Function *jlenter_func;
static Function *jlleave_func;
static Function *jlegal_func;
static Function *jl_alloc_obj_func;
static Function *jl_newbits_func;
static Function *jl_typeof_func;
static Function *jl_simdloop_marker_func;
static Function *jl_simdivdep_marker_func;
static Function *jl_write_barrier_func;
static Function *jlisa_func;
static Function *jlsubtype_func;
static Function *jlapplytype_func;
static Function *setjmp_func;
static Function *memcmp_derived_func;
static Function *box_int8_func;
static Function *box_uint8_func;
static Function *box_int16_func;
static Function *box_uint16_func;
static Function *box_int32_func;
static Function *box_char_func;
static Function *box_uint32_func;
static Function *box_int64_func;
static Function *box_uint64_func;
static Function *box_float32_func;
static Function *box_float64_func;
static Function *box_ssavalue_func;
static Function *expect_func;
static Function *jldlsym_func;
static Function *jltypeassert_func;
//static Function *jlgetnthfield_func;
static Function *jlgetnthfieldchecked_func;
//static Function *jlsetnthfield_func;
static Function *jlgetcfunctiontrampoline_func;
#ifdef _OS_WINDOWS_
static Function *resetstkoflw_func;
#if defined(_CPU_X86_64_)
Function *juliapersonality_func;
#endif
#endif
static Function *diff_gc_total_bytes_func;
static Function *jlarray_data_owner_func;
static GlobalVariable *jlgetworld_global;
// placeholder functions
static Function *gcroot_flush_func;
static Function *gc_preserve_begin_func;
static Function *gc_preserve_end_func;
static Function *except_enter_func;
static Function *pointer_from_objref_func;
static std::vector<Type *> two_pvalue_llvmt;
static std::vector<Type *> three_pvalue_llvmt;
static std::vector<Type *> four_pvalue_llvmt;
static std::map<jl_fptr_args_t, Function*> builtin_func_map;
// --- code generation ---
extern "C" {
int globalUnique = 0;
}
template<typename T>
static void add_return_attr(T *f, Attribute::AttrKind Kind)
{
f->addAttribute(AttributeList::ReturnIndex, Kind);
}
static MDNode *best_tbaa(jl_value_t *jt) {
jt = jl_unwrap_unionall(jt);
if (!jl_is_datatype(jt))
return tbaa_value;
if (jl_is_abstracttype(jt))
return tbaa_value;
// If we're here, we know all subtypes are (im)mutable, even if we
// don't know what the exact type is
return jl_is_mutable(jt) ? tbaa_mutab : tbaa_immut;
}
// tracks whether codegen is currently able to simply stack-allocate this type
// note that this is guaranteed to include jl_isbits
static bool jl_justbits(jl_value_t* t)
{
return jl_is_immutable_datatype(t) && ((jl_datatype_t*)t)->layout && ((jl_datatype_t*)t)->layout->npointers == 0;
}
// metadata tracking for a llvm Value* during codegen
struct jl_cgval_t {
Value *V; // may be of type T* or T, or set to NULL if ghost (or if the value has not been initialized yet, for a variable definition)
// For unions, we may need to keep a reference to the boxed part individually.
// If this is non-NULL, then, at runtime, we satisfy the invariant that (for the corresponding
// runtime values) if `(TIndex | 0x80) != 0`, then `Vboxed == V` (by value).
// For convenience, we also set this value of isboxed values, in which case
// it is equal (at compile time) to V.
Value *Vboxed;
Value *TIndex; // if `V` is an unboxed (tagged) Union described by `typ`, this gives the DataType index (1-based, small int) as an i8
jl_value_t *constant; // constant value (rooted in linfo.def.roots)
jl_value_t *typ; // the original type of V, never NULL
bool isboxed; // whether this value is a jl_value_t* allocated on the heap with the right type tag
bool isghost; // whether this value is "ghost"
MDNode *tbaa; // The related tbaa node. Non-NULL iff this holds an address.
bool ispointer() const
{
// whether this value is compatible with `data_pointer`
return tbaa != nullptr;
}
jl_cgval_t(Value *V, Value *gcroot, bool isboxed, jl_value_t *typ, Value *tindex) : // general constructor (with pointer type auto-detect)
V(V), // V is allowed to be NULL in a jl_varinfo_t context, but not during codegen contexts
Vboxed(isboxed ? V : nullptr),
TIndex(tindex),
constant(NULL),
typ(typ),
isboxed(isboxed),
isghost(false),
tbaa(isboxed ? best_tbaa(typ) : nullptr)
{
assert(gcroot == nullptr);
assert(!(isboxed && TIndex != NULL));
assert(TIndex == NULL || TIndex->getType() == T_int8);
}
explicit jl_cgval_t(jl_value_t *typ) : // ghost value constructor
// mark explicit to avoid being used implicitly for conversion from NULL (use jl_cgval_t() instead)
V(NULL),
Vboxed(NULL),
TIndex(NULL),
constant(((jl_datatype_t*)typ)->instance),
typ(typ),
isboxed(false),
isghost(true),
tbaa(nullptr)
{
assert(jl_is_datatype(typ));
assert(constant);
}
jl_cgval_t(const jl_cgval_t &v, jl_value_t *typ, Value *tindex) : // copy constructor with new type
V(v.V),
Vboxed(v.Vboxed),
TIndex(tindex),
constant(v.constant),
typ(typ),
isboxed(v.isboxed),
isghost(v.isghost),
tbaa(v.tbaa)
{
// this constructor expects we had a badly or equivalently typed version
// make sure we aren't discarding the actual type information
if (v.TIndex) {
assert((TIndex == NULL) == jl_is_concrete_type(typ));
}
else {
assert(isboxed || v.typ == typ || tindex);
}
}
jl_cgval_t() : // undef / unreachable / default constructor
V(UndefValue::get(T_void)),
Vboxed(NULL),
TIndex(NULL),
constant(NULL),
typ(jl_bottom_type),
isboxed(false),
isghost(true),
tbaa(nullptr)
{
}
};
// per-local-variable information
struct jl_varinfo_t {
Instruction *boxroot; // an address, if the var might be in a jl_value_t** stack slot (marked tbaa_const, if appropriate)
jl_cgval_t value; // a stack slot or constant value
Value *pTIndex; // i8* stack slot for the value.TIndex tag describing `value.V`
DILocalVariable *dinfo;
// if the variable might be used undefined and is not boxed
// this i1 flag is true when it is defined
Value *defFlag;
bool isSA; // whether all stores dominate all uses
bool isVolatile;
bool isArgument;
bool usedUndef;
bool used;
jl_varinfo_t() : boxroot(NULL),
value(jl_cgval_t()),
pTIndex(NULL),
dinfo(NULL),
defFlag(NULL),
isSA(false),
isVolatile(false),
isArgument(false),
usedUndef(false),
used(false)
{
}
};
struct jl_returninfo_t {
Function *decl;
enum CallingConv {
Boxed = 0,
Register,
SRet,
Union,
Ghosts
} cc;
size_t union_bytes;
size_t union_align;
size_t union_minalign;
};
static jl_returninfo_t get_specsig_function(Module *M, const std::string &name, jl_value_t *sig, jl_value_t *jlrettype);
// information about the context of a piece of code: its enclosing
// function and module, and visible local variables and labels.
class jl_codectx_t {
public:
IRBuilder<> builder;
Function *f = NULL;
// local var info. globals are not in here.
std::vector<jl_varinfo_t> slots;
std::map<int, jl_varinfo_t> phic_slots;
std::vector<jl_cgval_t> SAvalues;
std::vector<std::tuple<jl_cgval_t, BasicBlock *, AllocaInst *, PHINode *, jl_value_t *>> PhiNodes;
std::vector<bool> ssavalue_assigned;
jl_module_t *module = NULL;
jl_method_instance_t *linfo = NULL;
jl_code_info_t *source = NULL;
jl_array_t *code = NULL;
size_t world = 0;
jl_array_t *roots = NULL;
const char *name = NULL;
StringRef file{};
ssize_t *line = NULL;
Value *spvals_ptr = NULL;
Value *argArray = NULL;
Value *argCount = NULL;
std::string funcName;
int vaSlot = -1; // name of vararg argument
bool has_sret = false;
int nReqArgs = 0;
int nargs = 0;
int nvargs = -1;
CallInst *ptlsStates = NULL;
Value *signalPage = NULL;
Value *world_age_field = NULL;
bool debug_enabled = false;
const jl_cgparams_t *params = NULL;
jl_codectx_t(LLVMContext &llvmctx)
: builder(llvmctx) { }
~jl_codectx_t() {
assert(this->roots == NULL);
}
};
static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval = -1);
static Value *global_binding_pointer(jl_codectx_t &ctx, jl_module_t *m, jl_sym_t *s,
jl_binding_t **pbnd, bool assign);
static jl_cgval_t emit_checked_var(jl_codectx_t &ctx, Value *bp, jl_sym_t *name, bool isvol, MDNode *tbaa);
static jl_cgval_t emit_sparam(jl_codectx_t &ctx, size_t i);
static Value *emit_condition(jl_codectx_t &ctx, const jl_cgval_t &condV, const std::string &msg);
static void allocate_gc_frame(jl_codectx_t &ctx, BasicBlock *b0);
static void CreateTrap(IRBuilder<> &irbuilder);
static Value *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF,
jl_cgval_t *args, size_t nargs);
static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p);
static GlobalVariable *prepare_global_in(Module *M, GlobalVariable *G);
#define prepare_global(G) prepare_global_in(jl_Module, (G))
// --- convenience functions for tagging llvm values with julia types ---
static GlobalVariable *get_pointer_to_constant(Constant *val, StringRef name, Module &M)
{
GlobalVariable *gv = new GlobalVariable(
M,
val->getType(),
true,
GlobalVariable::PrivateLinkage,
val,
name);
gv->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
return gv;
}
static AllocaInst *emit_static_alloca(jl_codectx_t &ctx, Type *lty, int arraysize=1)
{
return new AllocaInst(lty,
0,
ConstantInt::get(T_int32, arraysize), "", /*InsertBefore=*/ctx.ptlsStates);
}
static inline jl_cgval_t ghostValue(jl_value_t *typ)
{
if (typ == jl_bottom_type)
return jl_cgval_t(); // Undef{}
if (typ == (jl_value_t*)jl_typeofbottom_type) {
// normalize TypeofBottom to Type{Union{}}
typ = (jl_value_t*)jl_wrap_Type(jl_bottom_type);
}
if (jl_is_type_type(typ)) {
// replace T::Type{T} with T, by assuming that T must be a leaftype of some sort
jl_cgval_t constant(NULL, NULL, true, typ, NULL);
constant.constant = jl_tparam0(typ);
return constant;
}
return jl_cgval_t(typ);
}
static inline jl_cgval_t ghostValue(jl_datatype_t *typ)
{
return ghostValue((jl_value_t*)typ);
}
static inline jl_cgval_t mark_julia_const(jl_value_t *jv)
{
jl_value_t *typ;
if (jl_is_type(jv)) {
typ = (jl_value_t*)jl_wrap_Type(jv); // TODO: gc-root this?
}
else {
typ = jl_typeof(jv);
if (type_is_ghost(julia_type_to_llvm(typ))) {
return ghostValue(typ);
}
}
jl_cgval_t constant(NULL, NULL, true, typ, NULL);
constant.constant = jv;
return constant;
}
static inline jl_cgval_t mark_julia_slot(Value *v, jl_value_t *typ, Value *tindex, MDNode *tbaa)
{
// this enables lazy-copying of immutable values and stack or argument slots
assert(tbaa);
jl_cgval_t tagval(v, NULL, false, typ, tindex);
tagval.tbaa = tbaa;
return tagval;
}
static inline jl_cgval_t value_to_pointer(jl_codectx_t &ctx, Value *v, jl_value_t *typ, Value *tindex)
{
Value *loc;
if (Constant *cv = dyn_cast<Constant>(v)) {
loc = get_pointer_to_constant(cv, "", *jl_Module);
}
else {
loc = emit_static_alloca(ctx, v->getType());
ctx.builder.CreateStore(v, loc);
}
return mark_julia_slot(loc, typ, tindex, tbaa_stack);
}
static inline jl_cgval_t value_to_pointer(jl_codectx_t &ctx, const jl_cgval_t &v)
{
if (v.ispointer())
return v;
return value_to_pointer(ctx, v.V, v.typ, v.TIndex);
}
static inline jl_cgval_t mark_julia_type(jl_codectx_t &ctx, Value *v, bool isboxed, jl_value_t *typ)
{
if (jl_is_datatype(typ) && jl_is_datatype_singleton((jl_datatype_t*)typ)) {
// no need to explicitly load/store a constant/ghost value
return ghostValue(typ);
}
if (jl_is_type_type(typ)) {
jl_value_t *tp0 = jl_tparam0(typ);
if (jl_is_concrete_type(tp0) || tp0 == jl_bottom_type) {
// replace T::Type{T} with T
return ghostValue(typ);
}
}
Type *T = julia_type_to_llvm(typ);
if (type_is_ghost(T)) {
return ghostValue(typ);
}
if (v && !isboxed && v->getType()->isAggregateType()) {
// eagerly put this back onto the stack
// llvm mem2reg pass will remove this if unneeded
return value_to_pointer(ctx, v, typ, NULL);
}
return jl_cgval_t(v, NULL, isboxed, typ, NULL);
}
static inline jl_cgval_t mark_julia_type(jl_codectx_t &ctx, Value *v, bool isboxed, jl_datatype_t *typ)
{
return mark_julia_type(ctx, v, isboxed, (jl_value_t*)typ);
}
// see if it might be profitable (and cheap) to change the type of v to typ
static inline jl_cgval_t update_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_value_t *typ)
{
if (v.typ == typ || v.typ == jl_bottom_type || v.constant || typ == (jl_value_t*)jl_any_type || jl_egal(v.typ, typ))
return v; // fast-path
if (jl_is_concrete_type(v.typ) && !jl_is_kind(v.typ)) {
if (jl_is_concrete_type(typ) && !jl_is_kind(typ)) {
// type mismatch: changing from one leaftype to another
CreateTrap(ctx.builder);
return jl_cgval_t();
}
return v; // doesn't improve type info
}
if (v.TIndex) {
jl_value_t *utyp = jl_unwrap_unionall(typ);
if (jl_is_datatype(utyp)) {
bool alwaysboxed;
if (jl_is_concrete_type(utyp))
alwaysboxed = !jl_justbits(utyp);
else
alwaysboxed = !((jl_datatype_t*)utyp)->abstract && ((jl_datatype_t*)utyp)->mutabl;
if (alwaysboxed) {
// discovered that this union-split type must actually be isboxed
if (v.Vboxed) {
return jl_cgval_t(v.Vboxed, nullptr, true, typ, NULL);
}
else {
// type mismatch (there weren't any boxed values in the union)
CreateTrap(ctx.builder);
return jl_cgval_t();
}
}
}
if (!jl_is_concrete_type(typ))
return v; // not generally worth trying to change type info (which would require recomputing tindex)
}
Type *T = julia_type_to_llvm(typ);
if (type_is_ghost(T))
return ghostValue(typ);
return jl_cgval_t(v, typ, NULL);
}
static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_value_t *typ);
// --- allocating local variables ---
static jl_sym_t *slot_symbol(jl_codectx_t &ctx, int s)
{
return (jl_sym_t*)jl_array_ptr_ref(ctx.source->slotnames, s);
}
static void store_def_flag(jl_codectx_t &ctx, const jl_varinfo_t &vi, bool val)
{
assert((!vi.boxroot || vi.pTIndex) && "undef check is null pointer for boxed things");
assert(vi.usedUndef && vi.defFlag && "undef flag codegen corrupted");
ctx.builder.CreateStore(ConstantInt::get(T_int1, val), vi.defFlag, vi.isVolatile);
}
static void alloc_def_flag(jl_codectx_t &ctx, jl_varinfo_t& vi)
{
assert((!vi.boxroot || vi.pTIndex) && "undef check is null pointer for boxed things");
if (vi.usedUndef) {
vi.defFlag = emit_static_alloca(ctx, T_int1);
store_def_flag(ctx, vi, false);
}
}
// --- utilities ---
static void CreateTrap(IRBuilder<> &irbuilder)
{
Function *f = irbuilder.GetInsertBlock()->getParent();
Function *trap_func = Intrinsic::getDeclaration(
f->getParent(),
Intrinsic::trap);
irbuilder.CreateCall(trap_func);
irbuilder.CreateUnreachable();
BasicBlock *newBB = BasicBlock::Create(irbuilder.getContext(), "after_noret", f);
irbuilder.SetInsertPoint(newBB);
}
#if 0 // this code is likely useful, but currently unused
#ifndef JL_NDEBUG
static void CreateConditionalAbort(IRBuilder<> &irbuilder, Value *test)
{
Function *f = irbuilder.GetInsertBlock()->getParent();
BasicBlock *abortBB = BasicBlock::Create(jl_LLVMContext, "debug_abort", f);
BasicBlock *postBB = BasicBlock::Create(jl_LLVMContext, "post_abort", f);
irbuilder.CreateCondBr(test, abortBB, postBB);
irbuilder.SetInsertPoint(abortBB);
Function *trap_func = Intrinsic::getDeclaration(
f->getParent(),
Intrinsic::trap);
irbuilder.CreateCall(trap_func);
irbuilder.CreateUnreachable();
irbuilder.SetInsertPoint(postBB);
}
#endif
#endif
static void emit_write_barrier(jl_codectx_t&, Value*, Value*);
#include "cgutils.cpp"
static void jl_rethrow_with_add(const char *fmt, ...)
{
jl_ptls_t ptls = jl_get_ptls_states();
if (jl_typeis(ptls->exception_in_transit, jl_errorexception_type)) {
char *str = jl_string_data(jl_fieldref(ptls->exception_in_transit,0));
char buf[1024];
va_list args;
va_start(args, fmt);
int nc = vsnprintf(buf, sizeof(buf), fmt, args);
va_end(args);
nc += snprintf(buf+nc, sizeof(buf)-nc, ": %s", str);
jl_value_t *msg = jl_pchar_to_string(buf, nc);
JL_GC_PUSH1(&msg);
jl_throw(jl_new_struct(jl_errorexception_type, msg));
}
jl_rethrow();
}
static jl_cgval_t convert_julia_type_union(jl_codectx_t &ctx, const jl_cgval_t &v, jl_value_t *typ)
{
// previous value was a split union, compute new index, or box
Value *new_tindex = ConstantInt::get(T_int8, 0x80);
SmallBitVector skip_box(1, true);
Value *tindex = ctx.builder.CreateAnd(v.TIndex, ConstantInt::get(T_int8, 0x7f));
if (jl_is_uniontype(typ)) {
// compute the TIndex mapping from v.typ -> typ
unsigned counter = 0;
for_each_uniontype_small(
// for each old union-split value
[&](unsigned idx, jl_datatype_t *jt) {
unsigned new_idx = get_box_tindex(jt, typ);
bool t;
if (new_idx) {
// found a matching element,
// match it against either the unboxed index
Value *cmp = ctx.builder.CreateICmpEQ(tindex, ConstantInt::get(T_int8, idx));
new_tindex = ctx.builder.CreateSelect(cmp, ConstantInt::get(T_int8, new_idx), new_tindex);
t = true;
}
else if (!jl_subtype((jl_value_t*)jt, typ)) {
// new value doesn't need to be boxed
// since it isn't part of the new union
t = true;
}
else {
// will actually need to box this element
// since it appeared as a leaftype in the original type
// but not in the remark type
t = false;
}
skip_box.resize(idx + 1, t);
},
v.typ,
counter);
}
// some of the values are still unboxed
if (!isa<Constant>(new_tindex)) {
Value *wasboxed = NULL;
// If the old value was boxed and unknown (type tag 0x80),
// it is possible that the tag was actually one of the types
// that are now explicitly represented. To find out, we need
// to compare typeof(v.Vboxed) (i.e. the type of the unknown
// value) against all the types that are now explicitly
// selected and select the appropriate one as our new tindex.
if (v.Vboxed) {
wasboxed = ctx.builder.CreateAnd(v.TIndex, ConstantInt::get(T_int8, 0x80));
new_tindex = ctx.builder.CreateOr(wasboxed, new_tindex);
wasboxed = ctx.builder.CreateICmpNE(wasboxed, ConstantInt::get(T_int8, 0));
BasicBlock *currBB = ctx.builder.GetInsertBlock();
// We lazily create a BB for this, once we decide that we
// actually need it.
Value *union_box_dt = NULL;
BasicBlock *union_isaBB = NULL;
auto maybe_setup_union_isa = [&]() {
if (!union_isaBB) {
union_isaBB = BasicBlock::Create(jl_LLVMContext, "union_isa", ctx.f);
ctx.builder.SetInsertPoint(union_isaBB);
union_box_dt = emit_typeof(ctx, v.Vboxed);
}
};
// If we don't find a match. The type remains unknown
// (0x80). We could use `v.Tindex`, here, since we know
// it has to be 0x80, but it seems likely the backend
// will like the explicit constant better.
Value *union_box_tindex = ConstantInt::get(T_int8, 0x80);
unsigned counter = 0;
for_each_uniontype_small(
// for each new union-split value
[&](unsigned idx, jl_datatype_t *jt) {
unsigned old_idx = get_box_tindex(jt, v.typ);
if (old_idx == 0) {
// didn't handle this item before, select its new union index
maybe_setup_union_isa();
Value *cmp = ctx.builder.CreateICmpEQ(maybe_decay_untracked(literal_pointer_val(ctx, (jl_value_t*)jt)), union_box_dt);
union_box_tindex = ctx.builder.CreateSelect(cmp, ConstantInt::get(T_int8, 0x80 | idx), union_box_tindex);
}
},
typ,
counter);
if (union_box_dt) {
BasicBlock *postBB = BasicBlock::Create(jl_LLVMContext, "post_union_isa", ctx.f);
ctx.builder.CreateBr(postBB);
ctx.builder.SetInsertPoint(currBB);
Value *wasunknown = ctx.builder.CreateICmpEQ(v.TIndex, ConstantInt::get(T_int8, 0x80));
ctx.builder.CreateCondBr(wasunknown, union_isaBB, postBB);
ctx.builder.SetInsertPoint(postBB);
PHINode *tindex_phi = ctx.builder.CreatePHI(T_int8, 2);
tindex_phi->addIncoming(new_tindex, currBB);
tindex_phi->addIncoming(union_box_tindex, union_isaBB);
new_tindex = tindex_phi;
}
}
if (!skip_box.all()) {
// some values weren't unboxed in the new union
// box them now (tindex above already selected 0x80 = box for them)
Value *boxv = box_union(ctx, v, skip_box);
if (v.Vboxed) {
// If the value is boxed both before and after, we don't need
// to touch it at all. Otherwise we're either transitioning
// unboxed->boxed, or leaving an unboxed value in place.
Value *isboxed = ctx.builder.CreateICmpNE(
ctx.builder.CreateAnd(new_tindex, ConstantInt::get(T_int8, 0x80)),
ConstantInt::get(T_int8, 0));
boxv = ctx.builder.CreateSelect(
ctx.builder.CreateAnd(wasboxed, isboxed), v.Vboxed, boxv);
}
if (v.V == NULL) {
// v.V might be NULL if it was all ghost objects before
return jl_cgval_t(boxv, NULL, false, typ, new_tindex);
} else {
Value *isboxv = ctx.builder.CreateIsNotNull(boxv);
Value *slotv;
MDNode *tbaa;
if (v.ispointer()) {
slotv = v.V;
tbaa = v.tbaa;
}
else {
slotv = emit_static_alloca(ctx, v.V->getType());
ctx.builder.CreateStore(v.V, slotv);
tbaa = tbaa_stack;
}
slotv = ctx.builder.CreateSelect(isboxv,
decay_derived(boxv),
decay_derived(emit_bitcast(ctx, slotv, boxv->getType())));
jl_cgval_t newv = jl_cgval_t(slotv, NULL, false, typ, new_tindex);
newv.Vboxed = boxv;
newv.tbaa = tbaa;
return newv;
}
}
}
else {
return jl_cgval_t(boxed(ctx, v), NULL, true, typ, NULL);
}
return jl_cgval_t(v, typ, new_tindex);
}
// given a value marked with type `v.typ`, compute the mapping and/or boxing to return a value of type `typ`
// TODO: should this set TIndex when trivial (such as 0x80 or concrete types) ?
static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_value_t *typ)
{
if (typ == (jl_value_t*)jl_typeofbottom_type)
return ghostValue(typ); // normalize TypeofBottom to Type{Union{}}
if (v.typ == typ || v.typ == jl_bottom_type || jl_egal(v.typ, typ))
return v; // fast-path
Type *T = julia_type_to_llvm(typ);
if (type_is_ghost(T))
return ghostValue(typ);
Value *new_tindex = NULL;
if (jl_is_concrete_type(typ)) {
if (v.TIndex && !jl_justbits(typ)) {
// discovered that this union-split type must actually be isboxed
if (v.Vboxed) {
return jl_cgval_t(v.Vboxed, nullptr, true, typ, NULL);
}
else {
// type mismatch: there weren't any boxed values in the union
CreateTrap(ctx.builder);
return jl_cgval_t();
}
}
if (jl_is_concrete_type(v.typ) && !jl_is_kind(v.typ)) {
if (jl_is_concrete_type(typ) && !jl_is_kind(typ)) {
// type mismatch: changing from one leaftype to another
CreateTrap(ctx.builder);
return jl_cgval_t();
}
}
}
else {
bool makeboxed = false;
if (v.TIndex) {
return convert_julia_type_union(ctx, v, typ);
}
else if (!v.isboxed && jl_is_uniontype(typ)) {
// previous value was unboxed (leaftype), statically compute union tindex
assert(jl_is_concrete_type(v.typ));
unsigned new_idx = get_box_tindex((jl_datatype_t*)v.typ, typ);
if (new_idx) {
new_tindex = ConstantInt::get(T_int8, new_idx);
if (v.V && !v.ispointer()) {
// TODO: remove this branch once all consumers of v.TIndex understand how to handle a non-ispointer value
Value *slotv = emit_static_alloca(ctx, v.V->getType());
ctx.builder.CreateStore(v.V, slotv);
jl_cgval_t newv = jl_cgval_t(slotv, NULL, false, typ, new_tindex);
newv.tbaa = tbaa_stack;
return newv;
}
}
else if (jl_subtype(v.typ, typ)) {
makeboxed = true;
}
else {
// unreachable
CreateTrap(ctx.builder);
return jl_cgval_t();
}
}
else if (!v.isboxed) {
makeboxed = true;
}
if (makeboxed) {
// convert to a simple isboxed value
return jl_cgval_t(boxed(ctx, v), NULL, true, typ, NULL);
}
}
return jl_cgval_t(v, typ, new_tindex);
}
// Snooping on which functions are being compiled, and how long it takes
static JL_STREAM *dump_compiles_stream = NULL;
static bool nested_compile = false;
static uint64_t last_time = 0;
extern "C" JL_DLLEXPORT
void jl_dump_compiles(void *s)
{
dump_compiles_stream = (JL_STREAM*)s;
}
// --- entry point ---
//static int n_emit=0;
static std::unique_ptr<Module> emit_function(
jl_method_instance_t *lam,
jl_code_info_t *src,
size_t world,
jl_llvm_functions_t *declarations,
const jl_cgparams_t *params);
void jl_add_linfo_in_flight(StringRef name, jl_method_instance_t *linfo, const DataLayout &DL);
const char *name_from_method_instance(jl_method_instance_t *li)
{
return jl_is_method(li->def.method) ? jl_symbol_name(li->def.method->name) : "top-level scope";
}
// Use of `li` is not clobbered in JL_TRY
JL_GCC_IGNORE_START("-Wclobbered")
// this generates llvm code for the lambda info
// and adds the result to the jitlayers
// (and the shadow module), but doesn't yet compile
// or generate object code for it
extern "C"
jl_llvm_functions_t jl_compile_linfo(jl_method_instance_t **pli, jl_code_info_t *src, size_t world, const jl_cgparams_t *params)
{
// N.B.: `src` may have not been rooted by the caller.
JL_TIMING(CODEGEN);
jl_method_instance_t *li = *pli;
assert(jl_is_method_instance(li));
jl_llvm_functions_t decls = {};
if (params != &jl_default_cgparams /* fast path */ &&
!compare_cgparams(params, &jl_default_cgparams) && params->cached)
jl_error("functions compiled with custom codegen params mustn't be cached");
// Fast path for the already-compiled case
if (jl_is_method(li->def.method)) {
decls = li->functionObjectsDecls;
bool already_compiled = params->cached && decls.functionObject != NULL;
if (!src) {
if ((already_compiled || li->invoke == jl_fptr_const_return) &&
(li->min_world <= world && li->max_world >= world)) {
return decls;
}
} else if (already_compiled) {
return decls;
}
}
JL_GC_PUSH1(&src);
JL_LOCK(&codegen_lock);
decls = li->functionObjectsDecls;
// Codegen lock held in this block
{
// Step 1: Re-check if this was already compiled (it may have been while
// we waited at the lock).
if (!jl_is_method(li->def.method)) {
src = (jl_code_info_t*)li->inferred;
if (decls.functionObject != NULL || !src || !jl_is_code_info(src) || li->invoke == jl_fptr_const_return) {
goto locked_out;
}
}
else if (!src) {
// If the caller didn't provide the source,
// try to infer it for ourself, but first, re-check if it's already compiled.
assert(li->min_world <= world && li->max_world >= world);
if ((params->cached && decls.functionObject != NULL) || li->invoke == jl_fptr_const_return)
goto locked_out;
// see if it is inferred
src = (jl_code_info_t*)li->inferred;
if (src) {
if ((jl_value_t*)src != jl_nothing)
src = jl_uncompress_ast(li->def.method, (jl_array_t*)src);
if (!jl_is_code_info(src)) {
src = jl_type_infer(pli, world, 0);
li = *pli;
}
if (!src || li->invoke == jl_fptr_const_return)
goto locked_out;
}
else {
// declare a failure to compile
goto locked_out;
}
}
else if (params->cached && decls.functionObject != NULL) {
// similar to above, but never returns a NULL
// decl (unless compile fails), even if invoke == jl_fptr_const_return
goto locked_out;
}
else {
if ((jl_value_t*)src != jl_nothing)
src = jl_uncompress_ast(li->def.method, (jl_array_t*)src);
}
assert(jl_is_code_info(src));
// Step 2: setup global state
bool last_n_c = nested_compile;
if (!nested_compile && dump_compiles_stream != NULL)
last_time = jl_hrtime();
nested_compile = true;
// Step 3. actually do the work of emitting the function
std::unique_ptr<Module> m;
JL_TRY {
jl_llvm_functions_t *pdecls;
if (!params->cached)
pdecls = &decls;
else if (li->min_world <= world && li->max_world >= world)
pdecls = &li->functionObjectsDecls;
else if (!jl_is_method(li->def.method)) // toplevel thunk
pdecls = &li->functionObjectsDecls;
else
pdecls = &decls;
m = emit_function(li, src, world, pdecls, params);
if (params->cached && world)
decls = li->functionObjectsDecls;
//n_emit++;
}
JL_CATCH {
// something failed! this is very bad, since other WIP may be pointing to this function
// but there's not much we can do now. try to clear much of the WIP anyways.
li->functionObjectsDecls.functionObject = NULL;
li->functionObjectsDecls.specFunctionObject = NULL;
nested_compile = last_n_c;
JL_UNLOCK(&codegen_lock); // Might GC
const char *mname = name_from_method_instance(li);
jl_rethrow_with_add("error compiling %s", mname);
}
const char *f = decls.functionObject;
const char *specf = decls.specFunctionObject;
if (JL_HOOK_TEST(params, module_activation)) {
JL_HOOK_CALL(params, module_activation, 1, jl_box_voidpointer(wrap(m.release())));
} else {
// Step 4. Prepare debug info to receive this function
// record that this function name came from this linfo,
// so we can build a reverse mapping for debug-info.
bool toplevel = !jl_is_method(li->def.method);
if (!toplevel) {
const DataLayout &DL = m->getDataLayout();
// but don't remember toplevel thunks because
// they may not be rooted in the gc for the life of the program,
// and the runtime doesn't notify us when the code becomes unreachable :(
if (specf)
jl_add_linfo_in_flight(specf, li, DL);
if (strcmp(f, "jl_fptr_args") && strcmp(f, "jl_fptr_sparam"))
jl_add_linfo_in_flight(f, li, DL);
}
// Step 5. Add the result to the execution engine now
jl_finalize_module(m.release(), !toplevel);
}
if (// don't alter `inferred` when the code is not directly being used
world &&
// don't change inferred state
li->inferred) {
if (// keep code when keeping everything
!(JL_DELETE_NON_INLINEABLE) ||
// keep code when debugging level >= 2
jl_options.debug_level > 1) {
// update the stored code
if (li->inferred != (jl_value_t*)src) {
if (jl_is_method(li->def.method))
src = (jl_code_info_t*)jl_compress_ast(li->def.method, src);
li->inferred = (jl_value_t*)src;
jl_gc_wb(li, src);
}
}
else if (// don't delete toplevel code
jl_is_method(li->def.method) &&
// and there is something to delete (test this before calling jl_ast_flag_inlineable)
li->inferred != jl_nothing &&
// don't delete inlineable code, unless it is constant
(li->invoke == jl_fptr_const_return || !jl_ast_flag_inlineable((jl_array_t*)li->inferred)) &&
// don't delete code when generating a precompile file
!imaging_mode) {
// if not inlineable, code won't be needed again
li->inferred = jl_nothing;
}
}
// Step 6: Done compiling: Restore global state
nested_compile = last_n_c;
}
JL_UNLOCK(&codegen_lock); // Might GC
// If logging of the compilation stream is enabled then dump the function to the stream
// ... unless li->def isn't defined here meaning the function is a toplevel thunk and
// would have its CodeInfo printed in the stream, which might contain double-quotes that
// would not be properly escaped given the double-quotes added to the stream below.
if (dump_compiles_stream != NULL && jl_is_method(li->def.method)) {
uint64_t this_time = jl_hrtime();
jl_printf(dump_compiles_stream, "%" PRIu64 "\t\"", this_time - last_time);
jl_static_show(dump_compiles_stream, li->specTypes);
jl_printf(dump_compiles_stream, "\"\n");
last_time = this_time;
}
JL_GC_POP();
return decls;
locked_out:
JL_UNLOCK(&codegen_lock);
JL_GC_POP();
return decls;
}
JL_GCC_IGNORE_STOP
#define getModuleFlag(m,str) m->getModuleFlag(str)
static void jl_setup_module(Module *m, const jl_cgparams_t *params = &jl_default_cgparams)
{
if (JL_HOOK_TEST(params, module_setup)) {
JL_HOOK_CALL(params, module_setup, 1, jl_box_voidpointer(wrap(m)));
return;
}
// Some linkers (*cough* OS X) don't understand DWARF v4, so we use v2 in
// imaging mode. The structure of v4 is slightly nicer for debugging JIT
// code.
if (!getModuleFlag(m,"Dwarf Version")) {
int dwarf_version = 4;
#ifdef _OS_DARWIN_
if (imaging_mode)
dwarf_version = 2;
#endif
m->addModuleFlag(llvm::Module::Warning, "Dwarf Version", dwarf_version);
}
if (!getModuleFlag(m,"Debug Info Version"))
m->addModuleFlag(llvm::Module::Error, "Debug Info Version",
llvm::DEBUG_METADATA_VERSION);
m->setDataLayout(jl_data_layout);
m->setTargetTriple(jl_TargetMachine->getTargetTriple().str());
}
// this ensures that llvmf has been emitted to the execution engine,
// returning the function pointer to it
extern void jl_callback_triggered_linfos(void);
static uint64_t getAddressForFunction(StringRef fname)
{
JL_TIMING(LLVM_EMIT);
if (fname == "jl_fptr_args")
return (uintptr_t)&jl_fptr_args;
else if (fname == "jl_fptr_sparam")
return (uintptr_t)&jl_fptr_sparam;
#ifdef JL_DEBUG_BUILD
llvm::raw_fd_ostream out(1, false);
#endif
jl_finalize_function(fname);
uint64_t ret = jl_ExecutionEngine->getFunctionAddress(fname);
// delay executing trace callbacks until here to make sure there's no
// recursive compilation.
jl_callback_triggered_linfos();
return ret;
}
// convenience helper exported for usage from gdb
extern "C" JL_DLLEXPORT
uint64_t jl_get_llvm_fptr(void *function)
{
Function *F = (Function*)function;
uint64_t addr = getAddressForFunction(F->getName());
if (!addr) {
if (auto exp_addr = jl_ExecutionEngine->findUnmangledSymbol(F->getName()).getAddress()) {
addr = exp_addr.get();
}
}
return addr;
}
static jl_method_instance_t *jl_get_unspecialized(jl_method_instance_t *method)
{
// one unspecialized version of a function can be shared among all cached specializations
jl_method_t *def = method->def.method;
if (def->source == NULL) {
return method;
}
if (def->unspecialized == NULL) {
JL_LOCK(&def->writelock);
if (def->unspecialized == NULL) {
def->unspecialized = jl_get_specialized(def, def->sig, jl_emptysvec);
jl_gc_wb(def, def->unspecialized);
}
JL_UNLOCK(&def->writelock);
}
return def->unspecialized;
}
// this compiles li and emits fptr
extern "C"
jl_callptr_t jl_generate_fptr(jl_method_instance_t **pli, jl_llvm_functions_t decls, size_t world)
{
jl_method_instance_t *li = *pli;
jl_callptr_t fptr;
fptr = li->invoke;
if (fptr != jl_fptr_trampoline)
return fptr;
JL_LOCK(&codegen_lock);
fptr = li->invoke;
if (fptr != jl_fptr_trampoline) {
JL_UNLOCK(&codegen_lock);
return fptr;
}
jl_method_instance_t *unspec = NULL;
if (jl_is_method(li->def.method)) {
if (li->def.method->unspecialized) {
unspec = li->def.method->unspecialized;
}
const char *F = decls.functionObject;
const char *specF = decls.specFunctionObject;
if (!F || !jl_can_finalize_function(F) || (specF && !jl_can_finalize_function(specF))) {
// can't compile F in the JIT right now,
// so instead compile an unspecialized version
// and return its fptr instead
if (!unspec)
unspec = jl_get_unspecialized(li); // get-or-create the unspecialized version to cache the result
jl_code_info_t *src = (jl_code_info_t*)unspec->def.method->source;
if (src == NULL) {
assert(unspec->def.method->generator);
src = jl_code_for_staged(unspec);
}
fptr = unspec->invoke;
if (fptr != jl_fptr_trampoline) {
li->specptr = unspec->specptr;
li->inferred_const = unspec->inferred_const;
if (li->inferred_const)
jl_gc_wb(li, li->inferred_const);
li->invoke = fptr;
JL_UNLOCK(&codegen_lock);
return fptr;
}
if (unspec == li) {
// discard decls so that this generated function will get compiled
// and cached permanently without optimizations
unspec->functionObjectsDecls.functionObject = NULL;
unspec->functionObjectsDecls.specFunctionObject = NULL;
}
assert(src);
decls = jl_compile_linfo(&unspec, src, unspec->min_world, &jl_default_cgparams); // this does not change unspec
li = unspec;
}
}
const char *F = decls.functionObject;
const char *specF = decls.specFunctionObject;
assert(F && jl_can_finalize_function(F));
assert(specF && jl_can_finalize_function(specF));
if (!strcmp(F, "jl_fptr_args"))
fptr = &jl_fptr_args;
else if (!strcmp(F, "jl_fptr_sparam"))
fptr = &jl_fptr_sparam;
else
fptr = (jl_callptr_t)(uintptr_t)getAddressForFunction(F);
assert(fptr != NULL);
void *specptr = (void*)(uintptr_t)getAddressForFunction(specF);
assert(specptr != NULL);
// the fptr should be cached somewhere also
if (li->invoke == jl_fptr_trampoline) {
// once set, don't change invoke-ptr, as that leads to race conditions
// with the (not) simultaneous updates to invoke and specptr
li->specptr.fptr = specptr;
li->invoke = fptr;
}
if (li != *pli) {
assert(unspec);
li = *pli;
li->specptr = unspec->specptr;
li->inferred_const = unspec->inferred_const;
if (li->inferred_const)
jl_gc_wb(li, li->inferred_const);
li->invoke = fptr;
}
JL_UNLOCK(&codegen_lock); // Might GC
return fptr;
}
static Function *jl_cfunction_object(jl_value_t *f, jl_value_t *declrt, jl_tupletype_t *argt);
// get the address of a C-callable entry point for a function
extern "C" JL_DLLEXPORT
void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t *argt)
{
JL_LOCK(&codegen_lock);
Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt);
void *ptr = (void*)getAddressForFunction(llvmf->getName());
JL_UNLOCK(&codegen_lock);
return ptr;
}
// convenience function for debugging from gdb (pre-OrcJIT)
// it generally helps to have define KEEP_BODIES if you plan on using this
extern "C" JL_DLLEXPORT
void *jl_function_ptr_by_llvm_name(char *name) {
#ifdef JL_MSAN_ENABLED
__msan_unpoison_string(name);
#endif
return (void*)jl_ExecutionEngine->FindFunctionNamed(name); // returns an llvm::Function*
}
// export a C-callable entry point for a function (dllexport'ed dlsym), with a given name
extern "C" JL_DLLEXPORT
void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name)
{
JL_LOCK(&codegen_lock);
Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt);
// force eager emission of the function (llvm 3.3 gets confused otherwise and tries to do recursive compilation)
uint64_t Addr = getAddressForFunction(llvmf->getName());
if (imaging_mode)
llvmf = cast<Function>(shadow_output->getNamedValue(llvmf->getName()));
// make the alias to the shadow_module
GlobalAlias *GA =
GlobalAlias::create(llvmf->getType()->getElementType(), llvmf->getType()->getAddressSpace(),
GlobalValue::ExternalLinkage, name, llvmf, shadow_output);
// make sure the alias name is valid for the current session
jl_ExecutionEngine->addGlobalMapping(GA, (void*)(uintptr_t)Addr);
JL_UNLOCK(&codegen_lock);
}
// --- native code info, and dump function to IR and ASM ---
// Get pointer to llvm::Function instance, compiling if necessary
// for use in reflection from Julia.
// this is paired with jl_dump_function_ir and jl_dump_function_asm in particular ways:
// misuse will leak memory or cause read-after-free
extern "C" JL_DLLEXPORT
void *jl_get_llvmf_defn(jl_method_instance_t *linfo, size_t world, bool getwrapper, bool optimize, const jl_cgparams_t params)
{
if (jl_is_method(linfo->def.method) && linfo->def.method->source == NULL &&
linfo->def.method->generator == NULL) {
// not a generic function
return NULL;
}
jl_code_info_t *src = (jl_code_info_t*)linfo->inferred;
JL_GC_PUSH1(&src);
if (!src || (jl_value_t*)src == jl_nothing) {
src = jl_type_infer(&linfo, world, 0);
if (!src && jl_is_method(linfo->def.method))
src = linfo->def.method->generator ? jl_code_for_staged(linfo) : (jl_code_info_t*)linfo->def.method->source;
}
if ((jl_value_t*)src == jl_nothing)
src = NULL;
if (src && !jl_is_code_info(src) && jl_is_method(linfo->def.method))
src = jl_uncompress_ast(linfo->def.method, (jl_array_t*)src);
if (src && !jl_is_code_info(src))
src = NULL;
if (!src)
jl_error("source not found for function");
// Backup the info for the nested compile
JL_LOCK(&codegen_lock);
// emit this function into a new module
jl_llvm_functions_t declarations;
std::unique_ptr<Module> m;
JL_TRY {
m = emit_function(linfo, src, world, &declarations, &params);
}
JL_CATCH {
// something failed!
m.reset();
JL_UNLOCK(&codegen_lock); // Might GC
const char *mname = name_from_method_instance(linfo);
jl_rethrow_with_add("error compiling %s", mname);
}
if (optimize)
jl_globalPM->run(*m.get());
// swap declarations for definitions and destroy declarations
const char *fname = declarations.functionObject;
const char *specfname = declarations.specFunctionObject;
Function *f = NULL;
Function *specf = NULL;
if (specfname) {
specf = cast<Function>(m->getNamedValue(specfname));
free(const_cast<char*>(specfname));
}
f = cast_or_null<Function>(m->getNamedValue(fname));
if (f) // don't try to free sentinel names like "jl_fptr_args" and "jl_fptr_sparam"
free(const_cast<char*>(fname));
assert(specf || f);
// clone the name from the runtime linfo, if it exists
// to give the user a (false) sense of stability
specfname = linfo->functionObjectsDecls.specFunctionObject;
if (specfname && specf) {
specf->setName(specfname);
}
fname = linfo->functionObjectsDecls.functionObject;
if (fname && f && strcmp(fname, "jl_fptr_args") && strcmp(fname, "jl_fptr_sparam")) {
f->setName(fname);
}
m.release(); // the return object `llvmf` will be the owning pointer
JL_UNLOCK(&codegen_lock); // Might GC
JL_GC_POP();
if ((getwrapper && f) || !specf)
return f;
else
return specf;
}
extern "C" JL_DLLEXPORT
void *jl_get_llvmf_decl(jl_method_instance_t *linfo, size_t world, bool getwrapper, const jl_cgparams_t params)
{
if (jl_is_method(linfo->def.method) && linfo->def.method->source == NULL &&
linfo->def.method->generator == NULL) {
// not a generic function
return NULL;
}
// compile this normally
jl_code_info_t *src = NULL;
if (linfo->inferred == NULL)
src = jl_type_infer(&linfo, world, 0);
jl_llvm_functions_t decls = jl_compile_linfo(&linfo, src, world, &jl_default_cgparams);
if (decls.functionObject == NULL && linfo->invoke == jl_fptr_const_return && jl_is_method(linfo->def.method)) {
// normally we don't generate native code for these functions, so need an exception here
// This leaks a bit of memory to cache native code that we'll never actually need
JL_LOCK(&codegen_lock);
decls = linfo->functionObjectsDecls;
if (decls.functionObject == NULL) {
src = jl_type_infer(&linfo, world, 0);
if (!src) {
src = linfo->def.method->generator ? jl_code_for_staged(linfo) : (jl_code_info_t*)linfo->def.method->source;
}
decls = jl_compile_linfo(&linfo, src, world, &params);
linfo->functionObjectsDecls = decls;
}
JL_UNLOCK(&codegen_lock);
}
if (!getwrapper && decls.specFunctionObject) {
if (!strcmp(decls.functionObject, "jl_fptr_args")) {
auto f = Function::Create(jl_func_sig, GlobalVariable::ExternalLinkage, decls.specFunctionObject);
add_return_attr(f, Attribute::NonNull);
f->addFnAttr(Thunk);
return f;
}
else if (!strcmp(decls.functionObject, "jl_fptr_sparam")) {
auto f = Function::Create(jl_func_sig_sparams, GlobalVariable::ExternalLinkage, decls.specFunctionObject);
add_return_attr(f, Attribute::NonNull);
f->addFnAttr(Thunk);
return f;
}
else {
jl_returninfo_t returninfo = get_specsig_function(NULL, decls.specFunctionObject, linfo->specTypes, linfo->rettype);
return returninfo.decl;
}
}
auto f = Function::Create(jl_func_sig, GlobalVariable::ExternalLinkage, decls.functionObject);
add_return_attr(f, Attribute::NonNull);
f->addFnAttr(Thunk);
return f;
}
// get a native disassembly for f (an LLVM function)
// warning: this takes ownership of, and destroys, f
extern "C" JL_DLLEXPORT
const jl_value_t *jl_dump_function_asm(void *f, int raw_mc, const char* asm_variant)
{
Function *llvmf = dyn_cast_or_null<Function>((Function*)f);
if (!llvmf)
jl_error("jl_dump_function_asm: Expected Function*");
uint64_t fptr = getAddressForFunction(llvmf->getName());
// Look in the system image as well
if (fptr == 0)
fptr = (uintptr_t)jl_ExecutionEngine->getPointerToGlobalIfAvailable(llvmf);
delete llvmf;
return jl_dump_fptr_asm(fptr, raw_mc, asm_variant);
}
// Logging for code coverage and memory allocation
const int logdata_blocksize = 32; // target getting nearby lines in the same general cache area and reducing calls to malloc by chunking
typedef uint64_t logdata_block[logdata_blocksize];
typedef StringMap< std::vector<logdata_block*> > logdata_t;
static void visitLine(jl_codectx_t &ctx, std::vector<logdata_block*> &vec, int line, Value *addend, const char* name)
{
unsigned block = line / logdata_blocksize;
line = line % logdata_blocksize;
if (vec.size() <= block)
vec.resize(block + 1);
if (vec[block] == NULL) {
vec[block] = (logdata_block*)calloc(1, sizeof(logdata_block));
}
logdata_block &data = *vec[block];
if (data[line] == 0)
data[line] = 1;
Value *v = ConstantExpr::getIntToPtr(
ConstantInt::get(T_size, (uintptr_t)&data[line]),
T_pint64);
ctx.builder.CreateStore(ctx.builder.CreateAdd(ctx.builder.CreateLoad(v, true, name),
addend),
v, true); // not atomic, so this might be an underestimate,
// but it's faster this way
}
// Code coverage
static logdata_t coverageData;
static void coverageVisitLine(jl_codectx_t &ctx, StringRef filename, int line)
{
assert(!imaging_mode);
if (filename == "" || filename == "none" || filename == "no file" || filename == "<missing>" || line < 0)
return;
visitLine(ctx, coverageData[filename], line, ConstantInt::get(T_int64, 1), "lcnt");
}
// Memory allocation log (malloc_log)
static logdata_t mallocData;
static void mallocVisitLine(jl_codectx_t &ctx, StringRef filename, int line)
{
assert(!imaging_mode);
if (filename == "" || filename == "none" || filename == "no file" || filename == "<missing>" || line < 0) {
jl_gc_sync_total_bytes();
return;
}
Value *addend = ctx.builder.CreateCall(prepare_call(diff_gc_total_bytes_func), {});
visitLine(ctx, mallocData[filename], line, addend, "bytecnt");
}
// Resets the malloc counts. Needed to avoid including memory usage
// from JITting.
extern "C" JL_DLLEXPORT void jl_clear_malloc_data(void)
{
logdata_t::iterator it = mallocData.begin();
for (; it != mallocData.end(); it++) {
std::vector<logdata_block*> &bytes = (*it).second;
std::vector<logdata_block*>::iterator itb;
for (itb = bytes.begin(); itb != bytes.end(); itb++) {
if (*itb) {
logdata_block &data = **itb;
for (int i = 0; i < logdata_blocksize; i++) {
if (data[i] > 0)
data[i] = 1;
}
}
}
}
jl_gc_sync_total_bytes();
}
extern "C" int isabspath(const char *in);
static void write_log_data(logdata_t &logData, const char *extension)
{
std::string base = std::string(jl_options.julia_bindir);
base = base + "/../share/julia/base/";
logdata_t::iterator it = logData.begin();
for (; it != logData.end(); it++) {
std::string filename = it->first();
std::vector<logdata_block*> &values = it->second;
if (!values.empty()) {
if (!isabspath(filename.c_str()))
filename = base + filename;
std::ifstream inf(filename.c_str());
if (inf.is_open()) {
std::string outfile = filename + extension;
std::ofstream outf(outfile.c_str(), std::ofstream::trunc | std::ofstream::out);
char line[1024];
int l = 1;
unsigned block = 0;
while (!inf.eof()) {
inf.getline(line, sizeof(line));
if (inf.fail() && !inf.bad()) {
// Read through lines longer than sizeof(line)
inf.clear();
inf.ignore(std::numeric_limits<std::streamsize>::max(), '\n');
}
logdata_block *data = NULL;
if (block < values.size()) {
data = values[block];
}
uint64_t value = data ? (*data)[l] : 0;
if (++l >= logdata_blocksize) {
l = 0;
block++;
}
outf.width(9);
if (value == 0)
outf << '-';
else
outf << (value - 1);
outf.width(0);
outf << " " << line << std::endl;
}
outf.close();
inf.close();
}
}
}
}
extern "C" int jl_getpid();
extern "C" void jl_write_coverage_data(void)
{
std::ostringstream stm;
stm << jl_getpid();
std::string outf = "." + stm.str() + ".cov";
write_log_data(coverageData, outf.c_str());
}
extern "C" void jl_write_malloc_log(void)
{
write_log_data(mallocData, ".mem");
}
// --- constant determination ---
static void show_source_loc(jl_codectx_t &ctx, JL_STREAM *out)
{
jl_printf(out, "in %s at %s", ctx.name, ctx.file.str().c_str());
}
extern "C" void jl_binding_deprecation_warning(jl_module_t *m, jl_binding_t *b);
static void cg_bdw(jl_codectx_t &ctx, jl_binding_t *b)
{
jl_binding_deprecation_warning(ctx.module, b);
if (b->deprecated == 1 && jl_options.depwarn) {
show_source_loc(ctx, JL_STDERR);
jl_printf(JL_STDERR, "\n");
}
}
static jl_value_t *static_apply_type(jl_codectx_t &ctx, const jl_cgval_t *args, size_t nargs)
{
jl_value_t **v = (jl_value_t**)alloca(sizeof(jl_value_t*) * nargs);
for (size_t i = 0; i < nargs; i++) {
if (!args[i].constant)
return NULL;
v[i] = args[i].constant;
}
assert(v[0] == jl_builtin_apply_type);
size_t last_age = jl_get_ptls_states()->world_age;
// call apply_type, but ignore errors. we know that will work in world 1.
jl_get_ptls_states()->world_age = 1;
jl_value_t *result = jl_apply_with_saved_exception_state(v, nargs, 1);
jl_get_ptls_states()->world_age = last_age;
return result;
}
// try to statically evaluate, NULL if not possible
static jl_value_t *static_eval(jl_codectx_t &ctx, jl_value_t *ex, int sparams=true, int allow_alloc=true)
{
if (!JL_FEAT_TEST(ctx, static_alloc)) allow_alloc = 0;
if (jl_is_symbol(ex)) {
jl_sym_t *sym = (jl_sym_t*)ex;
if (jl_is_const(ctx.module, sym))
return jl_get_global(ctx.module, sym);
return NULL;
}
if (jl_is_slot(ex))
return NULL;
if (jl_is_ssavalue(ex)) {
ssize_t idx = ((jl_ssavalue_t*)ex)->id - 1;
assert(idx >= 0);
if (ctx.ssavalue_assigned.at(idx)) {
return ctx.SAvalues.at(idx).constant;
}
return NULL;
}
if (jl_is_quotenode(ex))
return jl_fieldref(ex, 0);
if (jl_is_method_instance(ex))
return NULL;
jl_module_t *m = NULL;
jl_sym_t *s = NULL;
if (jl_is_globalref(ex)) {
s = jl_globalref_name(ex);
jl_binding_t *b = jl_get_binding(jl_globalref_mod(ex), s);
if (b && b->constp) {
if (b->deprecated)
cg_bdw(ctx, b);
return b->value;
}
return NULL;
}
if (jl_is_expr(ex)) {
jl_expr_t *e = (jl_expr_t*)ex;
if (e->head == call_sym) {
jl_value_t *f = static_eval(ctx, jl_exprarg(e, 0), sparams, allow_alloc);
if (f) {
if (jl_array_dim0(e->args) == 3 && f == jl_builtin_getfield) {
m = (jl_module_t*)static_eval(ctx, jl_exprarg(e, 1), sparams, allow_alloc);
// Check the tag before evaluating `s` so that a value of random
// type won't be corrupted.
if (!m || !jl_is_module(m))
return NULL;
// Assumes that the module is rooted somewhere.
s = (jl_sym_t*)static_eval(ctx, jl_exprarg(e, 2), sparams, allow_alloc);
if (s && jl_is_symbol(s)) {
jl_binding_t *b = jl_get_binding(m, s);
if (b && b->constp) {
if (b->deprecated)
cg_bdw(ctx, b);
return b->value;
}
}
}
else if (f==jl_builtin_tuple || f==jl_builtin_apply_type) {
size_t i;
size_t n = jl_array_dim0(e->args)-1;
if (n==0 && f==jl_builtin_tuple) return (jl_value_t*)jl_emptytuple;
if (!allow_alloc)
return NULL;
jl_value_t **v;
JL_GC_PUSHARGS(v, n+1);
v[0] = f;
for (i = 0; i < n; i++) {
v[i+1] = static_eval(ctx, jl_exprarg(e, i+1), sparams, allow_alloc);
if (v[i+1] == NULL) {
JL_GC_POP();
return NULL;
}
}
size_t last_age = jl_get_ptls_states()->world_age;
// here we know we're calling specific builtin functions that work in world 1.
jl_get_ptls_states()->world_age = 1;
jl_value_t *result = jl_apply_with_saved_exception_state(v, n+1, 1);
jl_get_ptls_states()->world_age = last_age;
JL_GC_POP();
return result;
}
}
}
else if (e->head == static_parameter_sym) {
size_t idx = jl_unbox_long(jl_exprarg(e, 0));
if (idx <= jl_svec_len(ctx.linfo->sparam_vals)) {
jl_value_t *e = jl_svecref(ctx.linfo->sparam_vals, idx - 1);
if (jl_is_typevar(e))
return NULL;
return e;
}
}
return NULL;
}
return ex;
}
static bool slot_eq(jl_value_t *e, int sl)
{
return jl_is_slot(e) && jl_slot_number(e)-1 == sl;
}
// --- code gen for intrinsic functions ---
#include "intrinsics.cpp"
// --- find volatile variables ---
// assigned in a try block and used outside that try block
static bool local_var_occurs(jl_value_t *e, int sl)
{
if (slot_eq(e, sl)) {
return true;
}
else if (jl_is_expr(e)) {
jl_expr_t *ex = (jl_expr_t*)e;
size_t alength = jl_array_dim0(ex->args);
for(int i=0; i < (int)alength; i++) {
if (local_var_occurs(jl_exprarg(ex,i),sl))
return true;
}
}
return false;
}
static std::set<int> assigned_in_try(jl_array_t *stmts, int s, long l)
{
std::set<int> av;
for(int i=s; i <= l; i++) {
jl_value_t *st = jl_array_ptr_ref(stmts,i);
if (jl_is_expr(st)) {
if (((jl_expr_t*)st)->head == assign_sym) {
jl_value_t *ar = jl_exprarg(st, 0);
if (jl_is_slot(ar)) {
av.insert(jl_slot_number(ar)-1);
}
}
}
}
return av;
}
static void mark_volatile_vars(jl_array_t *stmts, std::vector<jl_varinfo_t> &slots)
{
size_t slength = jl_array_dim0(stmts);
for (int i = 0; i < (int)slength; i++) {
jl_value_t *st = jl_array_ptr_ref(stmts, i);
if (jl_is_expr(st)) {
if (((jl_expr_t*)st)->head == enter_sym) {
int last = jl_unbox_long(jl_exprarg(st, 0));
std::set<int> as = assigned_in_try(stmts, i + 1, last);
for (int j = 0; j < (int)slength; j++) {
if (j < i || j > last) {
std::set<int>::iterator it = as.begin();
for (; it != as.end(); it++) {
if (local_var_occurs(jl_array_ptr_ref(stmts, j), *it)) {
jl_varinfo_t &vi = slots[*it];
vi.isVolatile = true;
}
}
}
}
}
}
}
}
// --- use analysis ---
// a very simple, conservative use analysis
// to eagerly remove slot assignments that are never read from
static void simple_use_analysis(jl_codectx_t &ctx, jl_value_t *expr)
{
if (jl_is_slot(expr)) {
int i = jl_slot_number(expr) - 1;
ctx.slots[i].used = true;
}
else if (jl_is_expr(expr)) {
jl_expr_t *e = (jl_expr_t*)expr;
if (e->head == method_sym) {
simple_use_analysis(ctx, jl_exprarg(e, 0));
if (jl_expr_nargs(e) > 1) {
simple_use_analysis(ctx, jl_exprarg(e, 1));
simple_use_analysis(ctx, jl_exprarg(e, 2));
}
}
else if (e->head == assign_sym) {
// don't consider assignment LHS as a variable "use"
simple_use_analysis(ctx, jl_exprarg(e, 1));
}
else {
size_t i, elen = jl_array_dim0(e->args);
for (i = 0; i < elen; i++) {
simple_use_analysis(ctx, jl_exprarg(e, i));
}
}
}
else if (jl_is_pinode(expr)) {
simple_use_analysis(ctx, jl_fieldref_noalloc(expr, 0));
}
else if (jl_is_upsilonnode(expr)) {
jl_value_t *val = jl_fieldref_noalloc(expr, 0);
if (val)
simple_use_analysis(ctx, val);
}
else if (jl_is_phicnode(expr)) {
jl_array_t *values = (jl_array_t*)jl_fieldref_noalloc(expr, 0);
size_t i, elen = jl_array_len(values);
for (i = 0; i < elen; i++) {
jl_value_t *v = jl_array_ptr_ref(values, i);
simple_use_analysis(ctx, v);
}
}
else if (jl_is_phinode(expr)) {
jl_array_t *values = (jl_array_t*)jl_fieldref_noalloc(expr, 1);
size_t i, elen = jl_array_len(values);
for (i = 0; i < elen; i++) {
jl_value_t *v = jl_array_ptr_ref(values, i);
if (v)
simple_use_analysis(ctx, v);
}
}
}
// --- gc root utils ---
// ---- Get Element Pointer (GEP) instructions within the GC frame ----
static void jl_add_method_root(jl_codectx_t &ctx, jl_value_t *val)
{
if (jl_is_concrete_type(val) || jl_is_bool(val) || jl_is_symbol(val) || val == jl_nothing ||
val == (jl_value_t*)jl_any_type || val == (jl_value_t*)jl_bottom_type || val == (jl_value_t*)jl_core_module)
return;
JL_GC_PUSH1(&val);
if (ctx.roots == NULL) {
ctx.roots = jl_alloc_vec_any(1);
jl_array_ptr_set(ctx.roots, 0, val);
}
else {
size_t rlen = jl_array_dim0(ctx.roots);
for (size_t i = 0; i < rlen; i++) {
if (jl_array_ptr_ref(ctx.roots,i) == val) {
JL_GC_POP();
return;
}
}
jl_array_ptr_1d_push(ctx.roots, val);
}
JL_GC_POP();
}
// --- generating function calls ---
static jl_cgval_t emit_globalref(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t *name)
{
jl_binding_t *bnd = NULL;
Value *bp = global_binding_pointer(ctx, mod, name, &bnd, false);
// TODO: refactor. this partially duplicates code in emit_var
if (bnd && bnd->value != NULL) {
if (bnd->constp) {
return mark_julia_const(bnd->value);
}
return mark_julia_type(ctx, tbaa_decorate(tbaa_binding, ctx.builder.CreateLoad(bp)), true, (jl_value_t*)jl_any_type);
}
// todo: use type info to avoid undef check