diff --git a/src/runtime/vm/translator/hopt/codegen.cpp b/src/runtime/vm/translator/hopt/codegen.cpp index 3ac7dbc6a69d6..50f2ac28fa2c5 100644 --- a/src/runtime/vm/translator/hopt/codegen.cpp +++ b/src/runtime/vm/translator/hopt/codegen.cpp @@ -3223,29 +3223,8 @@ Address CodeGenerator::cgLoadTypedValue(Type::Tag type, } // Check type if needed - if (label && type != Type::Gen) { - ConditionCode cc; - switch (type) { - case Type::Cell : { - m_as.cmp_imm32_disp_reg32(HPHP::KindOfRef, off + TVOFF(m_type), base); - cc = CC_GE; - break; - } - case Type::Uncounted : { - m_as.cmp_imm32_disp_reg32(HPHP::KindOfStaticString, - off + TVOFF(m_type), base); - cc = CC_G; - break; - } - case Type::UncountedInit : { - m_as.test_imm32_disp_reg32(HPHP::KindOfUncountedInitBit, - off + TVOFF(m_type), base); - cc = CC_Z; - break; - } - default : not_reached(); - } - emitGuardOrFwdJcc(inst, cc, label); + if (label) { + cgGuardType(type, base, off, label, inst); } // Load type if it's not dead @@ -3371,18 +3350,7 @@ Address CodeGenerator::cgLoad(Type::Tag type, return cgLoadTypedValue(type, dst, base, off, label, inst); } if (label != NULL && type != Type::Home) { - // generate a guard for the type - // see Translator.cpp checkType - DataType dataType = Type::toDataType(type); - ConditionCode cc; - if (IS_STRING_TYPE(dataType)) { - m_as.test_imm32_disp_reg32(KindOfStringBit, off + TVOFF(m_type), base); - cc = CC_Z; - } else { - m_as.cmp_imm32_disp_reg32(dataType, off + TVOFF(m_type), base); - cc = CC_NE; - } - emitGuardOrFwdJcc(inst, cc, label); + cgGuardType(type, base, off, label, inst); } if (type == Type::Uninit || type == Type::Null) { return start; // these are constants @@ -3473,9 +3441,9 @@ static void getLocalRegOffset(SSATmp* src, PhysReg& reg, int64& off) { Address CodeGenerator::cgLdLoc(IRInstruction* inst) { Address start = m_as.code.frontier; - Type::Tag type = inst->getType(); - SSATmp* dst = inst->getDst(); - LabelInstruction* label = inst->getLabel(); + Type::Tag type = inst->getType(); + SSATmp* dst = inst->getDst(); + LabelInstruction* label = inst->getLabel(); PhysReg fpReg; int64 offset; @@ -3520,6 +3488,83 @@ Address CodeGenerator::cgLdStack(IRInstruction* inst) { inst); } +Address CodeGenerator::cgGuardType(Type::Tag type, + PhysReg baseReg, + int64_t offset, + LabelInstruction* label, + IRInstruction* inst) { + ASSERT(label); + Address start = m_as.code.frontier; + int64_t typeOffset = offset + TVOFF(m_type); + ConditionCode cc; + + switch (type) { + case Type::StaticStr : + case Type::Str : { + m_as.test_imm32_disp_reg32(KindOfStringBit, typeOffset, baseReg); + cc = CC_Z; + break; + } + case Type::UncountedInit : { + m_as.test_imm32_disp_reg32(KindOfUncountedInitBit, typeOffset, baseReg); + cc = CC_Z; + break; + } + case Type::Uncounted : { + m_as.cmp_imm32_disp_reg32(KindOfRefCountThreshold, typeOffset, baseReg); + cc = CC_G; + break; + } + case Type::Cell : { + m_as.cmp_imm32_disp_reg32(KindOfRef, typeOffset, baseReg); + cc = CC_GE; + break; + } + case Type::Gen : { + return start; // nothing to check + } + default: { + DataType dataType = Type::toDataType(type); + ASSERT(dataType >= KindOfUninit); + m_as.cmp_imm32_disp_reg32(dataType, typeOffset, baseReg); + cc = CC_NZ; + break; + } + } + + emitGuardOrFwdJcc(inst, cc, label); + + return start; +} + +Address CodeGenerator::cgGuardStk(IRInstruction* inst) { + Type::Tag type = inst->getType(); + SSATmp* sp = inst->getSrc(0); + SSATmp* index = inst->getSrc(1); + LabelInstruction* label = inst->getLabel(); + + ASSERT(index->isConst()); + + return cgGuardType(type, + sp->getReg(0), + index->getConstValAsInt() * sizeof(Cell), + label, + inst); +} + +Address CodeGenerator::cgGuardLoc(IRInstruction* inst) { + Type::Tag type = inst->getType(); + SSATmp* index = inst->getSrc(0); + LabelInstruction* label = inst->getLabel(); + PhysReg fpReg; + int64 offset; + getLocalRegOffset(index, fpReg, offset); + ASSERT(fpReg == HPHP::VM::Transl::reg::rbp); + + return cgGuardType(type, fpReg, offset, label, inst); +} + + Address CodeGenerator::cgGuardType(IRInstruction* inst) { Address start = m_as.code.frontier; UNUSED Type::Tag type = inst->getType(); diff --git a/src/runtime/vm/translator/hopt/codegen.h b/src/runtime/vm/translator/hopt/codegen.h index b87f365392b14..b11ce54922ce3 100644 --- a/src/runtime/vm/translator/hopt/codegen.h +++ b/src/runtime/vm/translator/hopt/codegen.h @@ -102,6 +102,12 @@ class CodeGenerator { LabelInstruction* label, IRInstruction* inst = NULL); + Address cgGuardType(Type::Tag type, + PhysReg baseReg, + int64_t offset, + LabelInstruction* label, + IRInstruction* instr); + Address cgStMemWork(IRInstruction* inst, bool genStoreType); Address cgStRefWork(IRInstruction* inst, bool genStoreType); Address cgStLocWork(IRInstruction* inst, bool genStoreType); diff --git a/src/runtime/vm/translator/hopt/dce.cpp b/src/runtime/vm/translator/hopt/dce.cpp index ac1aee5aa10ff..9712d6dbd0f88 100644 --- a/src/runtime/vm/translator/hopt/dce.cpp +++ b/src/runtime/vm/translator/hopt/dce.cpp @@ -409,15 +409,19 @@ void eliminateDeadCode(Trace* trace, IRFactory* irFactory) { } } - // If main trace starts with guards, have them generate a patchable jump to the anchor trace + // If main trace starts with guards, have them generate a patchable jump + // to the anchor trace if (RuntimeOption::EvalHHIRDirectExit) { LabelInstruction* guardLabel = NULL; IRInstruction::List& instList = trace->getInstructionList(); // Check the beginning of the trace for guards - for (IRInstruction::Iterator it = instList.begin(); it != instList.end(); ++it) { + for (IRInstruction::Iterator it = instList.begin(); it != instList.end(); + ++it) { IRInstruction* inst = *it; Opcode opc = inst->getOpcode(); - if ((opc == LdLoc || opc == LdStack) && inst->getLabel()) { + if (inst->getLabel() && + (opc == LdLoc || opc == LdStack || + opc == GuardLoc || opc == GuardStk)) { LabelInstruction* exitLabel = inst->getLabel(); // Find the GuardFailure's label and confirm this branches there if (guardLabel == NULL) { diff --git a/src/runtime/vm/translator/hopt/hhbctranslator.cpp b/src/runtime/vm/translator/hopt/hhbctranslator.cpp index 698324ea8b401..3bb656f87669f 100644 --- a/src/runtime/vm/translator/hopt/hhbctranslator.cpp +++ b/src/runtime/vm/translator/hopt/hhbctranslator.cpp @@ -1425,7 +1425,8 @@ Trace* HhbcTranslator::guardTypeLocal(uint32 localIndex, if (nextTrace == NULL) { nextTrace = getGuardExit(); } - m_tb.genLdLoc(localIndex, type, nextTrace); + m_tb.genGuardLoc(localIndex, type, nextTrace); + m_typeGuards.push_back(TypeGuard(TypeGuard::Local, localIndex, type)); return nextTrace; } @@ -1438,6 +1439,8 @@ void HhbcTranslator::checkTypeLocal(uint32 localIndex, Type::Tag type) { } void HhbcTranslator::assertTypeLocal(uint32 localIndex, Type::Tag type) { + // Type assertions are currently implemented as loads. If the value doesn't + // get used, DCE gets rid of it. m_tb.genLdLoc(localIndex, type, NULL); } @@ -1462,7 +1465,9 @@ Trace* HhbcTranslator::guardTypeStack(uint32 stackIndex, if (nextTrace == NULL) { nextTrace = getGuardExit(); } - checkTypeStackAux(stackIndex, type, nextTrace); + m_tb.genGuardStk(stackIndex, type, nextTrace); + m_typeGuards.push_back(TypeGuard(TypeGuard::Stack, stackIndex, type)); + return nextTrace; } @@ -1474,12 +1479,31 @@ void HhbcTranslator::checkTypeStack(uint32 stackIndex, } void HhbcTranslator::assertTypeStack(uint32 stackIndex, Type::Tag type) { + // Type assertions are currently implemented as loads. If the value doesn't + // get used, DCE gets rid of it. + loadStack(stackIndex, type); +} + +void HhbcTranslator::loadStack(uint32 stackIndex, Type::Tag type) { ASSERT(stackIndex != (uint32)-1); // top() generates the LdStack if necessary, and sets 'type' accordingly SSATmp* tmp = top(type, stackIndex); replace(stackIndex, tmp); } +void HhbcTranslator::emitLoadDeps() { + for (auto guard : m_typeGuards) { + uint32 index = guard.getIndex(); + Type::Tag type = guard.getType(); + + if (guard.getKind() == TypeGuard::Local) { + m_tb.genLdLoc(index, type, NULL); + } else { + loadStack(index, type); + } + } +} + Trace* HhbcTranslator::guardRefs(int64 entryArDelta, const vector& mask, const vector& vals, diff --git a/src/runtime/vm/translator/hopt/hhbctranslator.h b/src/runtime/vm/translator/hopt/hhbctranslator.h index ca18972a9d103..36cbb84990c43 100644 --- a/src/runtime/vm/translator/hopt/hhbctranslator.h +++ b/src/runtime/vm/translator/hopt/hhbctranslator.h @@ -81,6 +81,29 @@ class FpiStack { std::stack stack; }; +class TypeGuard { + public: + enum Kind { + Local, + Stack + }; + + TypeGuard(Kind kind, uint32 index, Type::Tag type) + : m_kind(kind) + , m_index(index) + , m_type(type) { + } + + Kind getKind() const { return m_kind; } + uint32 getIndex() const { return m_index; } + Type::Tag getType() const { return m_type; } + + private: + Kind m_kind; + uint32 m_index; + Type::Tag m_type; +}; + class HhbcTranslator { public: HhbcTranslator(TraceBuilder& builder, const Func* func) @@ -316,6 +339,8 @@ class HhbcTranslator { void setThisAvailable(); + void emitLoadDeps(); + private: /* @@ -345,6 +370,8 @@ class HhbcTranslator { void emitInterpOneOrPunt(Type::Tag type, Trace* target = NULL); void emitBinaryArith(Opcode); void checkTypeStackAux(uint32 stackIndex, Type::Tag type, Trace* nextTrace); + void loadStack(uint32 stackIndex, Type::Tag type); + /* * Accessors for the current function being compiled and its @@ -387,20 +414,21 @@ class HhbcTranslator { /* * Fields */ - TraceBuilder& m_tb; - const Func* m_curFunc; - uint32 m_bcOff; - uint32 m_bcOffNextTrace; - bool m_firstBcOff; - bool m_lastBcOff; - bool m_hasRet; + TraceBuilder& m_tb; + const Func* m_curFunc; + uint32 m_bcOff; + uint32 m_bcOffNextTrace; + bool m_firstBcOff; + bool m_lastBcOff; + bool m_hasRet; // if set, then generate unbox instructions for memory accesses (Get // and Set bytecodes). Otherwise, memory accesses will bail the trace // on an access to a boxed value. - bool m_unboxPtrs; - uint32 m_stackDeficit; // offset of virtual sp from physical sp - EvalStack m_evalStack; - FpiStack m_fpiStack; + bool m_unboxPtrs; + uint32 m_stackDeficit; // offset of virtual sp from physical sp + EvalStack m_evalStack; + FpiStack m_fpiStack; + vector m_typeGuards; }; }}} // namespace HPHP::VM::JIT diff --git a/src/runtime/vm/translator/hopt/ir.cpp b/src/runtime/vm/translator/hopt/ir.cpp index 4b2b3c86b73c1..d45ef468d84f3 100644 --- a/src/runtime/vm/translator/hopt/ir.cpp +++ b/src/runtime/vm/translator/hopt/ir.cpp @@ -235,6 +235,9 @@ void IRInstruction::setExtendedSrc(uint32 i, SSATmp* newSrc) { void IRInstruction::printOpcode(std::ostream& ostream) { ostream << opcodeName(m_op); + if (m_op == GuardLoc || m_op == GuardStk) { + ostream << "<" << Type::Strings[m_type] << ">"; + } } void IRInstruction::printDst(std::ostream& ostream) { diff --git a/src/runtime/vm/translator/hopt/ir.h b/src/runtime/vm/translator/hopt/ir.h index 5ec3e6d138c2c..da89a4dd4b72c 100644 --- a/src/runtime/vm/translator/hopt/ir.h +++ b/src/runtime/vm/translator/hopt/ir.h @@ -113,7 +113,9 @@ static const TCA kIRDirectGuardActive = (TCA)0x03; // error (has an implicit exit edge) #define IR_OPCODES \ /* checks */ \ - OPC(GuardType, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0) \ + OPC(GuardType, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0) \ + OPC(GuardLoc, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0) \ + OPC(GuardStk, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0) \ OPC(GuardRefs, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0)/* XXX validate */ \ \ /* arith ops (integer) */ \ diff --git a/src/runtime/vm/translator/hopt/irtranslator.cpp b/src/runtime/vm/translator/hopt/irtranslator.cpp index 9be8a7fbe5976..4db6349d53c62 100644 --- a/src/runtime/vm/translator/hopt/irtranslator.cpp +++ b/src/runtime/vm/translator/hopt/irtranslator.cpp @@ -179,6 +179,12 @@ TranslatorX64::irCheckType(X64Assembler& a, return; } +void +TranslatorX64::irEmitLoadDeps() { + ASSERT(m_useHHIR); + m_hhbcTrans->emitLoadDeps(); +} + void TranslatorX64::irTranslateBinaryArithOp(const Tracelet& t, diff --git a/src/runtime/vm/translator/hopt/simplifier.cpp b/src/runtime/vm/translator/hopt/simplifier.cpp index 04c126449a524..88730f3c5c3b7 100644 --- a/src/runtime/vm/translator/hopt/simplifier.cpp +++ b/src/runtime/vm/translator/hopt/simplifier.cpp @@ -207,6 +207,8 @@ SSATmp* Simplifier::simplifyInst(Opcode opc, case DecRef: case DecRefNZ: case GuardType: + case GuardLoc: + case GuardStk: case LdThis: case LdLoc: case LdMemNR: diff --git a/src/runtime/vm/translator/hopt/tracebuilder.cpp b/src/runtime/vm/translator/hopt/tracebuilder.cpp index 8f843d26310ff..d1fce280dd814 100644 --- a/src/runtime/vm/translator/hopt/tracebuilder.cpp +++ b/src/runtime/vm/translator/hopt/tracebuilder.cpp @@ -655,6 +655,16 @@ Trace* TraceBuilder::genExitOnVarEnv(Trace* targetTrace) { return targetTrace; } +void TraceBuilder::genGuardLoc(uint32 id, Type::Tag type, Trace* exitTrace) { + genInstruction(GuardLoc, type, genLdHome(id), exitTrace); + setLocalType(id, type); +} + +void TraceBuilder::genGuardStk(uint32 id, Type::Tag type, Trace* exitTrace) { + genInstruction(GuardStk, type, m_spValue, genDefConst(id), exitTrace); +} + + SSATmp* TraceBuilder::genGuardType(SSATmp* src, Type::Tag type, Trace* target) { @@ -1640,6 +1650,18 @@ void TraceBuilder::setLocalValue(int id, SSATmp* value) { m_localTypes[id] = value->getType(); } +void TraceBuilder::setLocalType(int id, Type::Tag type) { + if (id == -1) { + return; + } + if (id >= (int)m_localValues.size()) { + m_localValues.resize(id + 1); + m_localTypes.resize(id + 1, Type::None); + } + m_localValues[id] = NULL; + m_localTypes[id] = type; +} + // Needs to be called if a local escapes as a by-ref void TraceBuilder::killLocalValue(int id) { if (id == -1 || id >= (int)m_localValues.size()) { diff --git a/src/runtime/vm/translator/hopt/tracebuilder.h b/src/runtime/vm/translator/hopt/tracebuilder.h index 8c82db40a8b9a..b68255e61995a 100644 --- a/src/runtime/vm/translator/hopt/tracebuilder.h +++ b/src/runtime/vm/translator/hopt/tracebuilder.h @@ -112,6 +112,8 @@ class TraceBuilder { const StringData* magicName); SSATmp* genAllocActRec(); // creates an uninitialized actrec SSATmp* genFreeActRec(); + void genGuardLoc(uint32 id, Type::Tag type, Trace* exitTrace); + void genGuardStk(uint32 id, Type::Tag type, Trace* exitTrace); SSATmp* genGuardType(SSATmp* src, Type::Tag type, Trace* nextTrace); void genGuardRefs(SSATmp* funcPtr, SSATmp* nParams, @@ -243,6 +245,8 @@ class TraceBuilder { SSATmp* getLocalValue(int id); Type::Tag getLocalType(int id); void setLocalValue(int id, SSATmp* value); + void setLocalType(int id, Type::Tag type); + void finalizeTrace(); template diff --git a/src/runtime/vm/translator/translator-x64.cpp b/src/runtime/vm/translator/translator-x64.cpp index 24c0f2cccce30..e9d3ebd370121 100644 --- a/src/runtime/vm/translator/translator-x64.cpp +++ b/src/runtime/vm/translator/translator-x64.cpp @@ -11090,6 +11090,10 @@ TranslatorX64::emitGuardChecks(X64Assembler& a, } } + if (m_useHHIR) { + irEmitLoadDeps(); + } + checkRefs(a, sk, refDeps, fail); if (Trace::moduleEnabled(Trace::stats, 2)) { diff --git a/src/runtime/vm/translator/translator-x64.h b/src/runtime/vm/translator/translator-x64.h index d5b7f3f6060ca..8574e8ef30089 100644 --- a/src/runtime/vm/translator/translator-x64.h +++ b/src/runtime/vm/translator/translator-x64.h @@ -809,6 +809,8 @@ PSEUDOINSTRS SrcRec& fail); void irCheckType(Asm&, const Location& l, const RuntimeType& rtt, SrcRec& fail); + void irEmitLoadDeps(); + void checkRefs(Asm&, const SrcKey&, const RefDeps&, SrcRec&); void emitDecRefThis(const ScratchReg& tmpReg);