Permalink
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
1722 lines (1492 sloc) 67.2 KB
//-------------------------------------------------------------------------------------------------------
// Copyright (C) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
//-------------------------------------------------------------------------------------------------------
#include "Backend.h"
#include "Core/CRC.h"
#include "NativeEntryPointData.h"
#include "JitTransferData.h"
///----------------------------------------------------------------------------
///
/// Encoder::Encode
///
/// Main entrypoint of encoder. Encode each IR instruction into the
/// appropriate machine encoding.
///
///----------------------------------------------------------------------------
void
Encoder::Encode()
{
NoRecoverMemoryArenaAllocator localArena(_u("BE-Encoder"), m_func->m_alloc->GetPageAllocator(), Js::Throw::OutOfMemory);
m_tempAlloc = &localArena;
#if ENABLE_OOP_NATIVE_CODEGEN
class AutoLocalAlloc {
public:
AutoLocalAlloc(Func * func) : localXdataAddr(nullptr), localAddress(nullptr), segment(nullptr), func(func) { }
~AutoLocalAlloc()
{
if (localAddress)
{
this->func->GetOOPThreadContext()->GetCodePageAllocators()->FreeLocal(this->localAddress, this->segment);
}
if (localXdataAddr)
{
this->func->GetOOPThreadContext()->GetCodePageAllocators()->FreeLocal(this->localXdataAddr, this->segment);
}
}
Func * func;
char * localXdataAddr;
char * localAddress;
void * segment;
} localAlloc(m_func);
#endif
uint32 instrCount = m_func->GetInstrCount();
size_t totalJmpTableSizeInBytes = 0;
JmpTableList * jumpTableListForSwitchStatement = nullptr;
m_encoderMD.Init(this);
m_encodeBufferSize = UInt32Math::Mul(instrCount, MachMaxInstrSize);
m_encodeBufferSize += m_func->m_totalJumpTableSizeInBytesForSwitchStatements;
m_encodeBuffer = AnewArray(m_tempAlloc, BYTE, m_encodeBufferSize);
#if DBG_DUMP
m_instrNumber = 0;
m_offsetBuffer = AnewArray(m_tempAlloc, uint, instrCount);
#endif
m_pragmaInstrToRecordMap = Anew(m_tempAlloc, PragmaInstrList, m_tempAlloc);
if (DoTrackAllStatementBoundary())
{
// Create a new list, if we are tracking all statement boundaries.
m_pragmaInstrToRecordOffset = Anew(m_tempAlloc, PragmaInstrList, m_tempAlloc);
}
else
{
// Set the list to the same as the throw map list, so that processing of the list
// of pragma are done on those only.
m_pragmaInstrToRecordOffset = m_pragmaInstrToRecordMap;
}
#if defined(_M_IX86) || defined(_M_X64)
// for BR shortening
m_inlineeFrameRecords = Anew(m_tempAlloc, InlineeFrameRecords, m_tempAlloc);
#endif
m_pc = m_encodeBuffer;
m_inlineeFrameMap = Anew(m_tempAlloc, ArenaInlineeFrameMap, m_tempAlloc);
m_sortedLazyBailoutRecordList = Anew(m_tempAlloc, ArenaLazyBailoutRecordList, m_tempAlloc);
IR::PragmaInstr* pragmaInstr = nullptr;
uint32 pragmaOffsetInBuffer = 0;
#ifdef _M_X64
bool inProlog = false;
#endif
bool isCallInstr = false;
// CRC Check to ensure the integrity of the encoded bytes.
uint initialCRCSeed = 0;
errno_t err = rand_s(&initialCRCSeed);
if (err != 0)
{
Fatal();
}
uint bufferCRC = initialCRCSeed;
FOREACH_INSTR_IN_FUNC(instr, m_func)
{
Assert(Lowerer::ValidOpcodeAfterLower(instr, m_func));
if (GetCurrentOffset() + MachMaxInstrSize < m_encodeBufferSize)
{
ptrdiff_t count;
#if DBG_DUMP
AssertMsg(m_instrNumber < instrCount, "Bad instr count?");
__analysis_assume(m_instrNumber < instrCount);
m_offsetBuffer[m_instrNumber++] = GetCurrentOffset();
#endif
if (instr->IsPragmaInstr())
{
switch (instr->m_opcode)
{
#ifdef _M_X64
case Js::OpCode::PrologStart:
m_func->m_prologEncoder.Begin(m_pc - m_encodeBuffer);
inProlog = true;
continue;
case Js::OpCode::PrologEnd:
m_func->m_prologEncoder.End();
inProlog = false;
continue;
#endif
case Js::OpCode::StatementBoundary:
pragmaOffsetInBuffer = GetCurrentOffset();
pragmaInstr = instr->AsPragmaInstr();
pragmaInstr->m_offsetInBuffer = pragmaOffsetInBuffer;
// will record after BR shortening with adjusted offsets
if (DoTrackAllStatementBoundary())
{
m_pragmaInstrToRecordOffset->Add(pragmaInstr);
}
break;
default:
continue;
}
}
else if (instr->IsBranchInstr() && instr->AsBranchInstr()->IsMultiBranch())
{
Assert(instr->GetSrc1() && instr->GetSrc1()->IsRegOpnd());
IR::MultiBranchInstr * multiBranchInstr = instr->AsBranchInstr()->AsMultiBrInstr();
if (multiBranchInstr->m_isSwitchBr &&
(multiBranchInstr->m_kind == IR::MultiBranchInstr::IntJumpTable || multiBranchInstr->m_kind == IR::MultiBranchInstr::SingleCharStrJumpTable))
{
BranchJumpTableWrapper * branchJumpTableWrapper = multiBranchInstr->GetBranchJumpTable();
if (jumpTableListForSwitchStatement == nullptr)
{
jumpTableListForSwitchStatement = Anew(m_tempAlloc, JmpTableList, m_tempAlloc);
}
jumpTableListForSwitchStatement->Add(branchJumpTableWrapper);
totalJmpTableSizeInBytes += (branchJumpTableWrapper->tableSize * sizeof(void*));
}
else
{
//Reloc Records
EncoderMD * encoderMD = &(this->m_encoderMD);
multiBranchInstr->MapMultiBrTargetByAddress([=](void ** offset) -> void
{
#if defined(_M_ARM32_OR_ARM64)
encoderMD->AddLabelReloc((byte*)offset);
#else
encoderMD->AppendRelocEntry(RelocTypeLabelUse, (void*)(offset), *(IR::LabelInstr**)(offset));
*((size_t*)offset) = 0;
#endif
});
}
}
else
{
isCallInstr = LowererMD::IsCall(instr);
if (pragmaInstr && (instr->isInlineeEntryInstr || isCallInstr))
{
// will record throw map after BR shortening with adjusted offsets
m_pragmaInstrToRecordMap->Add(pragmaInstr);
pragmaInstr = nullptr; // Only once per pragma instr -- do we need to make this record?
}
if (instr->HasBailOutInfo())
{
Assert(this->m_func->hasBailout);
Assert(LowererMD::IsCall(instr));
instr->GetBailOutInfo()->FinalizeBailOutRecord(this->m_func);
}
if (instr->isInlineeEntryInstr)
{
m_encoderMD.EncodeInlineeCallInfo(instr, GetCurrentOffset());
}
if (instr->m_opcode == Js::OpCode::InlineeStart)
{
Assert(!instr->isInlineeEntryInstr);
if (pragmaInstr)
{
m_pragmaInstrToRecordMap->Add(pragmaInstr);
pragmaInstr = nullptr;
}
Func* inlinee = instr->m_func;
if (inlinee->frameInfo && inlinee->frameInfo->record)
{
inlinee->frameInfo->record->Finalize(inlinee, GetCurrentOffset());
#if defined(_M_IX86) || defined(_M_X64)
// Store all records to be adjusted for BR shortening
m_inlineeFrameRecords->Add(inlinee->frameInfo->record);
#endif
}
continue;
}
}
count = m_encoderMD.Encode(instr, m_pc, m_encodeBuffer);
#if defined(_M_IX86) || defined(_M_X64)
bufferCRC = CalculateCRC(bufferCRC, count, m_pc);
#endif
#if DBG_DUMP
if (PHASE_TRACE(Js::EncoderPhase, this->m_func))
{
instr->Dump((IRDumpFlags)(IRDumpFlags_SimpleForm | IRDumpFlags_SkipEndLine | IRDumpFlags_SkipByteCodeOffset));
Output::SkipToColumn(80);
for (BYTE * current = m_pc; current < m_pc + count; current++)
{
Output::Print(_u("%02X "), *current);
}
Output::Print(_u("\n"));
Output::Flush();
}
#endif
#ifdef _M_X64
if (inProlog)
m_func->m_prologEncoder.EncodeInstr(instr, count & 0xFF);
#endif
m_pc += count;
#if defined(_M_IX86) || defined(_M_X64)
// for BR shortening.
if (instr->isInlineeEntryInstr)
m_encoderMD.AppendRelocEntry(RelocType::RelocTypeInlineeEntryOffset, (void*)(m_pc - MachPtr));
#endif
if (isCallInstr)
{
isCallInstr = false;
this->RecordInlineeFrame(instr->m_func, GetCurrentOffset());
}
if (instr->HasLazyBailOut())
{
this->SaveToLazyBailOutRecordList(instr, this->GetCurrentOffset());
}
if (instr->m_opcode == Js::OpCode::LazyBailOutThunkLabel)
{
this->SaveLazyBailOutThunkOffset(this->GetCurrentOffset());
}
}
else
{
Fatal();
}
} NEXT_INSTR_IN_FUNC;
ptrdiff_t codeSize = m_pc - m_encodeBuffer + totalJmpTableSizeInBytes;
BOOL isSuccessBrShortAndLoopAlign = false;
#if defined(_M_IX86) || defined(_M_X64)
// Shorten branches. ON by default
if (!PHASE_OFF(Js::BrShortenPhase, m_func))
{
uint brShortenedbufferCRC = initialCRCSeed;
isSuccessBrShortAndLoopAlign = ShortenBranchesAndLabelAlign(&m_encodeBuffer, &codeSize, &brShortenedbufferCRC, bufferCRC, totalJmpTableSizeInBytes);
if (isSuccessBrShortAndLoopAlign)
{
bufferCRC = brShortenedbufferCRC;
}
}
#endif
#if DBG_DUMP | defined(VTUNE_PROFILING)
if (this->m_func->DoRecordNativeMap())
{
// Record PragmaInstr offsets and throw maps
for (int32 i = 0; i < m_pragmaInstrToRecordOffset->Count(); i++)
{
IR::PragmaInstr *inst = m_pragmaInstrToRecordOffset->Item(i);
inst->Record(inst->m_offsetInBuffer);
}
}
#endif
if (m_pragmaInstrToRecordMap->Count() > 0)
{
if (m_func->IsOOPJIT())
{
int allocSize = m_pragmaInstrToRecordMap->Count();
Js::ThrowMapEntry * throwMap = NativeCodeDataNewArrayNoFixup(m_func->GetNativeCodeDataAllocator(), Js::ThrowMapEntry, allocSize);
for (int i = 0; i < allocSize; i++)
{
IR::PragmaInstr *inst = m_pragmaInstrToRecordMap->Item(i);
throwMap[i].nativeBufferOffset = inst->m_offsetInBuffer;
throwMap[i].statementIndex = inst->m_statementIndex;
}
m_func->GetJITOutput()->RecordThrowMap(throwMap, m_pragmaInstrToRecordMap->Count());
}
else
{
auto entryPointInfo = m_func->GetInProcJITEntryPointInfo();
auto functionBody = entryPointInfo->GetFunctionBody();
Js::SmallSpanSequenceIter iter;
for (int32 i = 0; i < m_pragmaInstrToRecordMap->Count(); i++)
{
IR::PragmaInstr *inst = m_pragmaInstrToRecordMap->Item(i);
functionBody->RecordNativeThrowMap(iter, inst->m_offsetInBuffer, inst->m_statementIndex, entryPointInfo, Js::LoopHeader::NoLoop);
}
}
}
// Assembly Dump Phase
// This phase exists to assist tooling that expects "assemblable" output - that is,
// output that, with minimal manual handling, could theoretically be fed to another
// assembler to make a valid function for the target platform. We don't guarantee a
// dump from this will _actually_ be assemblable, but it is significantly closer to
// that than our normal, annotated output
#if DBG_DUMP
if (PHASE_DUMP(Js::AssemblyPhase, m_func))
{
FOREACH_INSTR_IN_FUNC(instr, m_func)
{
bool hasPrintedForOpnds = false;
Func* localScopeFuncForLambda = m_func;
auto printOpnd = [&hasPrintedForOpnds, localScopeFuncForLambda](IR::Opnd* opnd)
{
if (hasPrintedForOpnds)
{
Output::Print(_u(", "));
}
switch (opnd->m_kind)
{
case IR::OpndKindInvalid:
AssertMsg(false, "Should be unreachable");
break;
case IR::OpndKindIntConst:
Output::Print(_u("%lli"), (long long int)opnd->AsIntConstOpnd()->GetValue());
break;
case IR::OpndKindInt64Const:
case IR::OpndKindFloatConst:
case IR::OpndKindFloat32Const:
case IR::OpndKindSimd128Const:
AssertMsg(false, "Not Yet Implemented");
break;
case IR::OpndKindHelperCall:
Output::Print(_u("%s"), IR::GetMethodName(opnd->AsHelperCallOpnd()->m_fnHelper));
break;
case IR::OpndKindSym:
Output::Print(_u("SYM("));
opnd->Dump(IRDumpFlags_SimpleForm, localScopeFuncForLambda);
Output::Print(_u(")"));
break;
case IR::OpndKindReg:
Output::Print(_u("%S"), RegNames[opnd->AsRegOpnd()->GetReg()]);
break;
case IR::OpndKindAddr:
Output::Print(_u("0x%p"), opnd->AsAddrOpnd()->m_address);
break;
case IR::OpndKindIndir:
{
IR::IndirOpnd* indirOpnd = opnd->AsIndirOpnd();
IR::RegOpnd* baseOpnd = indirOpnd->GetBaseOpnd();
IR::RegOpnd* indexOpnd = indirOpnd->GetIndexOpnd();
Output::Print(_u("["));
bool hasPrintedComponent = false;
if (baseOpnd != nullptr)
{
Output::Print(_u("%S"), RegNames[baseOpnd->GetReg()]);
hasPrintedComponent = true;
}
if (indexOpnd != nullptr)
{
if (hasPrintedComponent)
{
Output::Print(_u(" + "));
}
Output::Print(_u("%S * %u"), RegNames[indexOpnd->GetReg()], indirOpnd->GetScale());
hasPrintedComponent = true;
}
if (hasPrintedComponent)
{
Output::Print(_u(" + "));
}
Output::Print(_u("(%i)]"), indirOpnd->GetOffset());
break;
}
case IR::OpndKindLabel:
opnd->Dump(IRDumpFlags_SimpleForm, localScopeFuncForLambda);
break;
case IR::OpndKindMemRef:
opnd->DumpOpndKindMemRef(true, localScopeFuncForLambda);
break;
case IR::OpndKindRegBV:
AssertMsg(false, "Should be unreachable");
break;
case IR::OpndKindList:
AssertMsg(false, "Should be unreachable");
break;
default:
AssertMsg(false, "Missing operand type");
}
hasPrintedForOpnds = true;
};
switch(instr->GetKind())
{
case IR::InstrKindInvalid:
Assert(false);
break;
case IR::InstrKindJitProfiling:
case IR::InstrKindProfiled:
case IR::InstrKindInstr:
{
Output::SkipToColumn(4);
Output::Print(_u("%s "), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
Output::SkipToColumn(18);
IR::Opnd* dst = instr->GetDst();
IR::Opnd* src1 = instr->GetSrc1();
IR::Opnd* src2 = instr->GetSrc2();
if (dst != nullptr && (src1 == nullptr || !dst->IsRegOpnd() || !src1->IsRegOpnd() || dst->AsRegOpnd()->GetReg() != src1->AsRegOpnd()->GetReg())) // Print dst if it's there, and not the same reg as src1 (which is usually an instr that has a srcdest
{
printOpnd(dst);
}
if (src1 != nullptr)
{
printOpnd(src1);
}
if (src2 != nullptr)
{
printOpnd(src2);
}
break;
}
case IR::InstrKindBranch:
Output::SkipToColumn(4);
Output::Print(_u("%s "), Js::OpCodeUtil::GetOpCodeName(instr->m_opcode));
Output::SkipToColumn(18);
if (instr->AsBranchInstr()->IsMultiBranch())
{
Assert(instr->GetSrc1() != nullptr);
printOpnd(instr->GetSrc1());
}
else
{
Output::Print(_u("L%u"), instr->AsBranchInstr()->GetTarget()->m_id);
}
break;
case IR::InstrKindProfiledLabel:
case IR::InstrKindLabel:
Output::Print(_u("L%u:"), instr->AsLabelInstr()->m_id);
break;
case IR::InstrKindEntry:
case IR::InstrKindExit:
case IR::InstrKindPragma:
// No output
break;
case IR::InstrKindByteCodeUses:
AssertMsg(false, "Instruction kind shouldn't be present here");
break;
default:
Assert(false);
break;
}
Output::SetAlignAndPrefix(60, _u("; "));
instr->Dump();
Output::ResetAlignAndPrefix();
} NEXT_INSTR_IN_FUNC;
}
#endif
// End Assembly Dump Phase
BEGIN_CODEGEN_PHASE(m_func, Js::EmitterPhase);
// Copy to permanent buffer.
Assert(Math::FitsInDWord(codeSize));
ushort xdataSize;
ushort pdataCount;
#ifdef _M_X64
pdataCount = 1;
xdataSize = (ushort)m_func->m_prologEncoder.SizeOfUnwindInfo();
#elif defined(_M_ARM64)
pdataCount = 1;
xdataSize = XDATA_SIZE;
#elif defined(_M_ARM)
pdataCount = (ushort)m_func->m_unwindInfo.GetPDataCount(codeSize);
xdataSize = (UnwindInfoManager::MaxXdataBytes + 3) * pdataCount;
#else
xdataSize = 0;
pdataCount = 0;
#endif
OUTPUT_VERBOSE_TRACE(Js::EmitterPhase, _u("PDATA count:%u\n"), pdataCount);
OUTPUT_VERBOSE_TRACE(Js::EmitterPhase, _u("Size of XDATA:%u\n"), xdataSize);
OUTPUT_VERBOSE_TRACE(Js::EmitterPhase, _u("Size of code:%u\n"), codeSize);
TryCopyAndAddRelocRecordsForSwitchJumpTableEntries(m_encodeBuffer, codeSize, jumpTableListForSwitchStatement, totalJmpTableSizeInBytes);
CustomHeap::Allocation * allocation = nullptr;
bool inPrereservedRegion = false;
char * localAddress = nullptr;
#if ENABLE_OOP_NATIVE_CODEGEN
if (JITManager::GetJITManager()->IsJITServer())
{
EmitBufferAllocation<SectionAllocWrapper, PreReservedSectionAllocWrapper> * alloc = m_func->GetJITOutput()->RecordOOPNativeCodeSize(m_func, (DWORD)codeSize, pdataCount, xdataSize);
allocation = alloc->allocation;
inPrereservedRegion = alloc->inPrereservedRegion;
localAlloc.segment = (alloc->bytesCommitted > CustomHeap::Page::MaxAllocationSize) ? allocation->largeObjectAllocation.segment : allocation->page->segment;
localAddress = m_func->GetOOPThreadContext()->GetCodePageAllocators()->AllocLocal(allocation->address, alloc->bytesCommitted, localAlloc.segment);
localAlloc.localAddress = localAddress;
if (localAddress == nullptr)
{
Js::Throw::OutOfMemory();
}
}
else
#endif
{
EmitBufferAllocation<VirtualAllocWrapper, PreReservedVirtualAllocWrapper> * alloc = m_func->GetJITOutput()->RecordInProcNativeCodeSize(m_func, (DWORD)codeSize, pdataCount, xdataSize);
allocation = alloc->allocation;
inPrereservedRegion = alloc->inPrereservedRegion;
localAddress = allocation->address;
}
if (!inPrereservedRegion)
{
m_func->GetThreadContextInfo()->ResetIsAllJITCodeInPreReservedRegion();
}
// Relocs
m_encoderMD.ApplyRelocs((size_t)allocation->address, codeSize, &bufferCRC, isSuccessBrShortAndLoopAlign);
m_func->GetJITOutput()->RecordNativeCode(m_encodeBuffer, (BYTE *)localAddress);
#if defined(_M_IX86) || defined(_M_X64)
if (!JITManager::GetJITManager()->IsJITServer())
{
ValidateCRCOnFinalBuffer((BYTE *)allocation->address, codeSize, totalJmpTableSizeInBytes, m_encodeBuffer, initialCRCSeed, bufferCRC, isSuccessBrShortAndLoopAlign);
}
#endif
#ifdef TARGET_64
#ifdef _M_X64
PrologEncoder &unwindInfo = m_func->m_prologEncoder;
unwindInfo.FinalizeUnwindInfo((BYTE*)m_func->GetJITOutput()->GetCodeAddress(), (DWORD)codeSize);
#else
UnwindInfoManager &unwindInfo = m_func->m_unwindInfo;
unwindInfo.FinalizeUnwindInfo((BYTE*)localAddress, (DWORD)codeSize);
#endif
char * localXdataAddr = nullptr;
#if ENABLE_OOP_NATIVE_CODEGEN
if (JITManager::GetJITManager()->IsJITServer())
{
localXdataAddr = m_func->GetOOPThreadContext()->GetCodePageAllocators()->AllocLocal((char*)allocation->xdata.address, XDATA_SIZE, localAlloc.segment);
localAlloc.localXdataAddr = localXdataAddr;
if (localXdataAddr == nullptr)
{
Js::Throw::OutOfMemory();
}
}
else
#endif
{
localXdataAddr = (char*)allocation->xdata.address;
}
m_func->GetJITOutput()->RecordUnwindInfo(
unwindInfo.GetUnwindInfo(),
unwindInfo.SizeOfUnwindInfo(),
allocation->xdata.address,
(BYTE*)localXdataAddr);
#elif _M_ARM
m_func->m_unwindInfo.EmitUnwindInfo(m_func->GetJITOutput(), allocation);
if (m_func->IsOOPJIT())
{
size_t allocSize = XDataAllocator::GetAllocSize(allocation->xdata.pdataCount, allocation->xdata.xdataSize);
BYTE * xprocXdata = NativeCodeDataNewArrayNoFixup(m_func->GetNativeCodeDataAllocator(), BYTE, allocSize);
memcpy_s(xprocXdata, allocSize, allocation->xdata.address, allocSize);
m_func->GetJITOutput()->RecordXData(xprocXdata);
}
else
{
XDataAllocator::Register(&allocation->xdata, m_func->GetJITOutput()->GetCodeAddress(), (DWORD)m_func->GetJITOutput()->GetCodeSize());
m_func->GetInProcJITEntryPointInfo()->GetNativeEntryPointData()->SetXDataInfo(&allocation->xdata);
}
m_func->GetJITOutput()->SetCodeAddress(m_func->GetJITOutput()->GetCodeAddress() | 0x1); // Set thumb mode
#endif
const bool isSimpleJit = m_func->IsSimpleJit();
if (this->m_inlineeFrameMap->Count() > 0 &&
!(this->m_inlineeFrameMap->Count() == 1 && this->m_inlineeFrameMap->Item(0).record == nullptr))
{
if (!m_func->IsOOPJIT()) // in-proc JIT
{
m_func->GetInProcJITEntryPointInfo()->GetInProcNativeEntryPointData()->RecordInlineeFrameMap(m_inlineeFrameMap);
}
else // OOP JIT
{
NativeOffsetInlineeFrameRecordOffset* pairs = NativeCodeDataNewArrayZNoFixup(m_func->GetNativeCodeDataAllocator(), NativeOffsetInlineeFrameRecordOffset, this->m_inlineeFrameMap->Count());
this->m_inlineeFrameMap->Map([&pairs](int i, NativeOffsetInlineeFramePair& p)
{
pairs[i].offset = p.offset;
if (p.record)
{
pairs[i].recordOffset = NativeCodeData::GetDataChunk(p.record)->offset;
}
else
{
pairs[i].recordOffset = NativeOffsetInlineeFrameRecordOffset::InvalidRecordOffset;
}
});
m_func->GetJITOutput()->RecordInlineeFrameOffsetsInfo(NativeCodeData::GetDataChunk(pairs)->offset, this->m_inlineeFrameMap->Count());
}
}
this->SaveLazyBailOutJitTransferData();
if (this->m_func->pinnedTypeRefs != nullptr)
{
Assert(!isSimpleJit);
int pinnedTypeRefCount = this->m_func->pinnedTypeRefs->Count();
PinnedTypeRefsIDL* pinnedTypeRefs = nullptr;
if (this->m_func->IsOOPJIT())
{
pinnedTypeRefs = (PinnedTypeRefsIDL*)midl_user_allocate(offsetof(PinnedTypeRefsIDL, typeRefs) + sizeof(void*)*pinnedTypeRefCount);
if (!pinnedTypeRefs)
{
Js::Throw::OutOfMemory();
}
__analysis_assume(pinnedTypeRefs);
pinnedTypeRefs->count = pinnedTypeRefCount;
pinnedTypeRefs->isOOPJIT = true;
}
else
{
pinnedTypeRefs = HeapNewStructPlus(offsetof(PinnedTypeRefsIDL, typeRefs) + sizeof(void*)*pinnedTypeRefCount - sizeof(PinnedTypeRefsIDL), PinnedTypeRefsIDL);
pinnedTypeRefs->count = pinnedTypeRefCount;
pinnedTypeRefs->isOOPJIT = false;
}
int index = 0;
this->m_func->pinnedTypeRefs->Map([&pinnedTypeRefs, &index](void* typeRef) -> void
{
pinnedTypeRefs->typeRefs[index++] = ((JITType*)typeRef)->GetAddr();
});
if (PHASE_TRACE(Js::TracePinnedTypesPhase, this->m_func))
{
char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE];
Output::Print(_u("PinnedTypes: function %s(%s) pinned %d types.\n"),
this->m_func->GetJITFunctionBody()->GetDisplayName(), this->m_func->GetDebugNumberSet(debugStringBuffer), pinnedTypeRefCount);
Output::Flush();
}
this->m_func->GetJITOutput()->GetOutputData()->pinnedTypeRefs = pinnedTypeRefs;
}
// Save all equivalent type guards in a fixed size array on the JIT transfer data
if (this->m_func->equivalentTypeGuards != nullptr)
{
AssertMsg(!PHASE_OFF(Js::EquivObjTypeSpecPhase, this->m_func), "Why do we have equivalent type guards if we don't do equivalent object type spec?");
int equivalentTypeGuardsCount = this->m_func->equivalentTypeGuards->Count();
if (this->m_func->IsOOPJIT())
{
auto& equivalentTypeGuardOffsets = this->m_func->GetJITOutput()->GetOutputData()->equivalentTypeGuardOffsets;
size_t allocSize = offsetof(EquivalentTypeGuardOffsets, guards) + equivalentTypeGuardsCount * sizeof(EquivalentTypeGuardIDL);
equivalentTypeGuardOffsets = (EquivalentTypeGuardOffsets*)midl_user_allocate(allocSize);
if (equivalentTypeGuardOffsets == nullptr)
{
Js::Throw::OutOfMemory();
}
equivalentTypeGuardOffsets->count = equivalentTypeGuardsCount;
int i = 0;
this->m_func->equivalentTypeGuards->Map([&equivalentTypeGuardOffsets, &i](Js::JitEquivalentTypeGuard* srcGuard) -> void
{
equivalentTypeGuardOffsets->guards[i].offset = NativeCodeData::GetDataTotalOffset(srcGuard);
auto cache = srcGuard->GetCache();
equivalentTypeGuardOffsets->guards[i].cache.guardOffset = NativeCodeData::GetDataTotalOffset(cache->guard);
equivalentTypeGuardOffsets->guards[i].cache.hasFixedValue = cache->hasFixedValue;
equivalentTypeGuardOffsets->guards[i].cache.isLoadedFromProto = cache->isLoadedFromProto;
equivalentTypeGuardOffsets->guards[i].cache.nextEvictionVictim = cache->nextEvictionVictim;
equivalentTypeGuardOffsets->guards[i].cache.record.propertyCount = cache->record.propertyCount;
equivalentTypeGuardOffsets->guards[i].cache.record.propertyOffset = NativeCodeData::GetDataTotalOffset(cache->record.properties);
for (int j = 0; j < EQUIVALENT_TYPE_CACHE_SIZE; j++)
{
equivalentTypeGuardOffsets->guards[i].cache.types[j] = (intptr_t)PointerValue(cache->types[j]);
}
i++;
});
Assert(equivalentTypeGuardsCount == i);
}
else
{
Js::JitEquivalentTypeGuard** guards = HeapNewArrayZ(Js::JitEquivalentTypeGuard*, equivalentTypeGuardsCount);
Js::JitEquivalentTypeGuard** dstGuard = guards;
this->m_func->equivalentTypeGuards->Map([&dstGuard](Js::JitEquivalentTypeGuard* srcGuard) -> void
{
*dstGuard++ = srcGuard;
});
m_func->GetInProcJITEntryPointInfo()->GetJitTransferData()->SetEquivalentTypeGuards(guards, equivalentTypeGuardsCount);
}
}
// Save all property guards on the JIT transfer data in a map keyed by property ID. We will use this map when installing the entry
// point to register each guard for invalidation.
if (this->m_func->propertyGuardsByPropertyId != nullptr)
{
Assert(!isSimpleJit);
AssertMsg(!(PHASE_OFF(Js::ObjTypeSpecPhase, this->m_func) && PHASE_OFF(Js::FixedMethodsPhase, this->m_func)),
"Why do we have type guards if we don't do object type spec or fixed methods?");
#if DBG
int totalGuardCount = (this->m_func->singleTypeGuards != nullptr ? this->m_func->singleTypeGuards->Count() : 0)
+ (this->m_func->equivalentTypeGuards != nullptr ? this->m_func->equivalentTypeGuards->Count() : 0);
Assert(totalGuardCount > 0);
Assert(totalGuardCount == this->m_func->indexedPropertyGuardCount);
#endif
if (!this->m_func->IsOOPJIT())
{
int propertyCount = this->m_func->propertyGuardsByPropertyId->Count();
Assert(propertyCount > 0);
int guardSlotCount = 0;
this->m_func->propertyGuardsByPropertyId->Map([&guardSlotCount](Js::PropertyId propertyId, Func::IndexedPropertyGuardSet* set) -> void
{
guardSlotCount += set->Count();
});
size_t typeGuardTransferSize = // Reserve enough room for:
propertyCount * sizeof(Js::TypeGuardTransferEntry) + // each propertyId,
propertyCount * sizeof(Js::JitIndexedPropertyGuard*) + // terminating nullptr guard for each propertyId,
guardSlotCount * sizeof(Js::JitIndexedPropertyGuard*); // a pointer for each guard we counted above.
// The extra room for sizeof(Js::TypePropertyGuardEntry) allocated by HeapNewPlus will be used for the terminating invalid propertyId.
// Review (jedmiad): Skip zeroing? This is heap allocated so there shouldn't be any false recycler references.
Js::TypeGuardTransferEntry* typeGuardTransferRecord = HeapNewPlusZ(typeGuardTransferSize, Js::TypeGuardTransferEntry);
Func* func = this->m_func;
Js::TypeGuardTransferEntry* dstEntry = typeGuardTransferRecord;
this->m_func->propertyGuardsByPropertyId->Map([func, &dstEntry](Js::PropertyId propertyId, Func::IndexedPropertyGuardSet* srcSet) -> void
{
dstEntry->propertyId = propertyId;
int guardIndex = 0;
srcSet->Map([dstEntry, &guardIndex](Js::JitIndexedPropertyGuard* guard) -> void
{
dstEntry->guards[guardIndex++] = guard;
});
dstEntry->guards[guardIndex++] = nullptr;
dstEntry = reinterpret_cast<Js::TypeGuardTransferEntry*>(&dstEntry->guards[guardIndex]);
});
dstEntry->propertyId = Js::Constants::NoProperty;
dstEntry++;
Assert(reinterpret_cast<char*>(dstEntry) <= reinterpret_cast<char*>(typeGuardTransferRecord) + typeGuardTransferSize + sizeof(Js::TypeGuardTransferEntry));
m_func->GetInProcJITEntryPointInfo()->GetJitTransferData()->RecordTypeGuards(this->m_func->indexedPropertyGuardCount, typeGuardTransferRecord, typeGuardTransferSize);
}
else
{
Func* func = this->m_func;
this->m_func->GetJITOutput()->GetOutputData()->propertyGuardCount = this->m_func->indexedPropertyGuardCount;
auto entry = &this->m_func->GetJITOutput()->GetOutputData()->typeGuardEntries;
this->m_func->propertyGuardsByPropertyId->Map([func, &entry](Js::PropertyId propertyId, Func::IndexedPropertyGuardSet* srcSet) -> void
{
auto count = srcSet->Count();
(*entry) = (TypeGuardTransferEntryIDL*)midl_user_allocate(offsetof(TypeGuardTransferEntryIDL, guardOffsets) + count*sizeof(int));
if (!*entry)
{
Js::Throw::OutOfMemory();
}
__analysis_assume(*entry);
(*entry)->propId = propertyId;
(*entry)->guardsCount = count;
(*entry)->next = nullptr;
auto& guardOffsets = (*entry)->guardOffsets;
int guardIndex = 0;
srcSet->Map([&guardOffsets, &guardIndex](Js::JitIndexedPropertyGuard* guard) -> void
{
guardOffsets[guardIndex++] = NativeCodeData::GetDataTotalOffset(guard);
});
Assert(guardIndex == count);
entry = &(*entry)->next;
});
}
}
// Save all constructor caches on the JIT transfer data in a map keyed by property ID. We will use this map when installing the entry
// point to register each cache for invalidation.
if (this->m_func->ctorCachesByPropertyId != nullptr)
{
Assert(!isSimpleJit);
AssertMsg(!(PHASE_OFF(Js::ObjTypeSpecPhase, this->m_func) && PHASE_OFF(Js::FixedMethodsPhase, this->m_func)),
"Why do we have constructor cache guards if we don't do object type spec or fixed methods?");
int propertyCount = this->m_func->ctorCachesByPropertyId->Count();
Assert(propertyCount > 0);
int cacheSlotCount = 0;
this->m_func->ctorCachesByPropertyId->Map([&cacheSlotCount](Js::PropertyId propertyId, Func::CtorCacheSet* cacheSet) -> void
{
cacheSlotCount += cacheSet->Count();
});
if (m_func->IsOOPJIT())
{
Func* func = this->m_func;
m_func->GetJITOutput()->GetOutputData()->ctorCachesCount = propertyCount;
m_func->GetJITOutput()->GetOutputData()->ctorCacheEntries = (CtorCacheTransferEntryIDL**)midl_user_allocate(propertyCount * sizeof(CtorCacheTransferEntryIDL*));
CtorCacheTransferEntryIDL** entries = m_func->GetJITOutput()->GetOutputData()->ctorCacheEntries;
if (!entries)
{
Js::Throw::OutOfMemory();
}
__analysis_assume(entries);
uint propIndex = 0;
m_func->ctorCachesByPropertyId->Map([func, entries, &propIndex](Js::PropertyId propertyId, Func::CtorCacheSet* srcCacheSet) -> void
{
entries[propIndex] = (CtorCacheTransferEntryIDL*)midl_user_allocate(srcCacheSet->Count() * sizeof(intptr_t) + sizeof(CtorCacheTransferEntryIDL));
if (!entries[propIndex])
{
Js::Throw::OutOfMemory();
}
__analysis_assume(entries[propIndex]);
entries[propIndex]->propId = propertyId;
int cacheIndex = 0;
srcCacheSet->Map([entries, propIndex, &cacheIndex](intptr_t cache) -> void
{
entries[propIndex]->caches[cacheIndex++] = cache;
});
entries[propIndex]->cacheCount = cacheIndex;
propIndex++;
});
}
else
{
Assert(m_func->GetInProcJITEntryPointInfo()->GetNativeEntryPointData()->GetConstructorCacheCount() > 0);
size_t ctorCachesTransferSize = // Reserve enough room for:
propertyCount * sizeof(Js::CtorCacheGuardTransferEntry) + // each propertyId,
propertyCount * sizeof(Js::ConstructorCache*) + // terminating null cache for each propertyId,
cacheSlotCount * sizeof(Js::JitIndexedPropertyGuard*); // a pointer for each cache we counted above.
// The extra room for sizeof(Js::CtorCacheGuardTransferEntry) allocated by HeapNewPlus will be used for the terminating invalid propertyId.
// Review (jedmiad): Skip zeroing? This is heap allocated so there shouldn't be any false recycler references.
Js::CtorCacheGuardTransferEntry* ctorCachesTransferRecord = HeapNewPlusZ(ctorCachesTransferSize, Js::CtorCacheGuardTransferEntry);
Func* func = this->m_func;
Js::CtorCacheGuardTransferEntry* dstEntry = ctorCachesTransferRecord;
this->m_func->ctorCachesByPropertyId->Map([func, &dstEntry](Js::PropertyId propertyId, Func::CtorCacheSet* srcCacheSet) -> void
{
dstEntry->propertyId = propertyId;
int cacheIndex = 0;
srcCacheSet->Map([dstEntry, &cacheIndex](intptr_t cache) -> void
{
dstEntry->caches[cacheIndex++] = cache;
});
dstEntry->caches[cacheIndex++] = 0;
dstEntry = reinterpret_cast<Js::CtorCacheGuardTransferEntry*>(&dstEntry->caches[cacheIndex]);
});
dstEntry->propertyId = Js::Constants::NoProperty;
dstEntry++;
Assert(reinterpret_cast<char*>(dstEntry) <= reinterpret_cast<char*>(ctorCachesTransferRecord) + ctorCachesTransferSize + sizeof(Js::CtorCacheGuardTransferEntry));
m_func->GetInProcJITEntryPointInfo()->GetJitTransferData()->RecordCtorCacheGuards(ctorCachesTransferRecord, ctorCachesTransferSize);
}
}
m_func->GetJITOutput()->FinalizeNativeCode();
END_CODEGEN_PHASE(m_func, Js::EmitterPhase);
#if DBG_DUMP
m_func->m_codeSize = codeSize;
if (PHASE_DUMP(Js::EncoderPhase, m_func) || PHASE_DUMP(Js::BackEndPhase, m_func))
{
bool dumpIRAddressesValue = Js::Configuration::Global.flags.DumpIRAddresses;
Js::Configuration::Global.flags.DumpIRAddresses = true;
this->m_func->DumpHeader();
m_instrNumber = 0;
FOREACH_INSTR_IN_FUNC(instr, m_func)
{
__analysis_assume(m_instrNumber < instrCount);
instr->DumpGlobOptInstrString();
#ifdef TARGET_64
Output::Print(_u("%12IX "), m_offsetBuffer[m_instrNumber++] + (BYTE *)m_func->GetJITOutput()->GetCodeAddress());
#else
Output::Print(_u("%8IX "), m_offsetBuffer[m_instrNumber++] + (BYTE *)m_func->GetJITOutput()->GetCodeAddress());
#endif
instr->Dump();
} NEXT_INSTR_IN_FUNC;
Output::Flush();
Js::Configuration::Global.flags.DumpIRAddresses = dumpIRAddressesValue;
}
if (PHASE_DUMP(Js::EncoderPhase, m_func) && Js::Configuration::Global.flags.Verbose && !m_func->IsOOPJIT())
{
this->DumpInlineeFrameMap(m_func->GetJITOutput()->GetCodeAddress());
Output::Flush();
}
#endif
}
bool Encoder::DoTrackAllStatementBoundary() const
{
#if DBG_DUMP | defined(VTUNE_PROFILING)
return this->m_func->DoRecordNativeMap();
#else
return false;
#endif
}
void Encoder::TryCopyAndAddRelocRecordsForSwitchJumpTableEntries(BYTE *codeStart, size_t codeSize, JmpTableList * jumpTableListForSwitchStatement, size_t totalJmpTableSizeInBytes)
{
if (jumpTableListForSwitchStatement == nullptr)
{
return;
}
BYTE * jmpTableStartAddress = codeStart + codeSize - totalJmpTableSizeInBytes;
EncoderMD * encoderMD = &m_encoderMD;
jumpTableListForSwitchStatement->Map([&](uint index, BranchJumpTableWrapper * branchJumpTableWrapper) -> void
{
Assert(branchJumpTableWrapper != nullptr);
void ** srcJmpTable = branchJumpTableWrapper->jmpTable;
size_t jmpTableSizeInBytes = branchJumpTableWrapper->tableSize * sizeof(void*);
AssertMsg(branchJumpTableWrapper->labelInstr != nullptr, "Label not yet created?");
Assert(branchJumpTableWrapper->labelInstr->GetPC() == nullptr);
branchJumpTableWrapper->labelInstr->SetPC(jmpTableStartAddress);
memcpy(jmpTableStartAddress, srcJmpTable, jmpTableSizeInBytes);
for (int i = 0; i < branchJumpTableWrapper->tableSize; i++)
{
void * addressOfJmpTableEntry = jmpTableStartAddress + (i * sizeof(void*));
Assert((ptrdiff_t) addressOfJmpTableEntry - (ptrdiff_t) jmpTableStartAddress < (ptrdiff_t) jmpTableSizeInBytes);
#if defined(_M_ARM32_OR_ARM64)
encoderMD->AddLabelReloc((byte*) addressOfJmpTableEntry);
#else
encoderMD->AppendRelocEntry(RelocTypeLabelUse, addressOfJmpTableEntry, *(IR::LabelInstr**)addressOfJmpTableEntry);
*((size_t*)addressOfJmpTableEntry) = 0;
#endif
}
jmpTableStartAddress += (jmpTableSizeInBytes);
});
Assert(jmpTableStartAddress == codeStart + codeSize);
}
uint32 Encoder::GetCurrentOffset() const
{
Assert(m_pc - m_encodeBuffer <= UINT_MAX); // encode buffer size is uint32
return static_cast<uint32>(m_pc - m_encodeBuffer);
}
void Encoder::RecordInlineeFrame(Func* inlinee, uint32 currentOffset)
{
// The only restriction for not supporting loop bodies is that inlinee frame map is created on FunctionEntryPointInfo & not
// the base class EntryPointInfo.
if (!(this->m_func->IsLoopBody() && PHASE_OFF(Js::InlineInJitLoopBodyPhase, this->m_func)) && !this->m_func->IsSimpleJit())
{
InlineeFrameRecord* record = nullptr;
if (inlinee->frameInfo && inlinee->m_hasInlineArgsOpt)
{
record = inlinee->frameInfo->record;
Assert(record != nullptr);
}
if (m_inlineeFrameMap->Count() > 0)
{
// update existing record if the entry is the same.
NativeOffsetInlineeFramePair& lastPair = m_inlineeFrameMap->Item(m_inlineeFrameMap->Count() - 1);
if (lastPair.record == record)
{
lastPair.offset = currentOffset;
return;
}
}
NativeOffsetInlineeFramePair pair = { currentOffset, record };
m_inlineeFrameMap->Add(pair);
}
}
#if defined(_M_IX86) || defined(_M_X64)
/*
* ValidateCRCOnFinalBuffer
* - Validates the CRC that is last computed (could be either the one after BranchShortening or after encoding itself)
* - We calculate the CRC for jump table and dictionary after computing the code section.
* - Also, all reloc data are computed towards the end - after computing the code section - because we don't have to deal with the changes relocs while operating on the code section.
* - The version of CRC that we are validating with, doesn't have Relocs applied but the final buffer does - So we have to make adjustments while calculating the final buffer's CRC.
*/
void Encoder::ValidateCRCOnFinalBuffer(_In_reads_bytes_(finalCodeSize) BYTE * finalCodeBufferStart, size_t finalCodeSize, size_t jumpTableSize, _In_reads_bytes_(finalCodeSize) BYTE * oldCodeBufferStart, uint initialCrcSeed, uint bufferCrcToValidate, BOOL isSuccessBrShortAndLoopAlign)
{
RelocList * relocList = m_encoderMD.GetRelocList();
BYTE * currentStartAddress = finalCodeBufferStart;
BYTE * currentEndAddress = nullptr;
size_t crcSizeToCompute = 0;
size_t finalCodeSizeWithoutJumpTable = finalCodeSize - jumpTableSize;
uint finalBufferCRC = initialCrcSeed;
BYTE * oldPtr = nullptr;
if (relocList != nullptr)
{
for (int index = 0; index < relocList->Count(); index++)
{
EncodeRelocAndLabels * relocTuple = &relocList->Item(index);
//We will deal with the jump table and dictionary entries along with other reloc records in ApplyRelocs()
if ((BYTE*)m_encoderMD.GetRelocBufferAddress(relocTuple) >= oldCodeBufferStart && (BYTE*)m_encoderMD.GetRelocBufferAddress(relocTuple) < (oldCodeBufferStart + finalCodeSizeWithoutJumpTable))
{
BYTE* finalBufferRelocTuplePtr = (BYTE*)m_encoderMD.GetRelocBufferAddress(relocTuple) - oldCodeBufferStart + finalCodeBufferStart;
Assert(finalBufferRelocTuplePtr >= finalCodeBufferStart && finalBufferRelocTuplePtr < (finalCodeBufferStart + finalCodeSizeWithoutJumpTable));
uint relocDataSize = m_encoderMD.GetRelocDataSize(relocTuple);
if (relocDataSize != 0)
{
AssertMsg(oldPtr == nullptr || oldPtr < finalBufferRelocTuplePtr, "Assumption here is that the reloc list is strictly increasing in terms of bufferAddress");
oldPtr = finalBufferRelocTuplePtr;
currentEndAddress = finalBufferRelocTuplePtr;
crcSizeToCompute = currentEndAddress - currentStartAddress;
Assert(currentEndAddress >= currentStartAddress);
finalBufferCRC = CalculateCRC(finalBufferCRC, crcSizeToCompute, currentStartAddress);
for (uint i = 0; i < relocDataSize; i++)
{
finalBufferCRC = CalculateCRC(finalBufferCRC, 0);
}
currentStartAddress = currentEndAddress + relocDataSize;
}
}
}
}
currentEndAddress = finalCodeBufferStart + finalCodeSizeWithoutJumpTable;
crcSizeToCompute = currentEndAddress - currentStartAddress;
Assert(currentEndAddress >= currentStartAddress);
finalBufferCRC = CalculateCRC(finalBufferCRC, crcSizeToCompute, currentStartAddress);
//Include all offsets from the reloc records to the CRC.
m_encoderMD.ApplyRelocs((size_t)finalCodeBufferStart, finalCodeSize, &finalBufferCRC, isSuccessBrShortAndLoopAlign, true);
if (finalBufferCRC != bufferCrcToValidate)
{
Assert(false);
Fatal();
}
}
#endif
/*
* EnsureRelocEntryIntegrity
* - We compute the target address as the processor would compute it and check if the target is within the final buffer's bounds.
* - For relative addressing, Target = current m_pc + offset
* - For absolute addressing, Target = direct address
*/
void Encoder::EnsureRelocEntryIntegrity(size_t newBufferStartAddress, size_t codeSize, size_t oldBufferAddress, size_t relocAddress, uint offsetBytes, ptrdiff_t opndData, bool isRelativeAddr)
{
size_t targetBrAddress = 0;
size_t newBufferEndAddress = newBufferStartAddress + codeSize;
//Handle Dictionary addresses here - The target address will be in the dictionary.
if (relocAddress < oldBufferAddress || relocAddress >= (oldBufferAddress + codeSize))
{
targetBrAddress = (size_t)(*(size_t*)relocAddress);
}
else
{
size_t newBufferRelocAddr = relocAddress - oldBufferAddress + newBufferStartAddress;
if (isRelativeAddr)
{
targetBrAddress = (size_t)newBufferRelocAddr + offsetBytes + opndData;
}
else // Absolute Address
{
targetBrAddress = (size_t)opndData;
}
}
if (targetBrAddress < newBufferStartAddress || targetBrAddress >= newBufferEndAddress)
{
Assert(false);
Fatal();
}
}
void Encoder::ValidateCRC(uint bufferCRC, uint initialCRCSeed, _In_reads_bytes_(count) void* buffer, size_t count)
{
uint validationCRC = initialCRCSeed;
validationCRC = CalculateCRC(validationCRC, count, buffer);
if (validationCRC != bufferCRC)
{
//TODO: This throws internal error. Is this error type, Fine?
Fatal();
}
}
#if defined(_M_IX86) || defined(_M_X64)
///----------------------------------------------------------------------------
///
/// EncoderMD::ShortenBranchesAndLabelAlign
/// We try to shorten branches if the label instr is within 8-bits target range (-128 to 127)
/// and fix the relocList accordingly.
/// Also align LoopTop Label and TryCatchLabel
///----------------------------------------------------------------------------
BOOL
Encoder::ShortenBranchesAndLabelAlign(BYTE **codeStart, ptrdiff_t *codeSize, uint * pShortenedBufferCRC, uint bufferCrcToValidate, size_t jumpTableSize)
{
#ifdef ENABLE_DEBUG_CONFIG_OPTIONS
static uint32 globalTotalBytesSaved = 0, globalTotalBytesWithoutShortening = 0;
static uint32 globalTotalBytesInserted = 0; // loop alignment nops
#endif
uint32 brShortenedCount = 0;
bool codeChange = false; // any overall BR shortened or label aligned ?
BYTE* buffStart = *codeStart;
BYTE* buffEnd = buffStart + *codeSize;
ptrdiff_t newCodeSize = *codeSize;
RelocList* relocList = m_encoderMD.GetRelocList();
if (relocList == nullptr)
{
return false;
}
#if DBG
// Sanity check
m_encoderMD.VerifyRelocList(buffStart, buffEnd);
#endif
// Copy of original maps. Used to revert from BR shortening.
OffsetList *m_origInlineeFrameRecords = nullptr,
*m_origInlineeFrameMap = nullptr,
*m_origPragmaInstrToRecordOffset = nullptr;
OffsetList *m_origOffsetBuffer = nullptr;
// we record the original maps, in case we have to revert.
CopyMaps<false>(&m_origInlineeFrameRecords
, &m_origInlineeFrameMap
, &m_origPragmaInstrToRecordOffset
, &m_origOffsetBuffer );
// Here we mark BRs to be shortened and adjust Labels and relocList entries offsets.
FixUpMapIndex mapIndices;
int32 totalBytesSaved = 0;
// loop over all BRs, find the ones we can convert to short form
for (int32 j = 0; j < relocList->Count(); j++)
{
IR::LabelInstr *targetLabel;
int32 relOffset;
uint32 bytesSaved = 0;
BYTE* labelPc, *opcodeByte;
BYTE* shortBrPtr, *fixedBrPtr; // without shortening
EncodeRelocAndLabels &reloc = relocList->Item(j);
// If not a long branch, just fix the reloc entry and skip.
if (!reloc.isLongBr())
{
// if loop alignment is required, total bytes saved can change
int32 newTotalBytesSaved = m_encoderMD.FixRelocListEntry(j, totalBytesSaved, buffStart, buffEnd);
if (newTotalBytesSaved != totalBytesSaved)
{
AssertMsg(reloc.isAlignedLabel(), "Expecting aligned label.");
// we aligned a loop, fix maps
m_encoderMD.FixMaps((uint32)(reloc.getLabelOrigPC() - buffStart), totalBytesSaved, &mapIndices);
codeChange = true;
}
totalBytesSaved = newTotalBytesSaved;
continue;
}
AssertMsg(reloc.isLongBr(), "Cannot shorten already shortened branch.");
// long branch
opcodeByte = reloc.getBrOpCodeByte();
targetLabel = reloc.getBrTargetLabel();
AssertMsg(targetLabel != nullptr, "Branch to non-existing label");
labelPc = targetLabel->GetPC();
// compute the new offset of that Br because of previous shortening/alignment
shortBrPtr = fixedBrPtr = (BYTE*)reloc.m_ptr - totalBytesSaved;
if (*opcodeByte == 0xe9 /* JMP rel32 */)
{
bytesSaved = 3;
}
else if (*opcodeByte >= 0x80 && *opcodeByte < 0x90 /* Jcc rel32 */)
{
Assert(*(opcodeByte - 1) == 0x0f);
bytesSaved = 4;
// Jcc rel8 is one byte shorter in opcode, fix Br ptr to point to start of rel8
shortBrPtr--;
}
else
{
Assert(false);
}
// compute current distance to label
if (labelPc >= (BYTE*) reloc.m_ptr)
{
// forward Br. We compare using the unfixed m_ptr, because the label is ahead and its Pc is not fixed it.
relOffset = (int32)(labelPc - ((BYTE*)reloc.m_ptr + 4));
}
else
{
// backward Br. We compute relOffset after fixing the Br, since the label is already fixed.
// We also include the 3-4 bytes saved after shortening the Br since the Br itself is included in the relative offset.
relOffset = (int32)(labelPc - (shortBrPtr + 1));
}
// update Br offset (overwritten later if Br is shortened)
reloc.m_ptr = fixedBrPtr;
// can we shorten ?
if (relOffset >= -128 && relOffset <= 127)
{
uint32 brOffset;
brShortenedCount++;
// update with shortened br offset
reloc.m_ptr = shortBrPtr;
// fix all maps entries from last shortened br to this one, before updating total bytes saved.
brOffset = (uint32) ((BYTE*)reloc.m_origPtr - buffStart);
m_encoderMD.FixMaps(brOffset, totalBytesSaved, &mapIndices);
codeChange = true;
totalBytesSaved += bytesSaved;
// mark br reloc entry as shortened
#ifdef _M_IX86
reloc.setAsShortBr(targetLabel);
#else
reloc.setAsShortBr();
#endif
}
}
// Fix the rest of the maps, if needed.
if (totalBytesSaved != 0)
{
m_encoderMD.FixMaps((uint32)-1, totalBytesSaved, &mapIndices);
codeChange = true;
newCodeSize -= totalBytesSaved;
this->FixLazyBailOutThunkOffset(totalBytesSaved);
}
// no BR shortening or Label alignment happened, no need to copy code
if (!codeChange)
return codeChange;
#ifdef ENABLE_DEBUG_CONFIG_OPTIONS
globalTotalBytesWithoutShortening += (uint32)(*codeSize);
globalTotalBytesSaved += (uint32)(*codeSize - newCodeSize);
if (PHASE_TRACE(Js::BrShortenPhase, this->m_func))
{
OUTPUT_VERBOSE_TRACE(Js::BrShortenPhase, _u("func: %s, bytes saved: %d, bytes saved %%:%.2f, total bytes saved: %d, total bytes saved%%: %.2f, BR shortened: %d\n"),
this->m_func->GetJITFunctionBody()->GetDisplayName(), (*codeSize - newCodeSize), ((float)*codeSize - newCodeSize) / *codeSize * 100,
globalTotalBytesSaved, ((float)globalTotalBytesSaved) / globalTotalBytesWithoutShortening * 100 , brShortenedCount);
Output::Flush();
}
#endif
// At this point BRs are marked to be shortened, and relocList offsets are adjusted to new instruction length.
// Next, we re-write the code to shorten the BRs and adjust relocList offsets to point to new buffer.
// We also write NOPs for aligned loops.
BYTE* tmpBuffer = AnewArray(m_tempAlloc, BYTE, newCodeSize);
uint srcBufferCrc = *pShortenedBufferCRC; //This has the intial Random CRC seed to start with.
// start copying to new buffer
// this can possibly be done during fixing, but there is no evidence it is an overhead to justify the complexity.
BYTE *from = buffStart, *to = nullptr;
BYTE *dst_p = (BYTE*)tmpBuffer;
size_t dst_size = newCodeSize;
size_t src_size;
for (int32 i = 0; i < relocList->Count(); i++)
{
EncodeRelocAndLabels &reloc = relocList->Item(i);
// shorten BR and copy
if (reloc.isShortBr())
{
// validate that short BR offset is within 1 byte offset range.
// This handles the rare case with loop alignment breaks br shortening.
// Consider:
// BR $L1 // shortened
// ...
// L2: // aligned, and makes the BR $L1 non-shortable anymore
// ...
// BR $L2
// ...
// L1:
// In this case, we simply give up and revert the relocList.
if(!reloc.validateShortBrTarget())
{
revertRelocList();
// restore maps
CopyMaps<true>(&m_origInlineeFrameRecords
, &m_origInlineeFrameMap
, &m_origPragmaInstrToRecordOffset
, &m_origOffsetBuffer
);
return false;
}
// m_origPtr points to imm32 field in the original buffer
BYTE *opcodeByte = (BYTE*)reloc.m_origPtr - 1;
if (*opcodeByte == 0xe9 /* JMP rel32 */)
{
to = opcodeByte - 1;
}
else if (*opcodeByte >= 0x80 && *opcodeByte < 0x90 /* Jcc rel32 */)
{
Assert(*(opcodeByte - 1) == 0x0f);
to = opcodeByte - 2;
}
else
{
Assert(false);
}
src_size = to - from + 1;
AnalysisAssert(dst_size >= src_size);
memcpy_s(dst_p, dst_size, from, src_size);
srcBufferCrc = CalculateCRC(srcBufferCrc, (BYTE*)reloc.m_origPtr - from + 4, from);
*pShortenedBufferCRC = CalculateCRC(*pShortenedBufferCRC, src_size, dst_p);
dst_p += src_size;
dst_size -= src_size;
// fix the BR
// write new opcode
AnalysisAssert(dst_p < tmpBuffer + newCodeSize);
*dst_p = (*opcodeByte == 0xe9) ? (BYTE)0xeb : (BYTE)(*opcodeByte - 0x10);
*(dst_p + 1) = 0; // imm8
*pShortenedBufferCRC = CalculateCRC(*pShortenedBufferCRC, 2, dst_p);
dst_p += 2; // 1 byte for opcode + 1 byte for imm8
dst_size -= 2;
from = (BYTE*)reloc.m_origPtr + 4;
}
else if (reloc.m_type == RelocTypeInlineeEntryOffset)
{
to = (BYTE*)reloc.m_origPtr - 1;
CopyPartialBufferAndCalculateCRC(&dst_p, dst_size, from, to, pShortenedBufferCRC);
*(size_t*)dst_p = reloc.GetInlineOffset();
*pShortenedBufferCRC = CalculateCRC(*pShortenedBufferCRC, sizeof(size_t), dst_p);
dst_p += sizeof(size_t);
dst_size -= sizeof(size_t);
srcBufferCrc = CalculateCRC(srcBufferCrc, (BYTE*)reloc.m_origPtr + sizeof(size_t) - from , from);
from = (BYTE*)reloc.m_origPtr + sizeof(size_t);
}
// insert NOPs for aligned labels
else if ((!PHASE_OFF(Js::LoopAlignPhase, m_func) && reloc.isAlignedLabel()) && reloc.getLabelNopCount() > 0)
{
IR::LabelInstr *label = reloc.getLabel();
BYTE nop_count = reloc.getLabelNopCount();
AssertMsg((BYTE*)label < buffStart || (BYTE*)label >= buffEnd, "Invalid label pointer.");
AssertMsg((((uint32)(label->GetPC() - buffStart)) & 0xf) == 0, "Misaligned Label");
to = reloc.getLabelOrigPC() - 1;
CopyPartialBufferAndCalculateCRC(&dst_p, dst_size, from, to, pShortenedBufferCRC);
srcBufferCrc = CalculateCRC(srcBufferCrc, to - from + 1, from);
#ifdef ENABLE_DEBUG_CONFIG_OPTIONS
if (PHASE_TRACE(Js::LoopAlignPhase, this->m_func))
{
globalTotalBytesInserted += nop_count;
OUTPUT_VERBOSE_TRACE(Js::LoopAlignPhase, _u("func: %s, bytes inserted: %d, bytes inserted %%:%.4f, total bytes inserted:%d, total bytes inserted %%:%.4f\n"),
this->m_func->GetJITFunctionBody()->GetDisplayName(), nop_count, (float)nop_count / newCodeSize * 100, globalTotalBytesInserted, (float)globalTotalBytesInserted / (globalTotalBytesWithoutShortening - globalTotalBytesSaved) * 100);
Output::Flush();
}
#endif
BYTE * tmpDst_p = dst_p;
InsertNopsForLabelAlignment(nop_count, &dst_p);
*pShortenedBufferCRC = CalculateCRC(*pShortenedBufferCRC, nop_count, tmpDst_p);
dst_size -= nop_count;
from = to + 1;
}
}
// copy last chunk
//Exclude jumpTable content from CRC calculation.
//Though jumpTable is not part of the encoded bytes, codeSize has jumpTableSize included in it.
CopyPartialBufferAndCalculateCRC(&dst_p, dst_size, from, buffStart + *codeSize - 1, pShortenedBufferCRC, jumpTableSize);
srcBufferCrc = CalculateCRC(srcBufferCrc, buffStart + *codeSize - from - jumpTableSize, from);
m_encoderMD.UpdateRelocListWithNewBuffer(relocList, tmpBuffer, buffStart, buffEnd);
if (srcBufferCrc != bufferCrcToValidate)
{
Assert(false);
Fatal();
}
// switch buffers
*codeStart = tmpBuffer;
*codeSize = newCodeSize;
return true;
}
BYTE Encoder::FindNopCountFor16byteAlignment(size_t address)
{
return (16 - (BYTE) (address & 0xf)) % 16;
}
void Encoder::CopyPartialBufferAndCalculateCRC(BYTE ** ptrDstBuffer, size_t &dstSize, BYTE * srcStart, BYTE * srcEnd, uint* pBufferCRC, size_t jumpTableSize)
{
BYTE * destBuffer = *ptrDstBuffer;
size_t srcSize = srcEnd - srcStart + 1;
Assert(dstSize >= srcSize);
memcpy_s(destBuffer, dstSize, srcStart, srcSize);
Assert(srcSize >= jumpTableSize);
//Exclude the jump table content (which is at the end of the buffer) for calculating CRC - at this point.
*pBufferCRC = CalculateCRC(*pBufferCRC, srcSize - jumpTableSize, destBuffer);
*ptrDstBuffer += srcSize;
dstSize -= srcSize;
}
void Encoder::InsertNopsForLabelAlignment(int nopCount, BYTE ** ptrDstBuffer)
{
// write NOPs
for (int32 i = 0; i < nopCount; i++, (*ptrDstBuffer)++)
{
**ptrDstBuffer = 0x90;
}
}
void Encoder::revertRelocList()
{
RelocList* relocList = m_encoderMD.GetRelocList();
for (int32 i = 0; i < relocList->Count(); i++)
{
relocList->Item(i).revert();
}
}
template <bool restore>
void Encoder::CopyMaps(OffsetList **m_origInlineeFrameRecords
, OffsetList **m_origInlineeFrameMap
, OffsetList **m_origPragmaInstrToRecordOffset
, OffsetList **m_origOffsetBuffer
)
{
InlineeFrameRecords *recList = m_inlineeFrameRecords;
ArenaInlineeFrameMap *mapList = m_inlineeFrameMap;
PragmaInstrList *pInstrList = m_pragmaInstrToRecordOffset;
OffsetList *origRecList, *origMapList, *origPInstrList;
if (!restore)
{
Assert(*m_origInlineeFrameRecords == nullptr);
Assert(*m_origInlineeFrameMap == nullptr);
Assert(*m_origPragmaInstrToRecordOffset == nullptr);
*m_origInlineeFrameRecords = origRecList = Anew(m_tempAlloc, OffsetList, m_tempAlloc);
*m_origInlineeFrameMap = origMapList = Anew(m_tempAlloc, OffsetList, m_tempAlloc);
*m_origPragmaInstrToRecordOffset = origPInstrList = Anew(m_tempAlloc, OffsetList, m_tempAlloc);
#if DBG_DUMP
Assert((*m_origOffsetBuffer) == nullptr);
*m_origOffsetBuffer = Anew(m_tempAlloc, OffsetList, m_tempAlloc);
#endif
}
else
{
Assert((*m_origInlineeFrameRecords) && (*m_origInlineeFrameMap) && (*m_origPragmaInstrToRecordOffset));
origRecList = *m_origInlineeFrameRecords;
origMapList = *m_origInlineeFrameMap;
origPInstrList = *m_origPragmaInstrToRecordOffset;
Assert(origRecList->Count() == recList->Count());
Assert(origMapList->Count() == mapList->Count());
Assert(origPInstrList->Count() == pInstrList->Count());
#if DBG_DUMP
Assert(m_origOffsetBuffer);
Assert((uint32)(*m_origOffsetBuffer)->Count() == m_instrNumber);
#endif
}
for (int i = 0; i < recList->Count(); i++)
{
if (!restore)
{
origRecList->Add(recList->Item(i)->inlineeStartOffset);
}
else
{
recList->Item(i)->inlineeStartOffset = origRecList->Item(i);
}
}
for (int i = 0; i < mapList->Count(); i++)
{
if (!restore)
{
origMapList->Add(mapList->Item(i).offset);
}
else
{
mapList->Item(i).offset = origMapList->Item(i);
}
}
for (int i = 0; i < pInstrList->Count(); i++)
{
if (!restore)
{
origPInstrList->Add(pInstrList->Item(i)->m_offsetInBuffer);
}
else
{
pInstrList->Item(i)->m_offsetInBuffer = origPInstrList->Item(i);
}
}
if (restore)
{
(*m_origInlineeFrameRecords)->Delete();
(*m_origInlineeFrameMap)->Delete();
(*m_origPragmaInstrToRecordOffset)->Delete();
(*m_origInlineeFrameRecords) = nullptr;
(*m_origInlineeFrameMap) = nullptr;
(*m_origPragmaInstrToRecordOffset) = nullptr;
}
#if DBG_DUMP
for (uint i = 0; i < m_instrNumber; i++)
{
if (!restore)
{
(*m_origOffsetBuffer)->Add(m_offsetBuffer[i]);
}
else
{
m_offsetBuffer[i] = (*m_origOffsetBuffer)->Item(i);
}
}
if (restore)
{
(*m_origOffsetBuffer)->Delete();
(*m_origOffsetBuffer) = nullptr;
}
#endif
}
#endif
#if DBG_DUMP
void Encoder::DumpInlineeFrameMap(size_t baseAddress)
{
Output::Print(_u("Inlinee frame info mapping\n"));
Output::Print(_u("---------------------------------------\n"));
m_inlineeFrameMap->Map([=](uint index, NativeOffsetInlineeFramePair& pair) {
Output::Print(_u("%Ix"), baseAddress + pair.offset);
Output::SkipToColumn(20);
if (pair.record)
{
pair.record->Dump();
}
else
{
Output::Print(_u("<NULL>"));
}
Output::Print(_u("\n"));
});
}
#endif
void
Encoder::SaveToLazyBailOutRecordList(IR::Instr* instr, uint32 currentOffset)
{
BailOutInfo* bailOutInfo = instr->GetBailOutInfo();
Assert(instr->OnlyHasLazyBailOut() && bailOutInfo->bailOutRecord != nullptr);
#if DBG_DUMP
if (PHASE_DUMP(Js::LazyBailoutPhase, m_func))
{
Output::Print(_u("Offset: %u Instr: "), currentOffset);
instr->Dump();
Output::Print(_u("Bailout label: "));
bailOutInfo->bailOutInstr->Dump();
}
#endif
LazyBailOutRecord record(currentOffset, bailOutInfo->bailOutRecord);
this->m_sortedLazyBailoutRecordList->Add(record);
}
void
Encoder::SaveLazyBailOutThunkOffset(uint32 currentOffset)
{
AssertMsg(
this->m_lazyBailOutThunkOffset == 0,
"We should only have one thunk generated during final lowerer"
);
this->m_lazyBailOutThunkOffset = this->GetCurrentOffset();
}
void
Encoder::SaveLazyBailOutJitTransferData()
{
if (this->m_func->HasLazyBailOut())
{
Assert(this->m_sortedLazyBailoutRecordList->Count() > 0);
Assert(this->m_lazyBailOutThunkOffset != 0);
Assert(this->m_func->GetLazyBailOutRecordSlot() != nullptr);
auto nativeEntryPointData = this->m_func->GetInProcJITEntryPointInfo()->GetInProcNativeEntryPointData();
nativeEntryPointData->SetSortedLazyBailOutRecordList(this->m_sortedLazyBailoutRecordList);
nativeEntryPointData->SetLazyBailOutRecordSlotOffset(this->m_func->GetLazyBailOutRecordSlot()->m_offset);
nativeEntryPointData->SetLazyBailOutThunkOffset(this->m_lazyBailOutThunkOffset);
}
if (this->m_func->lazyBailoutProperties.Count() > 0)
{
const int count = this->m_func->lazyBailoutProperties.Count();
Js::PropertyId* lazyBailoutProperties = HeapNewArrayZ(Js::PropertyId, count);
Js::PropertyId* dstProperties = lazyBailoutProperties;
this->m_func->lazyBailoutProperties.Map([&](Js::PropertyId propertyId)
{
*dstProperties++ = propertyId;
});
this->m_func->GetInProcJITEntryPointInfo()->GetJitTransferData()->SetLazyBailoutProperties(lazyBailoutProperties, count);
}
}
void
Encoder::FixLazyBailOutThunkOffset(uint32 bytesSaved)
{
// Lazy bailout thunk is inserted at the end of the function,
// so just decrease the offset by the number of bytes saved
this->m_lazyBailOutThunkOffset -= bytesSaved;
}