Add a function to break on DF set

Add a function to generate DF flag check in JITCODE and trap when DF is
incorrectly set. Downstream project can add the check to any places they
want.

The motivation of this check is to catch a case where memory can be
corrupted due to incorrect DF flag affecting `rep` instructions. One
case is found in Openj9 but the memory corruption is hard to reproduce
and not all memory corruption can result in a test failure or crash.

However a wrong DF flag should be easier to detect if the check is
executed frequently enough.

Signed-off-by: Liqun Liu <liqunl@ca.ibm.com>

compiler/x/codegen/OMRX86Instruction.cpp

@@ -3923,6 +3923,29 @@ TR::AMD64RegImm64SymInstruction::autoSetReloKind()
 // Generate methods
 ////////////////////////////////////////////////////////////////////////////////
 
+TR::Instruction* generateBreakOnDFSet(TR::CodeGenerator *cg, TR::Instruction* cursor)
+   {
+   if (!cursor)
+      cursor = cg->getAppendInstruction();
+
+   TR::RealRegister *espReal = cg->machine()->getRealRegister(TR::RealRegister::esp);
+   cursor = generateInstruction(cursor, PUSHFD, cg);
+   TR::LabelSymbol* begLabel = generateLabelSymbol(cg);
+   TR::LabelSymbol* endLabel = generateLabelSymbol(cg);
+   begLabel->setStartInternalControlFlow();
+   endLabel->setEndInternalControlFlow();
+
+   const int32_t dfMask = 0x400;
+   cursor = generateLabelInstruction(cursor, LABEL, begLabel, cg);
+   cursor = generateMemImmInstruction(cursor, TEST2MemImm2, generateX86MemoryReference(espReal, 0, cg), dfMask, cg);
+   cursor = generateLabelInstruction(cursor, JE1, endLabel, cg);
+   cursor = generateInstruction(cursor, BADIA32Op, cg);
+   cursor = generateLabelInstruction(cursor, LABEL, endLabel, cg);
+   cursor = generateInstruction(cursor, POPFD, cg);
+
+   return cursor;
+   }
+
 TR::Instruction  *
 generateInstruction(TR::Instruction *prev, TR_X86OpCodes op, TR::CodeGenerator *cg)
    {

compiler/x/codegen/OMRX86Instruction.hpp

@@ -2937,6 +2937,21 @@ TR::X86RegImmInstruction  * generateRegImmInstruction(TR::Instruction *, TR_X86O
 TR::X86RegMemInstruction  * generateRegMemInstruction(TR::Instruction *, TR_X86OpCodes op, TR::Register * reg1, TR::MemoryReference  * mr, TR::CodeGenerator *cg);
 TR::X86RegRegInstruction  * generateRegRegInstruction(TR::Instruction *, TR_X86OpCodes op, TR::Register * reg1, TR::Register * reg2, TR::CodeGenerator *cg);
 
+/** \brief
+ *   Insert instructions to check DF flag is in the right state (zero) and trap if not
+ *
+ *  \param cg
+ *   The Code Generator
+ *
+ *  \param cursor
+ *   The instruction the generated sequence to be inserted after. If is NULL, the check
+ *   will be inserted after the most recently generated instruction
+ *
+ *  \return
+ *   Return the last instruction of the sequence
+ */
+TR::Instruction  * generateBreakOnDFSet(TR::CodeGenerator *cg, TR::Instruction* cursor = NULL);
+
 TR::Instruction  * generateInstruction(TR_X86OpCodes, TR::Node *, TR::RegisterDependencyConditions  * cond, TR::CodeGenerator *cg);
 TR::Instruction  * generateInstruction(TR_X86OpCodes op, TR::Node * node, TR::CodeGenerator *cg);
 TR::Instruction  * generateInstruction(TR::Instruction *prev, TR_X86OpCodes op, TR::CodeGenerator *cg);