Fixes CKE levels during RCD initialization

Change-Id: Ic00be58a3e972407e944ebdeff9a16c01c1ee3e9
CQ:SW432711
RTC:194935
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/59645
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Dev-Ready: STEPHEN GLANCY <sglancy@us.ibm.com>
Reviewed-by: Louis Stermole <stermole@us.ibm.com>
Reviewed-by: ANDRE A. MARIN <aamarin@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: HWSV CI <hwsv-ci+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/59652
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

src/import/chips/p9/procedures/hwp/memory/lib/ccs/ccs.C

@@ -207,11 +207,14 @@ fapi2::ReturnCode execute( const fapi2::Target<TARGET_TYPE_MCBIST>& i_target,
         uint64_t l_total_delay = 0;
         uint64_t l_delay = 0;
         uint64_t l_repeat = 0;
+        uint8_t l_current_cke = 0;
 
         // Shove the instructions into the CCS engine, in 32 instruction chunks, and execute them
         for (; l_inst_iter != i_program.iv_instructions.end()
              && l_inst_count < CCS_INSTRUCTION_DEPTH; ++l_inst_count, ++l_inst_iter)
         {
+            l_inst_iter->arr0.extractToRight<TT::ARR0_DDR_CKE, TT::ARR0_DDR_CKE_LEN>(l_current_cke);
+
             // Make sure this instruction leads to the next. Notice this limits this mechanism to pretty
             // simple (straight line) CCS programs. Anything with a loop or such will need another mechanism.
             l_inst_iter->arr1.insertFromRight<MCBIST_CCS_INST_ARR1_00_GOTO_CMD,
@@ -249,6 +252,9 @@ fapi2::ReturnCode execute( const fapi2::Target<TARGET_TYPE_MCBIST>& i_target,
         FAPI_INF("executing ccs instructions (%d:%d, %d) for %s",
                  i_program.iv_instructions.size(), l_inst_count, i_program.iv_poll.iv_initial_delay, mss::c_str(i_target));
 
+        // Deselect
+        l_des.arr0.insertFromRight<TT::ARR0_DDR_CKE, TT::ARR0_DDR_CKE_LEN>(l_current_cke);
+
         // Insert a DES as our last instruction. DES is idle state anyway and having this
         // here as an instruction forces the CCS engine to wait the delay specified in
         // the last instruction in this array (which it otherwise doesn't do.)

src/import/chips/p9/procedures/hwp/memory/lib/ccs/ccs.H

@@ -394,6 +394,7 @@ static void mrs_rcd_helper( fapi2::buffer<uint64_t>& i_arr0 )
 ///
 template< fapi2::TargetType T, typename TT = ccsTraits<T> >
 inline instruction_t<T> rcd_command( const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
+                                     const bool i_sim,
                                      const bool i_turn_on_cke = true)
 {
     fapi2::buffer<uint64_t> rcd_boilerplate_arr0;
@@ -404,13 +405,10 @@ inline instruction_t<T> rcd_command( const fapi2::Target<fapi2::TARGET_TYPE_DIMM
     //
     mrs_rcd_helper<fapi2::TARGET_TYPE_MCBIST>(rcd_boilerplate_arr0);
 
-    uint8_t l_sim = 0;
-    mss::is_simulation(l_sim);
-
     // Not adding i_turn_on_cke in the mrs_rcd helper because we only need this
     // for RCWs and there is no need to complicate/change the MRS cmd API with
     // uneeded functionality. Little duplication, but this isolates the change.
-    if( !l_sim )
+    if( !i_sim )
     {
         const uint64_t l_cke = i_turn_on_cke ? CKE_HIGH : CKE_LOW;
         rcd_boilerplate_arr0.insertFromRight<TT::ARR0_DDR_CKE, TT::ARR0_DDR_CKE_LEN>(l_cke);

src/import/chips/p9/procedures/hwp/memory/lib/dimm/bcw_load_ddr4.C

@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -66,8 +66,11 @@ fapi2::ReturnCode bcw_load_ddr4( const fapi2::Target<TARGET_TYPE_DIMM>& i_target
 {
     FAPI_INF("bcw_load_ddr4 %s", mss::c_str(i_target) );
 
-    // Per DDR4BC01
+    uint8_t l_sim = 0;
     uint64_t l_tDLLK = 0;
+    FAPI_TRY(mss::is_simulation(l_sim));
+
+    // Per DDR4BC01
     FAPI_TRY( tdllk(i_target, l_tDLLK), "Failed to get tDLLK for %s in bcw_load_ddr4", mss::c_str(i_target) );
 
     {
@@ -102,7 +105,7 @@ fapi2::ReturnCode bcw_load_ddr4( const fapi2::Target<TARGET_TYPE_DIMM>& i_target
         // We set the 4-bit buffer control words first (they live in function space 0
         // hw is supposed to default to function space 0 but Just.In.Case.
         FAPI_TRY( ddr4::function_space_select<0>(i_target, io_inst), "Failed function space select 0", mss::c_str(i_target));
-        FAPI_TRY( control_word_engine<BCW_4BIT>(i_target, l_bcw_4bit_data, io_inst) , "Failed control_word_engine",
+        FAPI_TRY( control_word_engine<BCW_4BIT>(i_target, l_bcw_4bit_data, l_sim, io_inst) , "Failed control_word_engine",
                   mss::c_str(i_target));
 
         // We set our 8-bit buffer control words but have to switch function space
@@ -112,14 +115,14 @@ fapi2::ReturnCode bcw_load_ddr4( const fapi2::Target<TARGET_TYPE_DIMM>& i_target
         {
             cw_data l_data(FUNC_SPACE_6, BUFF_TRAIN_CONFIG_CW, eff_dimm_ddr4_f6bc4x, mss::tmrc());
             FAPI_TRY( ddr4::function_space_select<6>(i_target, io_inst), "Failed function space select 6", mss::c_str(i_target) );
-            FAPI_TRY( control_word_engine<BCW_8BIT>(i_target, l_data, io_inst), "Failed control_word_engine",
+            FAPI_TRY( control_word_engine<BCW_8BIT>(i_target, l_data, l_sim, io_inst), "Failed control_word_engine",
                       mss::c_str(i_target) );
         }
 
         {
             cw_data l_data(FUNC_SPACE_5, DRAM_VREF_CW, eff_dimm_ddr4_f5bc6x, mss::tmrc());
             FAPI_TRY( ddr4::function_space_select<5>(i_target, io_inst), "Failed function space select 5", mss::c_str(i_target) );
-            FAPI_TRY( control_word_engine<BCW_8BIT>(i_target, l_data, io_inst), "Failed control_word_engine",
+            FAPI_TRY( control_word_engine<BCW_8BIT>(i_target, l_data, l_sim, io_inst), "Failed control_word_engine",
                       mss::c_str(i_target) );
         }
 

src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/control_word_ddr4.H

@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2016,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2016,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -274,49 +274,67 @@ struct cw_data
 template< control_word T, typename TT = cwTraits<T>, fapi2::TargetType OT >
 fapi2::ReturnCode control_word_engine(const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
                                       const cw_data& i_data,
+                                      const bool i_sim,
                                       std::vector< ccs::instruction_t<OT> >& io_inst,
                                       const bool i_turn_on_cke = true)
 {
-    ccs::instruction_t<OT> l_inst =  ccs::rcd_command<OT>(i_target, i_turn_on_cke);
-
-    // A12 is the RCW/ BCW selector
-    constexpr uint64_t DDR_ADDRESS_12 = 12;
-    l_inst.arr0.template writeBit<DDR_ADDRESS_12>(TT::CW_ACCESS);
-
-    // For user defined data, iv_data is user defined and iv_attr_get is a NO-OP
-    // For attribute defined data, iv_attr_get will define data and l_value initialization is overwritten
-    // I need l_value integral because the attribute accessor template deduction doesn't let me use buffers
-    // and since I'm passing in bcw data as const I can't pass in iv_data to the attribute accessor
-    // which would break const correctness
-    uint8_t l_value = i_data.iv_data;
-    FAPI_TRY( i_data.iv_attr_get(i_target, l_value) );
-
-    // Data to be written into the configuration registers
-    // 4-bit control are containned in bits DA0 thorugh DA3
-    // 8-bit control are contained in bits DA0 thorugh DA7
-    mss::swizzle< MCBIST_CCS_INST_ARR0_00_DDR_ADDRESS_0_13,
-        TT::DATA_LEN, TT::SWIZZLE_START >(fapi2::buffer<uint8_t>(l_value), l_inst.arr0);
-
-    // Selection of each word of control bits
-    mss::swizzle < MCBIST_CCS_INST_ARR0_00_DDR_ADDRESS_0_13 + TT::DATA_LEN,
-        TT::WORD_LEN, TT::SWIZZLE_START > (i_data.iv_number, l_inst.arr0);
-
-    // For changes to the control word setting [...] the controller needs to wait some time
-    // the last control word access, before further access to the DRAM can take place.
-    l_inst.arr1.template insertFromRight<MCBIST_CCS_INST_ARR1_00_IDLES,
-                         MCBIST_CCS_INST_ARR1_00_IDLES_LEN>(i_data.iv_delay);
-
-    FAPI_INF("F%d%s%02x%s value 0x%x (%d cycles) 0x%016llx:0x%016llx %s",
-             uint8_t(i_data.iv_func_space),
-             (TT::CW_ACCESS == 1 ? "BC" : "RC"),
-             uint8_t(i_data.iv_number),
-             (T == BCW_4BIT || T == RCW_4BIT ? "" : "X"),
-             l_value,
-             i_data.iv_delay,
-             uint64_t(l_inst.arr0), uint64_t(l_inst.arr1),
-             mss::c_str(i_target));
-
-    io_inst.push_back(l_inst);
+    // You're probably asking "Why always turn off CKE's? What is this madness?"
+    // Well, due to a vendor sensitivity, we need to have the CKE's off until we run RC09 at the very end
+    // Unfortunately, we need to have the CKE's off on the DIMM we are running second
+    // We also don't want to turn off the CKE's on the DIMM we are running first
+    // Therefore, we want to setup all RCW commands to have CKE's off across both DIMM's
+    // We then manually turn on the CKE's associated with a specific DIMM
+    constexpr bool CKE_OFF = false;
+    ccs::instruction_t<OT> l_inst =  ccs::rcd_command<OT>(i_target, i_sim, CKE_OFF);
+
+    // Turn on the CKE's for the ranks we're not touching, if it's needed
+    // Note: we only have the whole CKE field, not the per DIMM one by default
+    constexpr uint8_t CKE_PER_DIMM = 2;
+    const uint64_t CKE_START = MCBIST_CCS_INST_ARR0_00_DDR_CKE + (mss::index(i_target) * CKE_PER_DIMM);
+    FAPI_TRY(l_inst.arr0.writeBit(i_sim || i_turn_on_cke, CKE_START, CKE_PER_DIMM));
+
+    // Run everything else
+    {
+
+        // A12 is the RCW/ BCW selector
+        constexpr uint64_t DDR_ADDRESS_12 = 12;
+        l_inst.arr0.template writeBit<DDR_ADDRESS_12>(TT::CW_ACCESS);
+
+        // For user defined data, iv_data is user defined and iv_attr_get is a NO-OP
+        // For attribute defined data, iv_attr_get will define data and l_value initialization is overwritten
+        // I need l_value integral because the attribute accessor template deduction doesn't let me use buffers
+        // and since I'm passing in bcw data as const I can't pass in iv_data to the attribute accessor
+        // which would break const correctness
+        uint8_t l_value = i_data.iv_data;
+        FAPI_TRY( i_data.iv_attr_get(i_target, l_value) );
+
+        // Data to be written into the configuration registers
+        // 4-bit control are containned in bits DA0 thorugh DA3
+        // 8-bit control are contained in bits DA0 thorugh DA7
+        mss::swizzle< MCBIST_CCS_INST_ARR0_00_DDR_ADDRESS_0_13,
+            TT::DATA_LEN, TT::SWIZZLE_START >(fapi2::buffer<uint8_t>(l_value), l_inst.arr0);
+
+        // Selection of each word of control bits
+        mss::swizzle < MCBIST_CCS_INST_ARR0_00_DDR_ADDRESS_0_13 + TT::DATA_LEN,
+            TT::WORD_LEN, TT::SWIZZLE_START > (i_data.iv_number, l_inst.arr0);
+
+        // For changes to the control word setting [...] the controller needs to wait some time
+        // the last control word access, before further access to the DRAM can take place.
+        l_inst.arr1.template insertFromRight<MCBIST_CCS_INST_ARR1_00_IDLES,
+                             MCBIST_CCS_INST_ARR1_00_IDLES_LEN>(i_data.iv_delay);
+
+        FAPI_INF("F%d%s%02x%s value 0x%x (%d cycles) 0x%016llx:0x%016llx %s",
+                 uint8_t(i_data.iv_func_space),
+                 (TT::CW_ACCESS == 1 ? "BC" : "RC"),
+                 uint8_t(i_data.iv_number),
+                 (T == BCW_4BIT || T == RCW_4BIT ? "" : "X"),
+                 l_value,
+                 i_data.iv_delay,
+                 uint64_t(l_inst.arr0), uint64_t(l_inst.arr1),
+                 mss::c_str(i_target));
+
+        io_inst.push_back(l_inst);
+    }
 
 fapi_try_exit:
     return fapi2::current_err;
@@ -336,6 +354,7 @@ fapi_try_exit:
 template< control_word T, typename TT = cwTraits<T>, fapi2::TargetType OT >
 fapi2::ReturnCode control_word_engine(const fapi2::Target<fapi2::TARGET_TYPE_DIMM>& i_target,
                                       const std::vector<cw_data>& i_data_list,
+                                      const bool i_sim,
                                       std::vector< ccs::instruction_t<OT> >& io_inst,
                                       const bool i_turn_on_cke = true)
 {
@@ -347,7 +366,7 @@ fapi2::ReturnCode control_word_engine(const fapi2::Target<fapi2::TARGET_TYPE_DIM
 
     for (const auto& data : i_data_list)
     {
-        FAPI_TRY( control_word_engine<T>(i_target, data, io_inst, i_turn_on_cke) );
+        FAPI_TRY( control_word_engine<T>(i_target, data, i_sim, io_inst, i_turn_on_cke) );
     }
 
 fapi_try_exit:

src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/data_buffer_ddr4.H

@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2015,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2015,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -131,6 +131,9 @@ inline fapi2::ReturnCode function_space_select(const fapi2::Target< fapi2::TARGE
     // From DB02 spec - F[3:0]BC7x control word
     constexpr size_t MAX_FUNC_SPACE = 7;
 
+    uint8_t l_sim = 0;
+    mss::is_simulation(l_sim);
+
     // Function spaces are typically user selected.
     // Sometimes function spaces are selected based on rank number
     // or MDQ lane number which currently are also user inputs.
@@ -142,7 +145,7 @@ inline fapi2::ReturnCode function_space_select(const fapi2::Target< fapi2::TARGE
     // don't cares (XXX).  We choose 0 for simplicity.
     cw_data l_data( FUNC_SPACE_0, FUNC_SPACE_SELECT_CW, T, mss::tmrd() );
 
-    FAPI_TRY( control_word_engine<BCW_8BIT>(i_target, l_data, io_inst),
+    FAPI_TRY( control_word_engine<BCW_8BIT>(i_target, l_data, l_sim, io_inst),
               "%s. Failed control_word_engine for 8-bit BCW (F%dBC%02lxX)",
               mss::c_str(i_target), uint8_t(l_data.iv_func_space), uint8_t(l_data.iv_number) );
 
@@ -183,11 +186,14 @@ inline fapi2::ReturnCode function_space_select(const fapi2::Target< fapi2::TARGE
         fapi2::Assert(false);
     }
 
+    uint8_t l_sim = 0;
+    mss::is_simulation(l_sim);
+
     // function space bits for the function space selector are
     // don't cares (XXX).  We choose 0 for simplicity.
     cw_data l_data( FUNC_SPACE_0, FUNC_SPACE_SELECT_CW, i_func_space, mss::tmrd() );
 
-    FAPI_TRY( control_word_engine<BCW_8BIT>(i_target, l_data, io_inst),
+    FAPI_TRY( control_word_engine<BCW_8BIT>(i_target, l_data, l_sim, io_inst),
               "%s. Failed control_word_engine for 8-bit BCW (F%dBC%02lxX)",
               mss::c_str(i_target), uint8_t(l_data.iv_func_space), uint8_t(l_data.iv_number) );
 
@@ -214,11 +220,14 @@ static fapi2::ReturnCode settings_boilerplate(const fapi2::Target< fapi2::TARGET
         const cw_data& i_data,
         std::vector< ccs::instruction_t<fapi2::TARGET_TYPE_MCBIST> >& io_inst)
 {
+    uint8_t l_sim = 0;
+    mss::is_simulation(l_sim);
+
     FAPI_TRY( function_space_select(i_target, i_data.iv_func_space, io_inst),
               "%s. Failed to select function space %d",
               mss::c_str(i_target), uint8_t(i_data.iv_func_space) );
 
-    FAPI_TRY( control_word_engine<T>(i_target, i_data, io_inst),
+    FAPI_TRY( control_word_engine<T>(i_target, i_data, l_sim, io_inst),
               "%s. Failed control_word_engine for 8-bit BCW (F%dBC%02lxX)",
               mss::c_str(i_target), uint8_t(i_data.iv_func_space), uint8_t(i_data.iv_number) );
 

src/import/chips/p9/procedures/hwp/memory/lib/dimm/rcd_load.C

@@ -151,8 +151,11 @@ template<>
 fapi2::ReturnCode perform_rcd_load<KIND_RDIMM_DDR4>( const fapi2::Target<TARGET_TYPE_DIMM>& i_target,
         std::vector< ccs::instruction_t<TARGET_TYPE_MCBIST> >& i_inst)
 {
+    uint8_t l_sim = 0;
+    FAPI_TRY( mss::is_simulation(l_sim) );
+
     FAPI_DBG("perform rcd_load for %s [expecting rdimm (ddr4)]", mss::c_str(i_target));
-    FAPI_TRY( rcd_load_ddr4(i_target, i_inst),
+    FAPI_TRY( rcd_load_ddr4(i_target, l_sim, i_inst),
               "Failed rcd_load_ddr4() for %s", mss::c_str(i_target) );
 
 fapi_try_exit:
@@ -169,8 +172,11 @@ template<>
 fapi2::ReturnCode perform_rcd_load<KIND_LRDIMM_DDR4>( const fapi2::Target<TARGET_TYPE_DIMM>& i_target,
         std::vector< ccs::instruction_t<TARGET_TYPE_MCBIST> >& i_inst)
 {
+    uint8_t l_sim = 0;
+    FAPI_TRY( mss::is_simulation(l_sim) );
+
     FAPI_DBG("perform rcd_load for %s [expecting lrdimm (ddr4)]", mss::c_str(i_target));
-    FAPI_TRY( rcd_load_ddr4(i_target, i_inst) );
+    FAPI_TRY( rcd_load_ddr4(i_target, l_sim, i_inst) );
 
 fapi_try_exit:
     return fapi2::current_err;

src/import/chips/p9/procedures/hwp/memory/lib/dimm/rcd_load_ddr4.C

@@ -39,6 +39,7 @@
 #include <lib/dimm/rcd_load_ddr4.H>
 #include <lib/dimm/ddr4/control_word_ddr4.H>
 #include <lib/workarounds/draminit_workarounds.H>
+#include <lib/workarounds/ccs_workarounds.H>
 
 using fapi2::TARGET_TYPE_MCBIST;
 using fapi2::TARGET_TYPE_MCA;
@@ -53,10 +54,12 @@ namespace mss
 ///
 /// @brief Perform the rcd_load_ddr4 operations - TARGET_TYPE_DIMM specialization
 /// @param[in] i_target, a fapi2::Target<TARGET_TYPE_DIMM>
+/// @param[in] i_sim, true IFF in simulation mode
 /// @param[in,out] io_inst a vector of CCS instructions we should add to
 /// @return FAPI2_RC_SUCCESS if and only if ok
 ///
 fapi2::ReturnCode rcd_load_ddr4( const fapi2::Target<TARGET_TYPE_DIMM>& i_target,
+                                 const bool i_sim,
                                  std::vector< ccs::instruction_t<TARGET_TYPE_MCBIST> >& io_inst)
 {
     FAPI_INF("rcd_load_ddr4 %s", mss::c_str(i_target));
@@ -115,24 +118,28 @@ fapi2::ReturnCode rcd_load_ddr4( const fapi2::Target<TARGET_TYPE_DIMM>& i_target
     // Then with CKE high (We raise it w/the RCW): 4-bit RC09
 
     // Load 4-bit data
-    FAPI_TRY( control_word_engine<RCW_4BIT>(i_target, l_rcd_4bit_data, io_inst, CKE_LOW),
+    FAPI_TRY( control_word_engine<RCW_4BIT>(i_target, l_rcd_4bit_data, i_sim, io_inst, CKE_LOW),
               "Failed to load 4-bit control words for %s",
               mss::c_str(i_target));
 
     // Load 8-bit data
-    FAPI_TRY( control_word_engine<RCW_8BIT>(i_target, l_rcd_8bit_data, io_inst, CKE_LOW),
+    FAPI_TRY( control_word_engine<RCW_8BIT>(i_target, l_rcd_8bit_data, i_sim, io_inst, CKE_LOW),
               "Failed to load 8-bit control words for %s",
               mss::c_str(i_target));
 
     // Load RC09
-    FAPI_TRY( control_word_engine<RCW_4BIT>(i_target, l_rc09_4bit_data, io_inst, CKE_HIGH),
+    FAPI_TRY( control_word_engine<RCW_4BIT>(i_target, l_rc09_4bit_data, i_sim, io_inst, CKE_HIGH),
               "Failed to load 4-bit RC09 control word for %s",
               mss::c_str(i_target));
 
     // Toggle RC06 again to ensure the DRAM is reset properly
-    FAPI_TRY( mss::workarounds::rcw_reset_dram(i_target, io_inst), "%s failed to add reset workaround functionality",
+    FAPI_TRY( mss::workarounds::rcw_reset_dram(i_target, i_sim, io_inst), "%s failed to add reset workaround functionality",
               mss::c_str(i_target));
 
+    // Now, hold the CKE's high, so we don't power down the RCD and re power it back up
+    mss::ccs::workarounds::hold_cke_high(io_inst);
+
+
 fapi_try_exit:
     return fapi2::current_err;
 }