From 8cc600d326917bda81cbfabc28efc3482b3dbe12 Mon Sep 17 00:00:00 2001 From: Tsung Yeung Date: Tue, 16 Jul 2019 09:28:01 -0400 Subject: [PATCH] Re-order NVDIMM MRS restore sequence to comply with Jedec spec Also does the following: 1. Remove STR from RCD restore sequence (non-issue for IDT RCD) 2. MCFGP_VALID api for HB to consume during MPIPL Change-Id: I7f00d96d286e1568bcf2580afe9cf11111110b5e CQ:SW470208 Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/80470 Tested-by: FSP CI Jenkins Tested-by: Jenkins Server Reviewed-by: STEPHEN GLANCY Reviewed-by: Louis Stermole Tested-by: Hostboot CI Reviewed-by: Dean Sanner Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/80584 Tested-by: Jenkins OP Build CI Tested-by: Jenkins OP HW Reviewed-by: Daniel M Crowell --- .../hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.C | 92 ++++++++++++---- .../hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.H | 9 ++ .../hwp/memory/lib/dimm/ddr4/nvdimm_utils.C | 100 +++++------------- .../hwp/memory/lib/dimm/ddr4/nvdimm_utils.H | 17 ++- .../procedures/hwp/memory/lib/dimm/mrs_load.C | 12 ++- 5 files changed, 130 insertions(+), 100 deletions(-) diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.C b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.C index a3b88797be2..3ac509b6c1b 100644 --- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.C +++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.C @@ -146,28 +146,82 @@ fapi2::ReturnCode mrs_load( const fapi2::Target& i_target, const size_t DOUBLE_TMRD = 2 * mss::tmrd(); const size_t DOUBLE_TMOD = 2 * mss::tmod(i_target); - static const std::vector< mrs_data > MRS_DATA = + // tDLLK to wait for DLL Reset + uint64_t tDLLK = 0; + FAPI_TRY( mss::tdllk(i_target, tDLLK) ); + { - // JEDEC ordering of MRS per DDR4 power on sequence - { 3, mrs03, mrs03_decode, DOUBLE_TMRD }, - { 6, mrs06, mrs06_decode, DOUBLE_TMRD }, - { 5, mrs05, mrs05_decode, DOUBLE_TMRD }, - { 4, mrs04, mrs04_decode, DOUBLE_TMRD }, - { 2, mrs02, mrs02_decode, DOUBLE_TMRD }, - { 1, mrs01, mrs01_decode, DOUBLE_TMRD }, - // We need to wait tmod before zqcl, a non-mrs command - { 0, mrs00, mrs00_decode, DOUBLE_TMOD }, - }; - - std::vector< uint64_t > l_ranks; - FAPI_TRY( mss::rank::ranks(i_target, l_ranks) ); - - // Load MRS - for (const auto& d : MRS_DATA) + const std::vector< mrs_data > MRS_DATA = + { + // JEDEC ordering of MRS per DDR4 power on sequence + { 3, mrs03, mrs03_decode, DOUBLE_TMRD }, + { 6, mrs06, mrs06_decode, DOUBLE_TMRD }, + { 5, mrs05, mrs05_decode, DOUBLE_TMRD }, + { 4, mrs04, mrs04_decode, DOUBLE_TMRD }, + { 2, mrs02, mrs02_decode, DOUBLE_TMRD }, + { 1, mrs01, mrs01_decode, DOUBLE_TMRD }, + // We need to wait tmod before zqcl, a non-mrs command + // Adding Per Glancy's request, to ensure DLL locking time + { 0, mrs00, mrs00_decode, DOUBLE_TMOD + tDLLK }, + }; + + std::vector< uint64_t > l_ranks; + FAPI_TRY( mss::rank::ranks(i_target, l_ranks) ); + + // Load MRS + for (const auto& d : MRS_DATA) + { + for (const auto& r : l_ranks) + { + FAPI_TRY( mrs_engine(i_target, d, r, io_inst) ); + } + } + } + +fapi_try_exit: + return fapi2::current_err; +} + +/// +/// @brief Perform the mrs_load DDR4 operations for nvdimm restore - TARGET_TYPE_DIMM specialization +/// @param[in] i_target a fapi2::Target +/// @param[in] io_inst a vector of CCS instructions we should add to +/// @return FAPI2_RC_SUCCESS if and only if ok +/// +fapi2::ReturnCode mrs_load_nvdimm( const fapi2::Target& i_target, + std::vector< ccs::instruction_t >& io_inst) +{ + FAPI_INF("ddr4::mrs_load_nvdimm %s", mss::c_str(i_target)); + + // tDLLK to wait for DLL Reset + uint64_t tDLLK = 0; + FAPI_TRY( mss::tdllk(i_target, tDLLK) ); + { - for (const auto& r : l_ranks) + const std::vector< mrs_data > MRS_DATA = + { + // JEDEC ordering of MRS per DDR4 NVDIMM restore sequence + // Need to perform DLL off to on procedure (mrs01&mrs00) + // before all the other MRS's + { 1, mrs01, mrs01_decode, mss::tmrd() }, + { 0, mrs00, mrs00_decode, mss::tmod(i_target) + tDLLK }, + { 3, mrs03, mrs03_decode, mss::tmrd() }, + { 6, mrs06, mrs06_decode, mss::tmrd() }, + { 5, mrs05, mrs05_decode, mss::tmrd() }, + { 4, mrs04, mrs04_decode, mss::tmrd() }, + { 2, mrs02, mrs02_decode, mss::tmrd() }, + }; + + std::vector< uint64_t > l_ranks; + FAPI_TRY( mss::rank::ranks(i_target, l_ranks) ); + + // Load MRS + for (const auto& d : MRS_DATA) { - FAPI_TRY( mrs_engine(i_target, d, r, io_inst) ); + for (const auto& r : l_ranks) + { + FAPI_TRY( mrs_engine(i_target, d, r, io_inst) ); + } } } diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.H b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.H index f9d0ac06237..5e6aa5c3551 100644 --- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.H +++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/mrs_load_ddr4.H @@ -1510,6 +1510,15 @@ struct mrs06_data fapi2::ReturnCode mrs_load( const fapi2::Target& i_target, std::vector< ccs::instruction_t >& io_inst); +/// +/// @brief Perform the mrs_load DDR4 operations for nvdimm restore - TARGET_TYPE_DIMM specialization +/// @param[in] i_target a fapi2::Target +/// @param[in] io_inst a vector of CCS instructions we should add to +/// @return FAPI2_RC_SUCCESS if and only if ok +/// +fapi2::ReturnCode mrs_load_nvdimm( const fapi2::Target& i_target, + std::vector< ccs::instruction_t >& io_inst); + /// /// @brief Sets WR LVL mode /// @param[in] i_target a DIMM target diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.C b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.C index 9178379dd0d..0d517ffbce7 100644 --- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.C +++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.C @@ -327,49 +327,6 @@ fapi_try_exit: return fapi2::current_err; } -/// -/// @brief Disable powerdown mode in rc09 -/// @param[in] i_target, a fapi2::Target -/// @param[in,out] io_inst a vector of CCS instructions we should add to -/// @return FAPI2_RC_SUCCESS if and only if ok -/// -fapi2::ReturnCode rc09_disable_powerdown( const fapi2::Target& i_target, - std::vector< ccs::instruction_t >& io_inst) -{ - FAPI_INF("rc09_disable_powerdown %s", mss::c_str(i_target)); - - constexpr uint8_t POWER_DOWN_BIT = 4; - constexpr bool l_sim = false; - constexpr uint8_t FS0 = 0; // Function space 0 - constexpr uint64_t CKE_HIGH = mss::ON; - fapi2::buffer l_rc09_cw = 0; - std::vector l_ranks; - - FAPI_TRY(mss::eff_dimm_ddr4_rc09(i_target, l_rc09_cw)); - - // Clear power down enable bit. - l_rc09_cw.clearBit(); - - FAPI_TRY( mss::rank::ranks(i_target, l_ranks) ); - - // DES to ensure we exit powerdown properly - FAPI_DBG("deselect for %s", mss::c_str(i_target)); - io_inst.push_back( ccs::des_command() ); - - static const cw_data l_rc09_4bit_data( FS0, 9, l_rc09_cw, mss::tmrd() ); - - // Load RC09 - FAPI_TRY( control_word_engine(i_target, l_rc09_4bit_data, l_sim, io_inst, CKE_HIGH), - "Failed to load 4-bit RC09 control word for %s", - mss::c_str(i_target)); - - // Hold the CKE high - mss::ccs::workarounds::hold_cke_high(io_inst); - -fapi_try_exit: - return fapi2::current_err; -} - /// /// @brief Load the rcd control words /// @param[in] i_target, a fapi2::Target @@ -383,7 +340,7 @@ fapi2::ReturnCode rcd_load_nvdimm( const fapi2::Target& i_targ { FAPI_INF("rcd_load_nvdimm %s", mss::c_str(i_target)); - constexpr uint64_t CKE_LOW = mss::OFF; + constexpr uint64_t CKE_HIGH = mss::ON; constexpr bool l_sim = false; // Per DDR4RCD02, tSTAB is us. We want this in cycles for the CCS. @@ -431,17 +388,19 @@ fapi2::ReturnCode rcd_load_nvdimm( const fapi2::Target& i_targ }; // Load 4-bit data - FAPI_TRY( control_word_engine(i_target, l_rcd_4bit_data, l_sim, io_inst, CKE_LOW), + // Keeping the CKE high as this will be done not in powerdown/STR mode. Not affected for + // the RCD supplier on nvdimm + FAPI_TRY( control_word_engine(i_target, l_rcd_4bit_data, l_sim, io_inst, CKE_HIGH), "Failed to load 4-bit control words for %s", mss::c_str(i_target)); // Load 8-bit data - FAPI_TRY( control_word_engine(i_target, l_rcd_8bit_data, l_sim, io_inst, CKE_LOW), + FAPI_TRY( control_word_engine(i_target, l_rcd_8bit_data, l_sim, io_inst, CKE_HIGH), "Failed to load 8-bit control words for %s", mss::c_str(i_target)); // Load RC09 with CKE_LOW - FAPI_TRY( control_word_engine(i_target, l_rc09_4bit_data, l_sim, io_inst, CKE_LOW), + FAPI_TRY( control_word_engine(i_target, l_rc09_4bit_data, l_sim, io_inst, CKE_HIGH), "Failed to load 4-bit RC09 control word for %s", mss::c_str(i_target)); @@ -469,27 +428,6 @@ fapi2::ReturnCode rcd_restore( const fapi2::Target& i_target ) l_program.iv_poll.iv_initial_delay = 0; l_program.iv_poll.iv_initial_sim_delay = 0; - // We expect to come in with the port in STR. Before proceeding with - // restoring the RCD, power down needs to be disabled first on the RCD so - // the rest of the CWs can be restored with CKE low - for ( const auto& d : mss::find_targets(i_target) ) - { - FAPI_DBG("rc09_disable_powerdown for %s", mss::c_str(d)); - FAPI_TRY( rc09_disable_powerdown(d, l_program.iv_instructions), - "Failed rc09_disable_powerdown() for %s", mss::c_str(d) ); - }// dimms - - // Exit STR first so CKE is back to high and rcd isn't ignoring us - FAPI_TRY( self_refresh_exit( i_target ) ); - - FAPI_TRY( mss::ccs::workarounds::nvdimm::execute(l_mcbist, l_program, i_target), - "Failed to execute ccs for %s", mss::c_str(i_target) ); - - // Now, drive CKE back to low via STR entry instead of pde (we have data in the drams!) - FAPI_TRY( self_refresh_entry( i_target ) ); - - l_program = ccs::program(); //Reset the program - // Now, fill the program with instructions to program the RCD for ( const auto& d : mss::find_targets(i_target) ) { @@ -614,12 +552,12 @@ fapi2::ReturnCode post_restore_transition( const fapi2::Target& i_target, + const uint8_t i_state) +{ + const auto& l_mcs = mss::find_target(i_target); + fapi2::buffer l_data; + + FAPI_TRY( mss::getScom(l_mcs, MCS_MCFGP, l_data) ); + l_data.writeBit(i_state); + FAPI_INF("Changing MCS_MCFGP_VALID to %d on %s", i_state, mss::c_str(l_mcs)); + FAPI_TRY( mss::putScom(l_mcs, MCS_MCFGP, l_data) ); + +fapi_try_exit: + return fapi2::current_err; +} + /// /// @brief Preload the CCS with the EPOW sequence /// @param[in] i_target the target associated with this subroutine diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.H b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.H index 148273a9ae3..ad5fb0fa0d6 100644 --- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.H +++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/ddr4/nvdimm_utils.H @@ -181,15 +181,6 @@ fapi2::ReturnCode self_refresh_entry( const fapi2::Target& i_target ); template< fapi2::TargetType T > fapi2::ReturnCode self_refresh_exit( const fapi2::Target& i_target ); -/// -/// @brief Disable powerdown mode in rc09 -/// @param[in] i_target, a fapi2::Target -/// @param[in,out] io_inst a vector of CCS instructions we should add to -/// @return FAPI2_RC_SUCCESS if and only if ok -/// -fapi2::ReturnCode rc09_disable_powerdown( const fapi2::Target& i_target, - std::vector< ccs::instruction_t >& io_inst); - /// /// @brief Load the rcd control words /// @param[in] i_target, a fapi2::Target @@ -236,6 +227,14 @@ fapi2::ReturnCode wr_vref_latch( const fapi2::Target& i_ template< fapi2::TargetType T > fapi2::ReturnCode post_restore_transition( const fapi2::Target& i_target ); +/// +/// @brief Helper to change the BAR valid state. Consumed by hostboot +/// @param[in] i_target the target associated with this subroutine +/// @return FAPI2_RC_SUCCESS iff setup was successful +/// +fapi2::ReturnCode change_bar_valid_state( const fapi2::Target& i_target, + const uint8_t i_state); + /// /// @brief Preload the CCS with the EPOW sequence /// @param[in] i_target the target associated with this subroutine diff --git a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/mrs_load.C b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/mrs_load.C index 31a69f30df9..8928a6f988e 100644 --- a/src/import/chips/p9/procedures/hwp/memory/lib/dimm/mrs_load.C +++ b/src/import/chips/p9/procedures/hwp/memory/lib/dimm/mrs_load.C @@ -72,7 +72,17 @@ fapi2::ReturnCode mrs_load( const fapi2::Target(i_target) ) { FAPI_DBG("mrs load for %s", mss::c_str(d)); - FAPI_TRY( perform_mrs_load(d, l_program.iv_instructions) ); + + // TK - break out the nvdimm stuff into function + if (i_nvdimm_workaround) + { + FAPI_DBG("nvdimm workaround detected. loading mrs for restore sequence"); + FAPI_TRY( ddr4::mrs_load_nvdimm(d, l_program.iv_instructions) ); + } + else + { + FAPI_TRY( perform_mrs_load(d, l_program.iv_instructions) ); + } } // We have to configure the CCS engine to let it know which port these instructions are