Skip to content

Commit

Permalink
Updates LRDIMM code to utilize board swizzling
Browse files Browse the repository at this point in the history 
Change-Id: Icf88be43338526e94384bd2df1d9f478d88260c9
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/68284
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: Louis Stermole <stermole@us.ibm.com>
Reviewed-by: Devon A. Baughen <devon.baughen1@ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/68341
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
  • Loading branch information
@sglancy6 @dcrowell77
sglancy6 authored and dcrowell77 committed Nov 9, 2018
1 parent 0faf0e0 commit a690866
Show file tree
Hide file tree
Showing 2 changed files with 240 additions and 29 deletions.
164 changes: 135 additions & 29 deletions src/import/chips/p9/procedures/hwp/memory/lib/phy/mss_lrdimm_training.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
#include <lib/workarounds/ccs_workarounds.H>
#include <lib/ccs/ccs.H>
#include <lib/mc/port.H>
#include <lib/rosetta_map/rosetta_map.H>

namespace mss
{
Expand All @@ -57,6 +58,87 @@ namespace training
namespace lrdimm
{

///
/// @brief Swizzles a DQ from the MC perspective to the DIMM perspective
/// @param[in] i_target the MCA target on which to operate
/// @param[in] i_mc_dq the DQ on the MC perspective to swizzle to the buffer's perspective
/// @param[out] o_buffer_dq the DQ number from the buffer's perspective
/// @return fapi2::ReturnCode fapi2::FAPI2_RC_SUCCESS iff ok
///
fapi2::ReturnCode mc_to_dimm_dq(const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
const uint64_t i_mc_dq,
uint64_t& o_buffer_dq)
{
uint64_t l_c4_dq = 0;
uint8_t* l_dimm_dq_ptr = nullptr;
uint8_t l_dimm_to_c4[MAX_DQ_BITS] = {};

// First get the c4 DQ
FAPI_TRY(rosetta_map::mc_to_c4<rosetta_type::DQ>( i_target, i_mc_dq, l_c4_dq ));

// Now get the DIMM DQ
FAPI_TRY(vpd_dq_map(i_target, &l_dimm_to_c4[0]));
l_dimm_dq_ptr = std::find(l_dimm_to_c4, l_dimm_to_c4 + MAX_DQ_BITS, l_c4_dq);

// Check that we got a good value
FAPI_ASSERT(l_dimm_dq_ptr != l_dimm_to_c4 + MAX_DQ_BITS,
fapi2::MSS_LOOKUP_FAILED()
.set_KEY(l_c4_dq)
.set_DATA(i_mc_dq)
.set_TARGET(i_target));

// Now return that value
o_buffer_dq = *l_dimm_dq_ptr;

fapi_try_exit :
return fapi2::current_err;
}

///
/// @brief Swizzles a DQ from the DRAM perspective to the buffer's perspective
/// @param[in] i_target the MCA target on which to operate
/// @param[in] i_mc_dq the DQ on the MC perspective to swizzle to the buffer's perspective
/// @param[out] o_buffer_dq the DQ number from the buffer's perspective
/// @return fapi2::ReturnCode fapi2::FAPI2_RC_SUCCESS iff ok
///
fapi2::ReturnCode mc_to_buffer(const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
const uint64_t i_mc_dq,
uint64_t& o_buffer_dq)
{
FAPI_TRY(mc_to_dimm_dq(i_target, i_mc_dq, o_buffer_dq));

// Each buffer is a byte and we want the bit from the buffer's perspective
o_buffer_dq = o_buffer_dq % BITS_PER_BYTE;

fapi_try_exit :
return fapi2::current_err;
}

///
/// @brief Checks if a buffer's nibbles are swizzled
/// @param[in] i_target the MCA target on which to operate
/// @param[in] i_buffer the buffer on which to see if the nibbles are swizzled
/// @param[out] o_are_swizzled true if the buffer's nibbles are swizzled
/// @return fapi2::ReturnCode fapi2::FAPI2_RC_SUCCESS iff ok
///
fapi2::ReturnCode are_buffers_nibbles_swizzled(const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
const uint64_t i_buffer,
bool& o_are_swizzled)
{
const auto l_mc_dq = i_buffer * BITS_PER_BYTE;
uint64_t l_buffer_dq = 0;
FAPI_TRY(mc_to_buffer(i_target, l_mc_dq, l_buffer_dq));

// Now, checks if the nibble is swizzled
// We're swizzled if the 0'th DQ from the MC perspective is on buffer's nibble 1
o_are_swizzled = (1 == (l_buffer_dq / BITS_PER_NIBBLE));
FAPI_DBG("%s buffer:%u %s swizzled mc_dq:%u buffer_dq:%u",
mss::c_str(i_target), i_buffer, o_are_swizzled ? "are" : "not", l_mc_dq, l_buffer_dq);

fapi_try_exit :
return fapi2::current_err;
}

///
/// @brief Issues initial pattern write to all ranks in the rank pair
/// @param[in] i_target the MCA target on which to operate
Expand Down Expand Up @@ -246,38 +328,54 @@ fapi2::ReturnCode mrep::write_result_to_buffers_helper( const fapi2::Target<fapi
// Clears out the PBA container to ensure we don't issue undesired commands
o_container.clear();

// Get's the MCA
const auto& l_mca = mss::find_target<fapi2::TARGET_TYPE_MCA>(i_target);

// Looops through and generates the PBA commands
for(const auto& l_recorder : i_mrep_result)
{
const auto l_result_nibble0 = l_recorder.first.iv_final_delay;
const auto l_result_nibble1 = l_recorder.second.iv_final_delay;

// Function space is derived from the rank
// 2 is for Nibble 0, 3 is for Nibble 1
// Data corresponds to the final setting we have
// Delay is for PBA, bumping it way out so we don't have issues
constexpr uint64_t PBA_DELAY = 255;
constexpr uint64_t BCW_NIBBLE0 = 0x02;
constexpr uint64_t BCW_NIBBLE1 = 0x03;

const mss::cw_info MREP_FINAL_SET_BCW_N0( i_rank,
BCW_NIBBLE0,
l_result_nibble0,
PBA_DELAY,
mss::CW8_DATA_LEN,
mss::cw_info::BCW);
const mss::cw_info MREP_FINAL_SET_BCW_N1( i_rank,
BCW_NIBBLE1,
l_result_nibble1,
PBA_DELAY,
mss::CW8_DATA_LEN,
mss::cw_info::BCW);

// Each buffer contains two nibbles
// Each nibble corresponds to one BCW
// Add in the buffer control words
FAPI_TRY(o_container.add_command(i_target, l_buffer, MREP_FINAL_SET_BCW_N0));
FAPI_TRY(o_container.add_command(i_target, l_buffer, MREP_FINAL_SET_BCW_N1));
bool l_are_nibbles_swapped = false;
FAPI_TRY(are_buffers_nibbles_swizzled(l_mca, l_buffer, l_are_nibbles_swapped));

{
const auto l_result_nibble0 = l_are_nibbles_swapped ?
l_recorder.second.iv_final_delay :
l_recorder.first.iv_final_delay;
const auto l_result_nibble1 = l_are_nibbles_swapped ?
l_recorder.first.iv_final_delay :
l_recorder.second.iv_final_delay;

FAPI_DBG("%s MREP rank%u buffer:%u final values (0x%02x,0x%02x) %s swapped BC2x:0x%02x BC3x:0x%02x",
mss::c_str(l_mca), i_rank, l_buffer, l_recorder.first.iv_final_delay, l_recorder.second.iv_final_delay,
l_are_nibbles_swapped ? "are" : "not", l_result_nibble0, l_result_nibble1);

// Function space is derived from the rank
// 2 is for Nibble 0, 3 is for Nibble 1
// Data corresponds to the final setting we have
// Delay is for PBA, bumping it way out so we don't have issues
constexpr uint64_t PBA_DELAY = 255;
constexpr uint64_t BCW_NIBBLE0 = 0x02;
constexpr uint64_t BCW_NIBBLE1 = 0x03;

const mss::cw_info MREP_FINAL_SET_BCW_N0( i_rank,
BCW_NIBBLE0,
l_result_nibble0,
PBA_DELAY,
mss::CW8_DATA_LEN,
mss::cw_info::BCW);
const mss::cw_info MREP_FINAL_SET_BCW_N1( i_rank,
BCW_NIBBLE1,
l_result_nibble1,
PBA_DELAY,
mss::CW8_DATA_LEN,
mss::cw_info::BCW);

// Each buffer contains two nibbles
// Each nibble corresponds to one BCW
// Add in the buffer control words
FAPI_TRY(o_container.add_command(i_target, l_buffer, MREP_FINAL_SET_BCW_N0));
FAPI_TRY(o_container.add_command(i_target, l_buffer, MREP_FINAL_SET_BCW_N1));
}

++l_buffer;
}
Expand Down Expand Up @@ -517,6 +615,14 @@ fapi2::ReturnCode mrep::run( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_targ
FAPI_DBG("%s RP%u rank index number %u has rank %u", mss::c_str(i_target), i_rp, l_rank_index, l_rank);
const auto& l_dimm = l_dimms[mss::rank::get_dimm_from_rank(l_rank)];

// Added in for cronus debug - not needed for hostboot
#ifndef __HOSTBOOT_MODULE
// Prints the header
FAPI_DBG("%s CARD AAAAAAAAAA RCD BBBBBBBB", mss::c_str(i_target));
FAPI_DBG("%s CARD 0000000000 RCD 11111111", mss::c_str(i_target));
FAPI_DBG("%s CARD 0516273849 RCD 04152637", mss::c_str(i_target));
#endif

// Vector represents the number of LRDIMM buffers
// The pair represents the two nibbles that we need to calibrate within the buffer
std::vector<std::pair<recorder, recorder>> l_results_recorder(MAX_LRDIMM_BUFFERS);
Expand Down
105 changes: 105 additions & 0 deletions src/import/chips/p9/procedures/hwp/memory/lib/phy/mss_lrdimm_training.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
#include <lib/mcbist/mcbist.H>
#include <lib/dimm/ddr4/pba.H>
#include <lib/workarounds/ccs_workarounds.H>
#include <lib/rosetta_map/rosetta_map.H>

namespace mss
{
Expand Down Expand Up @@ -201,6 +202,40 @@ fapi2::ReturnCode mpr_pattern_wr_rank(const fapi2::Target<fapi2::TARGET_TYPE_MCA
const uint64_t i_rank,
const uint32_t i_pattern);

///
/// @brief Swizzles a DQ from the MC perspective to the DIMM perspective
/// @param[in] i_target the MCA target on which to operate
/// @param[in] i_mc_dq the DQ on the MC perspective to swizzle to the buffer's perspective
/// @param[out] o_buffer_dq the DQ number from the buffer's perspective
/// @return fapi2::ReturnCode fapi2::FAPI2_RC_SUCCESS iff ok
///
fapi2::ReturnCode mc_to_dimm_dq(const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
const uint64_t i_mc_dq,
uint64_t& o_buffer_dq);

///
/// @brief Swizzles a DQ from the MC perspective to the buffer's perspective
/// @param[in] i_target the MCA target on which to operate
/// @param[in] i_mc_dq the DQ on the MC perspective to swizzle to the buffer's perspective
/// @param[out] o_buffer_dq the DQ number from the buffer's perspective
/// @return fapi2::ReturnCode fapi2::FAPI2_RC_SUCCESS iff ok
///
fapi2::ReturnCode mc_to_buffer(const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
const uint64_t i_mc_dq,
uint64_t& o_buffer_dq);

///
/// @brief Checks if a buffer's nibbles are swizzled
/// @param[in] i_target the MCA target on which to operate
/// @param[in] i_buffer the buffer on which to see if the nibbles are swizzled
/// @param[out] o_are_swizzled true if the buffer's nibbles are swizzled
/// @return fapi2::ReturnCode fapi2::FAPI2_RC_SUCCESS iff ok
///
fapi2::ReturnCode are_buffers_nibbles_swizzled(const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target,
const uint64_t i_buffer,
bool& o_are_swizzled);

///
/// @class data_response
/// @brief A structure that stores the results from the CCS on a per-buffer per-beat level
///
Expand Down Expand Up @@ -246,9 +281,79 @@ struct data_response
l_data.extractToRight<0, BITS_PER_BYTE>(iv_buffer_beat[8][l_beat]);
}

// Added in for cronus debug - not needed for hostboot
#ifndef __HOSTBOOT_MODULE
FAPI_TRY(buffer_print(i_target));
#endif

fapi_try_exit:
return fapi2::current_err;
}

// Added in for cronus debug - not needed for hostboot
#ifndef __HOSTBOOT_MODULE
///
/// @brief Prints out the response data for LRDIMM's in terms of the DRAM position
/// @param[in] i_target MCA target on which to operate
/// @return fapi2::ReturnCode SUCCESS iff code exectutes successfully
/// @note Should be for an RC B2 - swizzle might not be good for non-RC B2
///
fapi2::ReturnCode buffer_print(const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_target)
{
// The below list is taken from rawcard B's JEDEC specification
// Assuming it remains the same for other raw card's, but this is just for debug prints for cronus anways
static constexpr uint8_t NIBBLE_TO_DRAM[MAX_DRAMS_X4] =
{
0,
5,
1,
6,
7,
2,
8,
3,
10,
14,
11,
15,
12,
16,
17,
13,
9,
4,
};
uint8_t l_results[MAX_DRAMS_X4] = {};

// Loops through all buffers
for(uint8_t l_buffer = 0; l_buffer < MAX_LRDIMM_BUFFERS; ++l_buffer)
{
uint64_t l_buffer_nibble0_dq = 0;
uint64_t l_buffer_nibble1_dq = 0;
uint64_t l_dram_0 = 0;
uint64_t l_dram_1 = 0;
FAPI_TRY(mc_to_dimm_dq(i_target, l_buffer * BITS_PER_BYTE, l_buffer_nibble0_dq));
FAPI_TRY(mc_to_dimm_dq(i_target, l_buffer * BITS_PER_BYTE + BITS_PER_NIBBLE, l_buffer_nibble1_dq));

l_dram_0 = NIBBLE_TO_DRAM[l_buffer_nibble0_dq / BITS_PER_NIBBLE];
l_dram_1 = NIBBLE_TO_DRAM[l_buffer_nibble1_dq / BITS_PER_NIBBLE];

l_results[l_dram_0] = iv_buffer_beat[l_buffer][0] & 0x0f;
l_results[l_dram_1] = (iv_buffer_beat[l_buffer][0] & 0xf0) >> BITS_PER_NIBBLE;
}

FAPI_DBG("%s CARDU %x%x%x%x%x RCD %x%x%x%x", mss::c_str(i_target),
l_results[0], l_results[1], l_results[2], l_results[3], l_results[4], l_results[10], l_results[11], l_results[12],
l_results[13]);

FAPI_DBG("%s CARDL %x%x%x%x%x RCD %x%x%x%x", mss::c_str(i_target),
l_results[5], l_results[6], l_results[7], l_results[8], l_results[9], l_results[14], l_results[15], l_results[16],
l_results[17]);

fapi_try_exit:
return fapi2::current_err;
}
#endif
};

// TK:LRDIMM Do we need separate coarse vs fine steps?
Expand Down

0 comments on commit a690866

Please sign in to comment.