Skip to content

Commit

Permalink
Add p9a_throttle_sync procedure
Browse files Browse the repository at this point in the history 
Change-Id: Ia89d521993387b4467b334f29a9be3f93bda8f75
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/79579
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Louis Stermole <stermole@us.ibm.com>
Reviewed-by: Michael D. Pardeik <pardeik@us.ibm.com>
Tested-by: HWSV CI <hwsv-ci+hostboot@us.ibm.com>
Tested-by: Hostboot CI <hostboot-ci+hostboot@us.ibm.com>
Reviewed-by: Jennifer A Stofer <stofer@us.ibm.com>
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/79583
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>
  • Loading branch information
@markypizz @crgeddes
markypizz authored and crgeddes committed Jul 11, 2019
1 parent bde6d57 commit 264bea8
Show file tree
Hide file tree
Showing 5 changed files with 431 additions and 0 deletions.
322 changes: 322 additions & 0 deletions src/import/chips/p9/procedures/hwp/nest/p9a_throttle_sync.C
View file
Expand Up @@ -22,3 +22,325 @@
/* permissions and limitations under the License. */
/* */
/* IBM_PROLOG_END_TAG */
/// ----------------------------------------------------------------------------
/// @file p9_throttle_sync.C
///
/// @brief Perform p9_throttle_sync HWP
///
/// The purpose of this procedure is to triggers sync command from a 'master'
/// MC to other MCs that have attached memory in a processor.
///
/// ----------------------------------------------------------------------------
/// *HWP HWP Owner : Joe McGill <jmcgill@us.ibm.com>
/// *HWP HWP Backup : Mark Pizzutillo <Mark.Pizzutillo@ibm.com>
/// *HWP FW Owner : Thi Tran <thi@us.ibm.com>
/// *HWP Team : Nest
/// *HWP Level : 3
/// *HWP Consumed by : HB
/// ----------------------------------------------------------------------------

//------------------------------------------------------------------------------
// Includes
//------------------------------------------------------------------------------
#include <p9a_throttle_sync.H>
#include <fapi2.H>
#include <generic/memory/lib/utils/find.H>
#include <generic/memory/lib/utils/c_str.H>
#include <lib/shared/axone_consts.H>

///
/// @brief Program MCMODE0 based on the functional targets
///
/// @param[in] i_mi_target The MC target to be programmed
/// @param[in] i_mi_targets Other MC targets.
///
/// @return FAPI2_RC_SUCCESS if success, else error code.
///
fapi2::ReturnCode prog_MCMODE0(
const fapi2::Target<fapi2::TARGET_TYPE_MI>& i_mc_target,
const std::vector< fapi2::Target<fapi2::TARGET_TYPE_MI> >& i_mc_targets)
{
FAPI_DBG("Entering prog_MCMODE0 on target %s", mss::c_str(i_mc_target));
// --------------------------------------------------------------
// Setup MCMODE0 for disabling MC SYNC to other-side and same-side
// partner unit.
// BIT27: set if other-side MC is non-functional, 0<->2, 1<->3
// BIT28: set if same-side MC is non-functional, 0<->1, 2<->3
// --------------------------------------------------------------
fapi2::buffer<uint64_t> l_scomData(0);
fapi2::buffer<uint64_t> l_scomMask(0);
bool l_other_side_functional = false;
bool l_same_side_functional = false;
uint8_t l_current_pos = 0;
uint8_t l_other_side_pos = 0;
uint8_t l_same_side_pos = 0;

FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, i_mc_target, l_current_pos),
"Error getting ATTR_CHIP_UNIT_POS on %s", mss::c_str(i_mc_target));

// Calculate the peer MC in the other side and in the same side.
l_other_side_pos = (l_current_pos + MAX_MC_PER_SIDE) % MAX_MC_PER_PROC;
l_same_side_pos = ((l_current_pos / MAX_MC_SIDES_PER_PROC) * MAX_MC_PER_SIDE)
+ ((l_current_pos % MAX_MC_PER_SIDE) + 1) % MAX_MC_PER_SIDE;

FAPI_DBG("Current pos: %i, other side pos: %i, same side pos: %i",
l_current_pos, l_other_side_pos, l_same_side_pos);

// Determine side functionality
for (const auto& l_mc : i_mc_targets)
{
uint8_t l_tmp_pos = 0;
FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_mc, l_tmp_pos),
"Error getting ATTR_CHIP_UNIT_POS on %s", mss::c_str(l_mc));

// The other side
if (l_tmp_pos == l_other_side_pos)
{
l_other_side_functional = true;
}

// The same side
if (l_tmp_pos == l_same_side_pos)
{
l_same_side_functional = true;
}
}

l_scomData.flush<0>();
l_scomMask.flush<0>();

if (!l_other_side_functional)
{
l_scomData.setBit<P9A_MI_MCMODE0_DISABLE_MC_SYNC>();
l_scomMask.setBit<P9A_MI_MCMODE0_DISABLE_MC_SYNC>();
}
else
{
l_scomData.clearBit<P9A_MI_MCMODE0_DISABLE_MC_SYNC>();
l_scomMask.setBit<P9A_MI_MCMODE0_DISABLE_MC_SYNC>();
}

if (!l_same_side_functional)
{
l_scomData.setBit<P9A_MI_MCMODE0_DISABLE_MC_PAIR_SYNC>();
l_scomMask.setBit<P9A_MI_MCMODE0_DISABLE_MC_PAIR_SYNC>();
}
else
{
l_scomData.clearBit<P9A_MI_MCMODE0_DISABLE_MC_PAIR_SYNC>();
l_scomMask.setBit<P9A_MI_MCMODE0_DISABLE_MC_PAIR_SYNC>();
}

FAPI_DBG("Writing MCS_MCMODE0 reg 0x%016llX: Mask 0x%016llX , Data 0x%016llX",
P9A_MI_MCMODE0, l_scomMask, l_scomData);

FAPI_TRY(fapi2::putScomUnderMask(i_mc_target, P9A_MI_MCMODE0, l_scomData, l_scomMask),
"putScomUnderMask() returns an error, P9A_MI_MCMODE0 reg 0x%016llX", P9A_MI_MCMODE0);

fapi_try_exit:
FAPI_DBG("Exiting prog_MCMODE0");
return fapi2::current_err;
}

///
/// @brief Programming master MCS
/// Writes P9A_MI_MCSYNC reg to set the input MCS as the master.
///
/// @param[in] i_mcsTarget The MCS target to be programmed as master.
/// @return FAPI2_RC_SUCCESS if success, else error code.
///
fapi2::ReturnCode prog_master(const fapi2::Target<fapi2::TARGET_TYPE_MI>& i_mc_target)
{
FAPI_DBG("Entering progMaster with target %s", mss::c_str(i_mc_target));
fapi2::buffer<uint64_t> l_scomData(0);
fapi2::buffer<uint64_t> l_scomMask(0);

// -------------------------------------------------------------------
// 1. Reset sync command
// -------------------------------------------------------------------

// Clear GO bit
l_scomMask.flush<0>().setBit<P9A_MI_MCSYNC_SYNC_GO>();
l_scomData.flush<0>();
FAPI_TRY(fapi2::putScomUnderMask(i_mc_target, P9A_MI_MCSYNC, l_scomData, l_scomMask),
"putScomUnderMask() returns an error (Reset), P9A_MI_MCSYNC reg 0x%016llX", P9A_MI_MCSYNC);

// --------------------------------------------------------------
// 2. Setup MC Sync Command Register data for master MCS or MI
// --------------------------------------------------------------

// Clear buffers
l_scomData.flush<0>();
l_scomMask.flush<0>();

// Force bit set in case cleared from last procedure run
l_scomData.setBit<EXPLR_SRQ_MBA_SYNCCNTLQ_SYNC_REF_EN>();
l_scomMask.setBit<EXPLR_SRQ_MBA_SYNCCNTLQ_SYNC_REF_EN>();

// Iterate through OCMBs to make sure refresh SYNC bit is set
for (const auto& l_ocmb : mss::find_targets<fapi2::TARGET_TYPE_OCMB_CHIP>(i_mc_target))
{
FAPI_DBG("Writing EXPLR_SRQ_MBA_SYNCCNTLQ reg 0x%016llX: Data 0x%016llX Mask 0x%016llX",
EXPLR_SRQ_MBA_SYNCCNTLQ, l_scomData, l_scomMask);

FAPI_TRY(fapi2::putScomUnderMask(l_ocmb, EXPLR_SRQ_MBA_SYNCCNTLQ, l_scomData, l_scomMask),
"Error writing to REG 0x%016llX of %s", EXPLR_SRQ_MBA_SYNCCNTLQ, mss::c_str(l_ocmb));
}

// Clear buffers
l_scomData.flush<0>();
l_scomMask.flush<0>();

// Setup MCSYNC_CHANNEL_SELECT
// Set ALL channels with or without DIMMs (bits 0:7)
l_scomData.setBit<P9A_MI_MCSYNC_CHANNEL_SELECT,
P9A_MI_MCSYNC_CHANNEL_SELECT_LEN>();
l_scomMask.setBit<P9A_MI_MCSYNC_CHANNEL_SELECT,
P9A_MI_MCSYNC_CHANNEL_SELECT_LEN>();

// Setup MCSYNC_SYNC_TYPE for SYNC ALL
l_scomData.setBit<P9A_MI_MCSYNC_SYNC_TYPE>();
l_scomMask.setBit<P9A_MI_MCSYNC_SYNC_TYPE>();

// Setup SYNC_GO (bit 16 is now used for both channels)
l_scomMask.setBit<P9A_MI_MCSYNC_SYNC_GO>();
l_scomData.setBit<P9A_MI_MCSYNC_SYNC_GO>();

// --------------------------------------------------------------
// 3. Write to MC Sync Command Register of master MCS or MI
// --------------------------------------------------------------
// Write to MCSYNC reg
FAPI_DBG("Writing P9A_MI_MCSYNC reg 0x%016llX: Mask 0x%016llX , Data 0x%016llX",
P9A_MI_MCSYNC, l_scomMask, l_scomData);

FAPI_TRY(fapi2::putScomUnderMask(i_mc_target, P9A_MI_MCSYNC, l_scomData, l_scomMask),
"putScomUnderMask() returns an error (Sync), P9A_MI_MCSYNC reg 0x%016llX", P9A_MI_MCSYNC);

// Note: No need to read Sync replay count and retry in P9.

// --------------------------------------------------------------
// 4. Clear refresh sync bit
// --------------------------------------------------------------
l_scomData.flush<0>();
l_scomMask.flush<0>().setBit<EXPLR_SRQ_MBA_SYNCCNTLQ_SYNC_REF_EN>();

// Iterate through OCMBs to clear refresh sync bit
for (const auto& l_ocmb : mss::find_targets<fapi2::TARGET_TYPE_OCMB_CHIP>(i_mc_target))
{
FAPI_DBG("Writing EXPLR_SRQ_MBA_SYNCCNTLQ reg 0x%016llX: Mask 0x%016llX , Data 0x%016llX on %s",
EXPLR_SRQ_MBA_SYNCCNTLQ, l_scomMask, l_scomData, mss::c_str(l_ocmb));

FAPI_TRY(fapi2::putScomUnderMask(l_ocmb, EXPLR_SRQ_MBA_SYNCCNTLQ, l_scomData, l_scomMask),
"putScomUnderMask() returns an error (Sync), EXPLR_SRQ_MBA_SYNCCNTLQ reg 0x%016llX",
EXPLR_SRQ_MBA_SYNCCNTLQ);
}

fapi_try_exit:
FAPI_DBG("Exiting progMaster");
return fapi2::current_err;
}

///
/// @brief Perform throttle sync on the Memory Controllers
///
/// @param[in] i_mc_targets vector of reference of MC targets (MCS or MI)
/// @return FAPI2_RC_SUCCESS if success, else error code.
///
fapi2::ReturnCode throttle_sync(
const std::vector< fapi2::Target<fapi2::TARGET_TYPE_MI> >& i_mc_targets)
{
mc_side_info_t l_mcSide[MAX_MC_SIDES_PER_PROC];
uint8_t l_sideNum = 0;
uint8_t l_pos = 0;
uint8_t l_numMasterProgrammed = 0;

// Initialization
for (l_sideNum = 0; l_sideNum < MAX_MC_SIDES_PER_PROC; l_sideNum++)
{
l_mcSide[l_sideNum].master_mc_found = false;
}

// ---------------------------------------------------------------------
// 1. Pick the first MCS/MI with DIMMS as potential master
// for both MC sides (MC01/MC23)
// ---------------------------------------------------------------------
for (const auto& l_mc : i_mc_targets)
{
uint8_t l_num_dimms = mss::find_targets<fapi2::TARGET_TYPE_DIMM>(l_mc).size();

if (l_num_dimms > 0)
{
// This MCS or MI has DIMMs attached, find out which MC side it
// belongs to:
// l_sideNum = 0 --> MC01
// 1 --> MC23
FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, l_mc, l_pos),
"Error getting ATTR_CHIP_UNIT_POS on %s", mss::c_str(l_mc));
l_sideNum = l_pos / MAX_MC_SIDES_PER_PROC;

FAPI_INF("MCS %u has DIMMs", l_pos);

// If there's no master MCS or MI marked for this side yet, mark
// this MCS as master
if (l_mcSide[l_sideNum].master_mc_found == false)
{
FAPI_INF("Mark MCS %u as master for MC side %u",
l_pos, l_sideNum);
l_mcSide[l_sideNum].master_mc_found = true;
l_mcSide[l_sideNum].master_mc = l_mc;
}
}

prog_MCMODE0(l_mc, i_mc_targets);
}

// --------------------------------------------------------------
// 2. Program the master MI
// --------------------------------------------------------------
for (l_sideNum = 0; l_sideNum < MAX_MC_SIDES_PER_PROC; l_sideNum++)
{
// If there is a potential master MI found for this side
if (l_mcSide[l_sideNum].master_mc_found == true)
{
// No master MI programmed for either side yet,
// go ahead and program this MI as master.
if (l_numMasterProgrammed == 0)
{
FAPI_TRY(prog_master(l_mcSide[l_sideNum].master_mc),
"programMaster() returns error on %s", mss::c_str(l_mcSide[l_sideNum].master_mc));
l_numMasterProgrammed++;
}
}
}

fapi_try_exit:
FAPI_DBG("Exiting");
return fapi2::current_err;
}

extern "C"
{
///
/// @brief p9a_throttle_sync procedure
///
/// @param[in] i_target TARGET_TYPE_PROC_CHIP target
/// @return FAPI2_RC_SUCCESS if success, else error code.
///
fapi2::ReturnCode p9a_throttle_sync(
const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target)
{
FAPI_DBG("Executing p9a_throttle_sync on %s", mss::c_str(i_target));

const auto l_miChiplets = i_target.getChildren<fapi2::TARGET_TYPE_MI>();

if (l_miChiplets.size() > 0)
{
FAPI_TRY(throttle_sync(l_miChiplets), "Error calling throttle_sync() with vector of MI Chiplets");
}

fapi_try_exit:
FAPI_DBG("Exiting");
return fapi2::current_err;
}

} // extern "C"
54 changes: 54 additions & 0 deletions src/import/chips/p9/procedures/hwp/nest/p9a_throttle_sync.H
View file
Expand Up @@ -22,3 +22,57 @@
/* permissions and limitations under the License. */
/* */
/* IBM_PROLOG_END_TAG */
/// ----------------------------------------------------------------------------
/// @file p9a_throttle_sync.H
///
/// @brief p9a_throttle_sync HWP
///
/// The purpose of this procedure is to triggers sync command from a 'master'
/// MC to other MCs that have attached memory in a processor.
///
/// ----------------------------------------------------------------------------
/// *HWP HWP Owner : Joe McGill <jmcgill@us.ibm.com>
/// *HWP HWP Backup : Mark Pizzutillo <Mark.Pizzutillo@ibm.com>
/// *HWP FW Owner : Thi Tran <thi@us.ibm.com>
/// *HWP Team : Nest
/// *HWP Level : 3
/// *HWP Consumed by : HB
/// ----------------------------------------------------------------------------
#ifndef _P9A_THROTTLE_SYNC_H_
#define _P9A_THROTTLE_SYNC_H_

//------------------------------------------------------------------------------
// Includes
//------------------------------------------------------------------------------
#include <fapi2.H>
#include <p9a_mc_scom_addresses_fld.H>
#include <p9a_misc_scom_addresses_fld.H>
#include <p9a_misc_scom_addresses.H>
#include <explorer_scom_addresses.H>
#include <explorer_scom_addresses_fld.H>

// Function pointer typedef definition for HWP call support
typedef fapi2::ReturnCode (*p9a_throttle_sync_FP_t)(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>&);

///
/// @brief Structure that holds the potential master MI for a MC side (MC01/MC23)
///
struct mc_side_info_t
{
bool master_mc_found = false;
fapi2::Target<fapi2::TARGET_TYPE_MI> master_mc; // Master MC for this MC side
};

extern "C"
{
///
/// @brief p9a_throttle_sync procedure
///
/// @param[in] i_target TARGET_TYPE_PROC_CHIP target
/// @return FAPI2_RC_SUCCESS if success, else error code.
///
fapi2::ReturnCode p9a_throttle_sync(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target);

} // extern "C"

#endif // _P9A_THROTTLE_SYNC_H_
6 changes: 6 additions & 0 deletions src/import/chips/p9/procedures/hwp/nest/p9a_throttle_sync.mk
View file
Expand Up @@ -22,3 +22,9 @@
# permissions and limitations under the License.
#
# IBM_PROLOG_END_TAG

PROCEDURE=p9a_throttle_sync
$(call ADD_MODULE_INCDIR,$(PROCEDURE),$(ROOTPATH)/chips/p9a/procedures/hwp/memory/)

$(call ADD_MODULE_INCDIR,$(PROCEDURE),$(ROOTPATH))
$(call BUILD_PROCEDURE)

0 comments on commit 264bea8

Please sign in to comment.