Skip to content

Commit

Permalink
L2 - Fabric updates for multi-chip support
Browse files Browse the repository at this point in the history 
  Refactor p9_build_smp code
    Add FBC effective config (attribute-only) HWPs
    Add/call FBC initfiles

  HWP flow
    p9_fbc_eff_config
    p9_fbc_eff_config_links
    p9_chiplet_scominit
    p9_smp_link_layer
    p9_fab_iovalid
    p9_fbc_eff_config_aggregate
    p9_build_smp

  Update engd used in build to e9035 u087

Change-Id: Ia7678b8a2593c207e0b4ae9db17e5e2d7bc82b91
Original-Change-Id: I9ab9e967847d380596e896a14e481ad8cf247b9a
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/21643
Tested-by: PPE CI
Tested-by: Hostboot CI
Tested-by: Jenkins Server
Reviewed-by: Thi N. Tran <thi@us.ibm.com>
Reviewed-by: Benjamin Gass <bgass@us.ibm.com>
Reviewed-by: Jennifer A. Stofer <stofer@us.ibm.com>
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/59101
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
Tested-by: Daniel M. Crowell <dcrowell@us.ibm.com>
  • Loading branch information
@jjmcgill @dcrowell77
jjmcgill authored and dcrowell77 committed May 26, 2018
1 parent ac37437 commit 5310a33
Show file tree
Hide file tree
Showing 2 changed files with 371 additions and 61 deletions.
344 changes: 283 additions & 61 deletions src/import/chips/p9/procedures/hwp/nest/p9_fab_iovalid.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,25 +41,207 @@
// Includes
//------------------------------------------------------------------------------
#include <p9_fab_iovalid.H>
#include <p9_fbc_utils.H>
#include <p9_fbc_smp_utils.H>


//------------------------------------------------------------------------------
// Constant definitions
//------------------------------------------------------------------------------
// EXTFIR/RAS FIR field constants
const uint8_t IOVALID_FIELD_NUM_BITS = 2;

// DL FIR register field constants
const uint8_t DL_FIR_LINK0_TRAINED_BIT = 0;
const uint8_t DL_FIR_LINK1_TRAINED_BIT = 1;

// TL FIR register field constants
const uint8_t TL_FIR_TRAINED_FIELD_LENGTH = 2;
const uint8_t TL_FIR_TRAINED_LINK_TRAINED = 0x3;

// TL Link Delay register field constants
const uint8_t TL_LINK_DELAY_FIELD_NUM_BITS = 12;


//------------------------------------------------------------------------------
// Function definitions
//------------------------------------------------------------------------------


/// @brief Validate DL/TL link layers are trained
///
/// @param[in] i_target Processor chip target
///
/// @return fapi2::ReturnCode. FAPI2_RC_SUCCESS if success, else error code.
fapi2::ReturnCode p9_fab_iovalid_link_validate(
const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
const p9_fbc_link_ctl_t& i_link_ctl)
{
FAPI_DBG("Start");
fapi2::buffer<uint64_t> l_dl_fir_reg;
fapi2::buffer<uint64_t> l_tl_fir_reg;
uint8_t l_tl_fir_trained_state = 0;

// validate DL training state
FAPI_TRY(fapi2::getScom(i_target, i_link_ctl.dl_fir_addr, l_dl_fir_reg),
"Error from getScom (0x%.16llX)", i_link_ctl.dl_fir_addr);
FAPI_ASSERT(l_dl_fir_reg.getBit<DL_FIR_LINK0_TRAINED_BIT>() &&
l_dl_fir_reg.getBit<DL_FIR_LINK1_TRAINED_BIT>(),
fapi2::P9_FAB_IOVALID_DL_NOT_TRAINED_ERR()
.set_TARGET(i_target)
.set_LOC_ENDP_TYPE(i_link_ctl.endp_type)
.set_LOC_ENDP_UNIT_ID(i_link_ctl.endp_unit_id)
.set_DL_FIR_REG(l_dl_fir_reg),
"Link DL training did not complete successfully!");

// validate TL training state
FAPI_TRY(fapi2::getScom(i_target, i_link_ctl.tl_fir_addr, l_tl_fir_reg),
"Error from getScom (0x%.16llX)", i_link_ctl.tl_fir_addr);
FAPI_TRY(l_tl_fir_reg.extractToRight(l_tl_fir_trained_state,
i_link_ctl.tl_fir_trained_field_start_bit,
TL_FIR_TRAINED_FIELD_LENGTH),
"Error extracting TL layer training state");

FAPI_ASSERT(l_tl_fir_trained_state == TL_FIR_TRAINED_LINK_TRAINED,
fapi2::P9_FAB_IOVALID_TL_NOT_TRAINED_ERR()
.set_TARGET(i_target)
.set_LOC_ENDP_TYPE(i_link_ctl.endp_type)
.set_LOC_ENDP_UNIT_ID(i_link_ctl.endp_unit_id)
.set_TL_FIR_REG(l_tl_fir_reg),
"Link TL training did not complete successfully!");

fapi_try_exit:
FAPI_DBG("End");
return fapi2::current_err;
}


/// @brief Compute single end link delay of individual link
///
/// @param[in] i_target Processor chip target
/// @param[in] i_link_ctl X/A link control structure for link
/// @param[out] o_link_delay Link delay
///
/// @return fapi2::ReturnCode. FAPI2_RC_SUCCESS if success, else error code.
fapi2::ReturnCode p9_fab_iovalid_get_link_delay(
const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
const p9_fbc_link_ctl_t& i_link_ctl,
uint32_t o_link_delay)
{
FAPI_DBG("Start");
fapi2::buffer<uint64_t> l_link_delay_reg;
uint32_t l_sublink_delay[2];

// read link delay register, extract hi/lo delay values & return their average
FAPI_TRY(fapi2::getScom(i_target, i_link_ctl.tl_link_delay_addr, l_link_delay_reg),
"Error from getScom (0x%.16llX)", i_link_ctl.tl_link_delay_addr);
FAPI_TRY(l_link_delay_reg.extractToRight(l_sublink_delay[0],
i_link_ctl.tl_link_delay_hi_start_bit,
TL_LINK_DELAY_FIELD_NUM_BITS),
"Error extracting link delay (hi>");
FAPI_TRY(l_link_delay_reg.extractToRight(l_sublink_delay[1],
i_link_ctl.tl_link_delay_lo_start_bit,
TL_LINK_DELAY_FIELD_NUM_BITS),
"Error extracting link delay (lo)");
o_link_delay = (l_sublink_delay[0] + l_sublink_delay[1]) / 2;

fapi_try_exit:
FAPI_DBG("End");
return fapi2::current_err;
}


/// @brief Get round trip training delays reported by both endpoints
/// of a given link
///
/// @brief Manipulate iovalid/FIR settings for a single fabric link
/// @tparam T template parameter, defines endpoint type
/// @param[in] i_loc_target Source side chip target
/// @param[in] i_loc_link_ctl X/A link control structure for link local end
/// @param[in] i_rem_link_ctl X/A link control structure for link remote end
/// @param[out] o_agg_link_delay Sum of local and remote end link delays
///
/// @return fapi2::ReturnCode. FAPI2_RC_SUCCESS if success, else error code.
template<fapi2::TargetType T>
fapi2::ReturnCode p9_fab_iovalid_get_link_delays(
const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_loc_chip_target,
const p9_fbc_link_ctl_t& i_loc_link_ctl,
const p9_fbc_link_ctl_t& i_rem_link_ctl,
uint32_t& o_agg_link_delay)
{
FAPI_DBG("Start");

bool l_found = false;
uint32_t l_loc_link_delay = 0xFFF;
uint32_t l_rem_link_delay = 0xFFF;
fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP> l_rem_chip_target;

// use local end link control structure to find associated endpoint target
auto l_endp_targets = i_loc_chip_target.getChildren<T>();

for (auto l_iter = l_endp_targets.begin();
(l_iter != l_endp_targets.end()) && !l_found;
l_iter++)
{
uint8_t l_loc_endp_unit_id;
FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_CHIP_UNIT_POS, *l_iter, l_loc_endp_unit_id),
"Error from FAPI_ATTR_GET (ATTR_CHIP_UNIT_POS, local)");

if ((static_cast<fapi2::TargetType>(i_loc_link_ctl.endp_type) == T) &&
(i_loc_link_ctl.endp_unit_id == l_loc_endp_unit_id))
{
// associated endpoint target found, use getOtherEnd/getParent to reach chip
// target of connected chip
fapi2::Target<T> l_rem_endp_target;
fapi2::ReturnCode l_rc = l_iter->getOtherEnd(l_rem_endp_target);
FAPI_ASSERT(!l_rc,
fapi2::P9_FAB_IOVALID_REM_ENDP_TARGET_ERR()
.set_LOC_TARGET(i_loc_chip_target)
.set_LOC_ENDP_TYPE(i_loc_link_ctl.endp_type)
.set_LOC_ENDP_UNIT_ID(i_loc_link_ctl.endp_unit_id)
.set_REM_ENDP_UNIT_ID(i_rem_link_ctl.endp_unit_id),
"Endpoint target at other end of link is invalid!");

l_rem_chip_target = l_rem_endp_target.template getParent<fapi2::TARGET_TYPE_PROC_CHIP>();
l_found = true;
}
}

FAPI_ASSERT(l_found,
fapi2::P9_FAB_IOVALID_LOC_ENDP_TARGET_ERR()
.set_LOC_TARGET(i_loc_chip_target)
.set_LOC_ENDP_TYPE(i_loc_link_ctl.endp_type)
.set_LOC_ENDP_UNIT_ID(i_loc_link_ctl.endp_unit_id),
"No matching local endpoint target found!");

// read link delay from local/remote chip targets
// link control structures provide register/bit offsets to collect
FAPI_TRY(p9_fab_iovalid_get_link_delay(
i_loc_chip_target,
i_loc_link_ctl,
l_loc_link_delay),
"Error from p9_fab_iovalid_get_link_delay (local)");

FAPI_TRY(p9_fab_iovalid_get_link_delay(
l_rem_chip_target,
i_rem_link_ctl,
l_rem_link_delay),
"Error from p9_fab_iovalid_get_link_delay (remote)");

o_agg_link_delay = l_loc_link_delay + l_rem_link_delay;

fapi_try_exit:
FAPI_DBG("End");
return fapi2::current_err;
}



///
/// @brief Manipulate iovalid/FIR settings for a single fabric link (X/A)
///
/// @param[in] i_target Reference to processor chip target
/// @param[in] i_ctl Reference to link control structure
/// @param[in] i_set_not_clear Define iovalid operation (true=set, false=clear)
///
/// @param[in] i_target Reference to processor chip target
/// @param[in] i_ctl Reference to link control structure
/// @param[op] i_set_not_clear Define iovalid operation (true=set, false=clear)
/// @return fapi::ReturnCode. FAPI2_RC_SUCCESS if success, else error code.
///
fapi2::ReturnCode
Expand Down Expand Up @@ -112,90 +294,130 @@ fapi_try_exit:
}



// NOTE: see doxygen comments in header
fapi2::ReturnCode
p9_fab_iovalid(const fapi2::Target<fapi2::TARGET_TYPE_PROC_CHIP>& i_target,
const bool i_set_not_clear)
const bool i_set_not_clear,
const bool i_manage_electrical,
const bool i_manage_optical)
{
FAPI_INF("Start");

uint8_t l_x_en_attr[P9_FBC_UTILS_MAX_X_LINKS];
uint8_t l_a_en_attr[P9_FBC_UTILS_MAX_A_LINKS];
std::vector<std::pair<p9_fbc_link_t, uint8_t>> l_valid_links;
std::vector<p9_fbc_link_ctl_t> l_link_ctls(P9_FBC_LINK_CTL_ARR,
P9_FBC_LINK_CTL_ARR + (sizeof(P9_FBC_LINK_CTL_ARR) / sizeof(P9_FBC_LINK_CTL_ARR[0])));
bool l_ctl_match_found = false;

// logical link (X/A) configuration parameters
// arrays indexed by link ID on local end
// enable on local end
uint8_t l_x_en[P9_FBC_UTILS_MAX_X_LINKS];
uint8_t l_a_en[P9_FBC_UTILS_MAX_A_LINKS];
// link ID on remote end
uint8_t l_x_rem_link_id[P9_FBC_UTILS_MAX_X_LINKS];
uint8_t l_a_rem_link_id[P9_FBC_UTILS_MAX_A_LINKS];
// aggregate (local+remote) delays
uint32_t l_x_agg_link_delay[P9_FBC_UTILS_MAX_X_LINKS];
uint32_t l_a_agg_link_delay[P9_FBC_UTILS_MAX_A_LINKS];

// read X/A link enable attributes, extract set of valid links
FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG,
i_target,
l_x_en_attr),
// seed arrays with attribute values
FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG, i_target, l_x_en),
"Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_X_ATTACHED_CHIP_CNFG");

for (uint8_t x = 0; x < P9_FBC_UTILS_MAX_X_LINKS; x++)
{
if (l_x_en_attr[x])
{
FAPI_DBG("Adding link X%d", x);
l_valid_links.push_back(std::make_pair(XBUS, x));
}
else
{
FAPI_DBG("Skipping link X%d", x);
}
}
FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_X_ATTACHED_LINK_ID, i_target, l_x_rem_link_id),
"Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_X_ATTACHED_LINK_ID");

FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_X_LINK_DELAY, i_target, l_x_agg_link_delay),
"Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_X_LINK_DELAY");

FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_A_ATTACHED_CHIP_CNFG,
i_target,
l_a_en_attr),
FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_A_ATTACHED_CHIP_CNFG, i_target, l_a_en),
"Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_A_ATTACHED_CHIP_CNFG");

for (uint8_t a = 0; a < P9_FBC_UTILS_MAX_A_LINKS; a++)
FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_A_ATTACHED_LINK_ID, i_target, l_a_rem_link_id),
"Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_A_ATTACHED_LINK_ID");

FAPI_TRY(FAPI_ATTR_GET(fapi2::ATTR_PROC_FABRIC_A_LINK_DELAY, i_target, l_a_agg_link_delay),
"Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_A_LINK_DELAY");

for (uint8_t l_link_id = 0; l_link_id < P9_FBC_UTILS_MAX_X_LINKS; l_link_id++)
{
if (l_a_en_attr[a])
if (l_x_en[l_link_id])
{
FAPI_DBG("Adding link A%d", a);
l_valid_links.push_back(std::make_pair(ABUS, a));
if ((i_manage_electrical &&
(P9_FBC_XBUS_LINK_CTL_ARR[l_link_id].endp_type == ELECTRICAL)) ||
(i_manage_optical &&
(P9_FBC_XBUS_LINK_CTL_ARR[l_link_id].endp_type == OPTICAL)))
{
FAPI_DBG("Updating link X%d", l_link_id);
FAPI_TRY(p9_fab_iovalid_update_link(i_target,
P9_FBC_XBUS_LINK_CTL_ARR[l_link_id],
i_set_not_clear),
"Error from p9_fab_iovalid_update_link (X)");

if (i_set_not_clear)
{
FAPI_DBG("Collecting link delay counter values");

if (P9_FBC_XBUS_LINK_CTL_ARR[l_link_id].endp_type == ELECTRICAL)
{
FAPI_TRY(p9_fab_iovalid_get_link_delays<fapi2::TARGET_TYPE_XBUS>(
i_target,
P9_FBC_XBUS_LINK_CTL_ARR[l_link_id],
P9_FBC_XBUS_LINK_CTL_ARR[l_x_rem_link_id[l_link_id]],
l_x_agg_link_delay[l_link_id]),
"Error from p9_fab_iovalid_get_link_delays (X, electrical)");
}
else
{
FAPI_TRY(p9_fab_iovalid_get_link_delays<fapi2::TARGET_TYPE_OBUS>(
i_target,
P9_FBC_XBUS_LINK_CTL_ARR[l_link_id],
P9_FBC_XBUS_LINK_CTL_ARR[l_x_rem_link_id[l_link_id]],
l_x_agg_link_delay[l_link_id]),
"Error from p9_fab_iovalid_get_link_delays (X, optical)");
}
}
}
}
else
{
FAPI_DBG("Skipping link A%d", a);
FAPI_DBG("Skipping link X%d", l_link_id);
}
}

// for each valid link, search vector table & call link update routine
for (auto l_link_iter = l_valid_links.begin(); l_link_iter != l_valid_links.end(); l_link_iter++)
for (uint8_t l_link_id = 0; l_link_id < P9_FBC_UTILS_MAX_A_LINKS; l_link_id++)
{
FAPI_DBG("Processing %s%d (action = %s)",
(l_link_iter->first == XBUS) ? ("X") : ("A)"),
l_link_iter->second,
(i_set_not_clear) ? ("set") : ("clear"));

l_ctl_match_found = false;

for (auto l_link_ctl_iter = l_link_ctls.begin();
(l_link_ctl_iter != l_link_ctls.end()) && (!l_ctl_match_found);
l_link_ctl_iter++)
if (l_a_en[l_link_id])
{
if ((l_link_ctl_iter->link_type == l_link_iter->first) &&
(l_link_ctl_iter->link_id == l_link_iter->second))
if (i_manage_optical &&
(P9_FBC_ABUS_LINK_CTL_ARR[l_link_id].endp_type == OPTICAL))
{
l_ctl_match_found = true;
FAPI_DBG("Updating link A%d", l_link_id);
FAPI_TRY(p9_fab_iovalid_update_link(i_target,
*l_link_ctl_iter,
P9_FBC_ABUS_LINK_CTL_ARR[l_link_id],
i_set_not_clear),
"Error from p9_fab_iovalid_update_link");
"Error from p9_fab_iovalid_update_link (A)");

if (i_set_not_clear)
{
FAPI_DBG("Collecting link delay counter values");
FAPI_TRY(p9_fab_iovalid_get_link_delays<fapi2::TARGET_TYPE_OBUS>(
i_target,
P9_FBC_ABUS_LINK_CTL_ARR[l_link_id],
P9_FBC_ABUS_LINK_CTL_ARR[l_a_rem_link_id[l_link_id]],
l_a_agg_link_delay[l_link_id]),
"Error from p9_fab_iovalid_get_link_delays (A)");
}
}
}

FAPI_ASSERT(l_ctl_match_found,
fapi2::P9_FAB_IOVALID_TABLE_ERR().set_TARGET(i_target).
set_LINK(l_link_iter->first).
set_LINK_ID(l_link_iter->second),
"No match found for link");
else
{
FAPI_DBG("Skipping link A%d", l_link_id);
}
}

// update link delay attributes
FAPI_TRY(FAPI_ATTR_SET(fapi2::ATTR_PROC_FABRIC_X_LINK_DELAY, i_target, l_x_agg_link_delay),
"Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_X_LINK_DELAY");

FAPI_TRY(FAPI_ATTR_SET(fapi2::ATTR_PROC_FABRIC_A_LINK_DELAY, i_target, l_a_agg_link_delay),
"Error from FAPI_ATTR_GET (ATTR_PROC_FABRIC_A_LINK_DELAY");

fapi_try_exit:
FAPI_INF("End");
return fapi2::current_err;
Expand Down

0 comments on commit 5310a33

Please sign in to comment.