Skip to content

Commit

Permalink
Add Node Communications Transfer Protocol
Browse files Browse the repository at this point in the history 
This commit adds a more flexible Node-to-Node Transfer Protocol
that will be used for the secureboot/trustedboot algorithm in
multi-node configurations.  The original message transfer has
been updated to use this new transfer protocol.

Change-Id: I12704e8d71d8c0aac52c286160322f9b845a5026
RTC:203641
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/71317
Reviewed-by: Ilya Smirnov <ismirno@us.ibm.com>
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
  • Loading branch information
@mabaiocchi @dcrowell77
mabaiocchi authored and dcrowell77 committed Feb 13, 2019
1 parent 2206846 commit c47875e
Show file tree
Hide file tree
Showing 8 changed files with 738 additions and 213 deletions.
1 change: 1 addition & 0 deletions src/usr/secureboot/node_comm/makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ OBJS += node_comm_dd.o
OBJS += node_comm_test.o
OBJS += node_comm.o
OBJS += node_comm_exchange.o
OBJS += node_comm_transfer.o

VPATH += $(ROOTPATH)/src/usr/targeting/common
EXTRAINCDIR += $(ROOTPATH)/src/include/usr
Expand Down
111 changes: 86 additions & 25 deletions src/usr/secureboot/node_comm/node_comm.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,17 +72,19 @@ namespace NODECOMM
* ABUS from a processor on another node.
*/
errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
uint64_t & o_data,
uint8_t & o_linkId,
uint8_t & o_mboxId)
const uint8_t i_linkId,
const uint8_t i_mboxId,
uint64_t & o_data)
{
errlHndl_t err = nullptr;
bool attn_found = false;
uint8_t actual_linkId = 0;
uint8_t actual_mboxId = 0;

const uint64_t interval_ns = NODE_COMM_POLL_DELAY_NS;
uint64_t time_polled_ns = 0;

TRACFCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusRecvMessage: pProc=0x%.08X",
TRACUCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusRecvMessage: pProc=0x%.08X",
get_huid(i_pProc));

do
Expand All @@ -94,8 +96,8 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
err = nodeCommMapAttn(i_pProc,
NCDD_MODE_ABUS,
attn_found,
o_linkId,
o_mboxId);
actual_linkId,
actual_mboxId);
if (err)
{
TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusRecvMessage: Error Back "
Expand All @@ -107,10 +109,10 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
}
if (attn_found == true)
{
TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommAbusRecvMessage: "
TRACUCOMP(g_trac_nc,INFO_MRK"nodeCommAbusRecvMessage: "
"nodeCommMapAttn attn_found (%d) for Tgt=0x%.08X, link=%d, "
"mbox=%d",
attn_found, get_huid(i_pProc), o_linkId, o_mboxId);
attn_found, get_huid(i_pProc), actual_linkId, actual_mboxId);
break;
}

Expand All @@ -131,7 +133,7 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
* @userdata2[32:63] Time Interval Between Polls in ns
* @devdesc Timed out waiting to receive message over
* ABUS Link Mailbox
* @custdesc Secure Boot failure
* @custdesc Trusted Boot failure
*/
err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
MOD_NC_RECV,
Expand All @@ -143,8 +145,11 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
NODE_COMM_POLL_DELAY_TOTAL_NS,
interval_ns));

// Bus Callout will be handled by caller since
// it should know expected peer target
// Since we know what bus we expected the message on, call it out
addNodeCommBusCallout(NCDD_MODE_ABUS,
i_pProc,
i_linkId,
err);

// Or HB code failed to do the procedure correctly
err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
Expand Down Expand Up @@ -172,6 +177,59 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,

if (attn_found == true)
{
// Verify that actual receive link/mboxIds were the same as the
// expected ones
if ((actual_linkId != i_linkId) ||
(actual_mboxId != i_mboxId))
{
TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommAbusRecvMessage: "
"Expected Link (%d) Mbox (%d) IDs DO NOT Match the "
"Actual Link (%d) Mbox (%d) IDs the message was "
"received on",
i_linkId, i_mboxId,
actual_linkId, actual_mboxId);

/*@
* @errortype
* @reasoncode RC_NCEX_MISMATCH_RECV_LINKS
* @moduleid MOD_NC_RECV
* @userdata1 Master Proc Target HUID
* @userdata2[0:15] Expected Link Id to receive message on
* @userdata2[16:31] Expected Mailbox Id to receive message on
* @userdata2[32:47] Actual Link Id message was received on
* @userdata2[48:63] Actual Mailbox Id message was receiveed on
* @devdesc Mismatch between expected and actual Link Mbox
* Ids a secure ABUS message was received on
* @custdesc Trusted Boot failure
*/
err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
MOD_NC_RECV,
RC_NCEX_MISMATCH_RECV_LINKS,
get_huid(i_pProc),
FOUR_UINT16_TO_UINT64(
i_linkId,
i_mboxId,
actual_linkId,
actual_mboxId));

// Since we know what bus we expected the message on, call it out
addNodeCommBusCallout(NCDD_MODE_ABUS,
i_pProc,
i_linkId,
err);

// Or HB code failed to do the procedure correctly
err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
HWAS::SRCI_PRIORITY_LOW);

// Grab FFDC from the target
getNodeCommFFDC(NCDD_MODE_ABUS,
i_pProc,
err);

break;
}

// Read message on proc with Link Mailbox found above
o_data = 0;
size_t expSize = sizeof(o_data);
Expand All @@ -180,15 +238,15 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
&o_data,
reqSize,
DEVICE_NODECOMM_ADDRESS(NCDD_MODE_ABUS,
o_linkId,
o_mboxId));
actual_linkId,
actual_mboxId));

if (err)
{
TRACFCOMP(g_trac_nc,ERR_MRK"nodeCommRecvMessage: Error Back From "
"Abus MBox Read: Tgt=0x%.08X, link=%d, mbox=%d: "
TRACE_ERR_FMT,
get_huid(i_pProc), o_linkId, o_mboxId,
get_huid(i_pProc), actual_linkId, actual_mboxId,
TRACE_ERR_ARGS(err));
break;
}
Expand All @@ -203,7 +261,8 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
"Tgt=0x%.08X, link=%d, mbox=%d attn_found=%d: "
"data=0x%.16llX. "
TRACE_ERR_FMT,
get_huid(i_pProc), o_linkId, o_mboxId, attn_found, o_data,
get_huid(i_pProc), actual_linkId, actual_mboxId,
attn_found, o_data,
TRACE_ERR_ARGS(err));

return err;
Expand All @@ -216,13 +275,13 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
* the current node to a processor on another node.
*/
errlHndl_t nodeCommAbusSendMessage(TARGETING::Target* i_pProc,
const uint64_t & i_data,
const uint8_t & i_linkId,
const uint8_t & i_mboxId)
const uint64_t i_data,
const uint8_t i_linkId,
const uint8_t i_mboxId)
{
errlHndl_t err = nullptr;

TRACFCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusSendMessage: iProc=0x%.08X "
TRACUCOMP(g_trac_nc,ENTER_MRK"nodeCommAbusSendMessage: iProc=0x%.08X "
"to send data=0x%.16llX through linkId=%d mboxId=%d",
get_huid(i_pProc), i_data, i_linkId, i_mboxId);

Expand Down Expand Up @@ -253,8 +312,10 @@ errlHndl_t nodeCommAbusSendMessage(TARGETING::Target* i_pProc,

} while( 0 );

TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusSendMessage: "
TRACFCOMP(g_trac_nc,EXIT_MRK"nodeCommAbusSendMessage: iProc=0x%.08X "
"send data=0x%.16llX through linkId=%d mboxId=%d: "
TRACE_ERR_FMT,
get_huid(i_pProc), i_data, i_linkId, i_mboxId,
TRACE_ERR_ARGS(err));

return err;
Expand Down Expand Up @@ -352,7 +413,7 @@ errlHndl_t nodeCommMapAttn(TARGETING::Target* i_pProc,
* @userdata2[32:63] Target HUID FIR was read from
* @devdesc Too many attentions were found in
* the Node Comm FIR Register
* @custdesc Secure Boot failure
* @custdesc Trusted Boot failure
*/
err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
MOD_NC_MAP_ATTN,
Expand Down Expand Up @@ -392,7 +453,7 @@ errlHndl_t nodeCommMapAttn(TARGETING::Target* i_pProc,
o_linkId = (bit / 2);
o_mboxId = (bit % 2);

TRACFCOMP(g_trac_nc,INFO_MRK"nodeCommMapAttn: tgt=0x%X: "
TRACUCOMP(g_trac_nc,INFO_MRK"nodeCommMapAttn: tgt=0x%X: "
"o_attn_found=%d, o_linkId=%d, mboxId=%d, "
TRACE_ERR_FMT,
get_huid(i_pProc), o_attn_found, o_linkId, o_mboxId,
Expand Down Expand Up @@ -492,7 +553,7 @@ errlHndl_t getObusTrainedLinks(TARGETING::Target* i_pObus,
/**
* @brief Add FFDC for the target to an error log
*/
void getNodeCommFFDC( node_comm_modes_t i_mode,
void getNodeCommFFDC( const node_comm_modes_t i_mode,
TARGETING::Target* i_pProc,
errlHndl_t &io_log)
{
Expand Down Expand Up @@ -556,9 +617,9 @@ void getNodeCommFFDC( node_comm_modes_t i_mode,
/**
* @brief Add a bus callout to an error log
*/
void addNodeCommBusCallout(node_comm_modes_t i_mode,
void addNodeCommBusCallout(const node_comm_modes_t i_mode,
TARGETING::Target* i_pProc,
uint8_t & i_linkId,
const uint8_t i_linkId,
errlHndl_t & io_log,
HWAS::callOutPriority i_priority)
{
Expand Down
37 changes: 21 additions & 16 deletions src/usr/secureboot/node_comm/node_comm.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ enum node_comm_registers_t : uint64_t
*
* @return uint64_t - Calculated Link Mailbox Register Address
*/
inline uint64_t getLinkMboxRegAddr(uint64_t i_reg,
node_comm_modes_t i_mode)
inline uint64_t getLinkMboxRegAddr(const uint64_t i_reg,
const node_comm_modes_t i_mode)
{
return (i_mode == NCDD_MODE_ABUS)
? (i_reg + NCDD_ABUS_REG_OFFSET)
Expand All @@ -125,7 +125,8 @@ inline uint64_t getLinkMboxRegAddr(uint64_t i_reg,
* @return uint64_t - Calculated Link Mailbox Register Base Address
* @note Returned value is Base/XBUS Address - see node_comm_registers_t
*/
inline uint64_t getLinkMboxReg(uint8_t i_linkId, uint8_t i_mboxId)
inline uint64_t getLinkMboxReg(const uint8_t i_linkId,
const uint8_t i_mboxId)
{
return NCDD_REG_LINK_MBOX_00 + (2*i_linkId) + i_mboxId;
}
Expand All @@ -140,8 +141,8 @@ inline uint64_t getLinkMboxReg(uint8_t i_linkId, uint8_t i_mboxId)
*
* @return void
*/
inline void getLinkMboxFromObusInstance(uint8_t i_obusInstance,
uint8_t i_obusRelLink,
inline void getLinkMboxFromObusInstance(const uint8_t i_obusInstance,
const uint8_t i_obusRelLink,
uint8_t & o_linkId,
uint8_t & o_mboxId)
{
Expand Down Expand Up @@ -172,7 +173,8 @@ enum node_comm_fir_reg_helpers_t : uint64_t
*
* @return uint64_t - Calculated FIR Register Attention Bit Mask
*/
inline uint64_t getLinkMboxFirAttnBit(uint8_t i_linkId, uint8_t i_mboxId)
inline uint64_t getLinkMboxFirAttnBit(const uint8_t i_linkId,
const uint8_t i_mboxId)
{
return (NCDD_START_OF_ATTN_BITS >> ((2*i_linkId) + i_mboxId));
}
Expand All @@ -184,24 +186,27 @@ inline uint64_t getLinkMboxFirAttnBit(uint8_t i_linkId, uint8_t i_mboxId)
*
* @param[in] i_pProc - Processor target to look for attentions on
* Can't be nullptr
* @param[in] i_linkId - Expected Link Id that received the message
* @param[in] i_mboxId - Expected Mailbox Id that received the message
* @param[out] o_data - Data received
* @param[out] o_linkId - Link Id that received the message
* @param[out] o_mboxId - Mailbox Id that received the message
*
* @note If message received on different LinkId/MailboxId than i_linkId and
* i_mboxId then an error will be created
*
* @return errlHndl_t Error log handle
* @retval nullptr Operation was successful
* @retval !nullptr Operation failed with valid error log
*/
errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
uint64_t & o_data,
uint8_t & o_linkId,
uint8_t & o_mboxId);
uint8_t i_linkId,
uint8_t i_mboxId,
uint64_t & o_data);

/**
* @brief This function sends a message over the ABUS from the processor of
* the current node to a processor on another node.
*
* @param[in] i_pProc - Processor target to look for attentions on
* @param[in] i_pProc - Processor target sending message
* Can't be nullptr
* @param[in] i_data - Data to be sent
* @param[in] i_linkId - Link Id Message is sent from
Expand All @@ -212,9 +217,9 @@ errlHndl_t nodeCommAbusRecvMessage(TARGETING::Target* i_pProc,
* @retval !nullptr Operation failed with valid error log
*/
errlHndl_t nodeCommAbusSendMessage(TARGETING::Target* i_pProc,
const uint64_t & i_data,
const uint8_t & i_linkId,
const uint8_t & i_mboxId);
uint64_t i_data,
uint8_t i_linkId,
uint8_t i_mboxId);


/**
Expand Down Expand Up @@ -286,7 +291,7 @@ void getNodeCommFFDC(node_comm_modes_t i_mode,
void addNodeCommBusCallout(
node_comm_modes_t i_mode,
TARGETING::Target* i_pProc,
uint8_t & i_linkId,
uint8_t i_linkId,
errlHndl_t & io_log,
HWAS::callOutPriority i_priority = HWAS::SRCI_PRIORITY_HIGH);

Expand Down
9 changes: 5 additions & 4 deletions src/usr/secureboot/node_comm/node_comm_dd.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ errlHndl_t nodeCommPerformOp( DeviceFW::OperationType i_opType,
{

// Check other input parameters
// @TODO RTC 203642 add check that i_opType is only READ or WRITE
const auto max_linkId = (mode==NCDD_MODE_ABUS)
? NCDD_MAX_ABUS_LINK_ID
: NCDD_MAX_XBUS_LINK_ID;
Expand Down Expand Up @@ -136,7 +137,7 @@ errlHndl_t nodeCommPerformOp( DeviceFW::OperationType i_opType,
* @userdata1[48:63] MailboxId Value
* @userdata2 Input Data Pointer
* @devdesc Invalid Input Args for Node Comm DD call
* @custdesc Secure Boot failure
* @custdesc Trusted Boot failure
*/
err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
MOD_NCDD_PERFORM_OP,
Expand Down Expand Up @@ -409,7 +410,7 @@ errlHndl_t ncddWrite (node_comm_args_t & i_args)
* @userdata1 Status Register Value
* @userdata2 Target HUID
* @devdesc Sent bit not set in Node Comm status/ctrl register
* @custdesc Secure Boot failure
* @custdesc Trusted Boot failure
*/
err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
MOD_NCDD_WRITE,
Expand Down Expand Up @@ -501,7 +502,7 @@ errlHndl_t ncddCheckStatus (node_comm_args_t & i_args,
* @userdata1 Status Register Value
* @userdata2 Target HUID
* @devdesc Error found in Node Comm status/ctrl register
* @custdesc Secure Boot failure
* @custdesc Trusted Boot failure
*/
err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
MOD_NCDD_CHECK_FOR_ERRORS,
Expand Down Expand Up @@ -597,7 +598,7 @@ errlHndl_t ncddWaitForCmdComp (node_comm_args_t & i_args,
* @userdata2[0:31] Status/Control Register Address
* @userdata2[32:63] Target HUID
* @devdesc Timed out waiting for command complete.
* @custdesc Secure Boot failure
* @custdesc Trusted Boot failure
*/
err = new ERRORLOG::ErrlEntry(ERRORLOG::ERRL_SEV_UNRECOVERABLE,
MOD_NCDD_WAIT_FOR_CMD_COMP,
Expand Down

0 comments on commit c47875e

Please sign in to comment.