Skip to content

Commit

Permalink
Add Mutex and Error Recovery for Node Communications
Browse files Browse the repository at this point in the history 
This commit adds the attributes needed for a mutex XBUS and ABUS lock
and then uses them in the Node Communications Device Driver.  It also
adds some additional error recovery to the Node Comm DD.  Plus, it adds
some additional SCOMs to the Read and Write Node Comm DD operations.

Change-Id: I27b94f29a6e3c2e3e2ba98fec48cc000c39add47
RTC:191008
Depends-on:I19510888c0922e5bb857cffc9426399e79e113ba
Depends-on:I11893af06b7a097b43106117d648e9a431c4f3ea
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/59292
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>
Reviewed-by: ILYA SMIRNOV <ismirno@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>
  • Loading branch information
@mabaiocchi @wghoffa
mabaiocchi authored and wghoffa committed May 31, 2018
1 parent 3ecd7cf commit 26d9aed
Show file tree
Hide file tree
Showing 9 changed files with 203 additions and 19 deletions.
1 change: 1 addition & 0 deletions src/build/tools/listdeps.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,7 @@
"libsecureboot_base.so" => '1',
"libscom.so" => '1',
"libxscom.so" => '1',
"libnode_comm.so" => '1',
};

# A list of the dependent libraries in each istep.
Expand Down
1 change: 0 additions & 1 deletion src/include/usr/isteps/istep09list.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,6 @@ const DepModInfo g_istep09Dependancies = {
DEP_LIB(libnestmemutils.so),
DEP_LIB(libfab_iovalid.so),
DEP_LIB(libimageprocs.so),
DEP_LIB(libnode_comm.so),
NULL
}
};
Expand Down
2 changes: 2 additions & 0 deletions src/include/usr/secureboot/secure_reasoncodes.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ namespace SECUREBOOT
MOD_NC_XBUS_TEST = 0x22,
MOD_NC_MAP_ATTN = 0x23,
MOD_NCDD_PERFORM_OP = 0x24,
MOD_NCDD_WRITE = 0x25,
};

enum SECUREReasonCode
Expand Down Expand Up @@ -87,6 +88,7 @@ namespace SECUREBOOT
RC_NC_NO_ATTN_FOUND = SECURE_COMP_ID | 0x23,
RC_NC_TOO_MANY_ATTNS_FOUND = SECURE_COMP_ID | 0x24,
RC_NCDD_INVALID_ARGS = SECURE_COMP_ID | 0x25,
RC_NCDD_DATA_NOT_SENT = SECURE_COMP_ID | 0x26,

// Reason codes 0xA0 - 0xEF reserved for trustedboot_reasoncodes.H
};
Expand Down
21 changes: 20 additions & 1 deletion src/usr/secureboot/node_comm/node_comm.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
extern trace_desc_t* g_trac_nc;

// Easy macro replace for unit testing - TRACD vs TRACF
#define TRACUCOMP(args...) TRACFCOMP(args)
#define TRACUCOMP(args...) TRACDCOMP(args)

namespace SECUREBOOT
{
Expand Down Expand Up @@ -69,6 +69,9 @@ enum node_comm_registers_t : uint64_t
// XBUS values are default values
// NCDD_ABUS_REG_OFFSET added if in ABUS mode
NCDD_REG_FIR = 0x5013400,
NCDD_REG_FIR_WOX_AND = 0x5013401,
NCDD_REG_FIR_WOX_OR = 0x5013402,

NCDD_REG_CTRL = 0x501342E,
NCDD_REG_DATA = 0x501342F,

Expand Down Expand Up @@ -133,6 +136,22 @@ enum node_comm_fir_reg_helpers_t : uint64_t
NCDD_START_OF_ATTN_BITS = 0x0000000008000000,
};


/**
* @brief Calculate Link Mailbox FIR Register Attention Bit Mask
* based on linkId and mboxId
*
* @param[in] i_linkId - Link Id of the operation
* @param[in] i_mboxId - Mailbox Id of the operation
*
* @return uint64_t - Calculated FIR Register Attention Bit Mask
*/
inline uint64_t getLinkMboxFirAttnBit(uint8_t i_linkId, uint8_t i_mboxId)
{
return (NCDD_START_OF_ATTN_BITS >> ((2*i_linkId) + i_mboxId));
}


/**
* @brief Map Attention Bits in XBUS/ABUS FIR Register to specific Link Mailbox
*
Expand Down
156 changes: 144 additions & 12 deletions src/usr/secureboot/node_comm/node_comm_dd.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ errlHndl_t nodeCommPerformOp( DeviceFW::OperationType i_opType,
va_list i_args )
{
errlHndl_t err = nullptr;
bool unlock_mutex = false;
node_comm_args_t node_comm_args;

uint64_t mode = va_arg( i_args, uint64_t );
Expand Down Expand Up @@ -160,7 +161,12 @@ errlHndl_t nodeCommPerformOp( DeviceFW::OperationType i_opType,
node_comm_args.mboxId, (*node_comm_args.data_ptr));

// Mutex Lock
// @TODO RTC:191008 Support mutex
(mode==NCDD_MODE_ABUS)
? mutex_lock(node_comm_args.tgt->
getHbMutexAttr<TARGETING::ATTR_HB_NODE_COMM_ABUS_MUTEX>())
: mutex_lock(node_comm_args.tgt->
getHbMutexAttr<TARGETING::ATTR_HB_NODE_COMM_XBUS_MUTEX>());
unlock_mutex = true;

/***********************************************/
/* Node Comm Read */
Expand Down Expand Up @@ -190,9 +196,6 @@ errlHndl_t nodeCommPerformOp( DeviceFW::OperationType i_opType,
}
while (0);

// Mutex Unlock
// @TODO RTC:191008 Support mutex

// If err, add trace and FFDC to log
if (err)
{
Expand All @@ -214,6 +217,17 @@ errlHndl_t nodeCommPerformOp( DeviceFW::OperationType i_opType,
}
}

// Mutex unlock
if (unlock_mutex == true)
{
(mode==NCDD_MODE_ABUS)
? mutex_unlock(node_comm_args.tgt->
getHbMutexAttr<TARGETING::ATTR_HB_NODE_COMM_ABUS_MUTEX>())
: mutex_unlock(node_comm_args.tgt->
getHbMutexAttr<TARGETING::ATTR_HB_NODE_COMM_XBUS_MUTEX>());
}


TRACFCOMP (g_trac_nc, EXIT_MRK"nodeCommPerformOp: %s: %s: "
"tgt=0x%X, LinkId=%d, MboxId=%d, data=0x%.16llX. "
TRACE_ERR_FMT,
Expand Down Expand Up @@ -255,6 +269,58 @@ errlHndl_t ncddRead(node_comm_args_t & i_args)
break;
}

// Since Read above was successful, clear the data register
uint64_t clear_data = 0x0;
err = ncddRegisterOp( DeviceFW::WRITE,
&clear_data,
link_mbox_reg,
i_args );

if(err)
{
TRACFCOMP(g_trac_nc,ERR_MRK"ncddRead: SCOM deviceWrite call "
"failed for Target HUID 0x%08X and address 0x%016llX. "
TRACE_ERR_FMT,
i_args.tgt_huid,
link_mbox_reg,
TRACE_ERR_ARGS(err));
break;
}


// Since Read above was successful, clear the FIR bit
uint64_t fir_attn_bit = getLinkMboxFirAttnBit(i_args.linkId, i_args.mboxId);

// Invert the fir bit and WOX_AND it into the register
uint64_t clear_fir_bit = ~fir_attn_bit;

uint64_t reg_addr = getLinkMboxRegAddr(NCDD_REG_FIR_WOX_AND, i_args.mode);

TRACUCOMP(g_trac_nc,"ncddRead: Clearing FIR bit 0x%.16llX based on "
"linkId=%d, mboxId=%d, mode=%s, by writing 0x%.16llX to FIR Reg "
"Addr 0x%.16llX on Target 0x%X",
fir_attn_bit, i_args.linkId, i_args.mboxId,
(i_args.mode == NCDD_MODE_ABUS)
? NCDD_ABUS_STRING : NCDD_XBUS_STRING,
clear_fir_bit, reg_addr, i_args.tgt_huid);

err = ncddRegisterOp( DeviceFW::WRITE,
&clear_fir_bit,
NCDD_REG_FIR_WOX_AND,
i_args );

if(err)
{
TRACFCOMP(g_trac_nc,ERR_MRK"ncddRead: SCOM deviceWrite call "
"failed for Target HUID 0x%08X and address 0x%016llX. "
TRACE_ERR_FMT,
i_args.tgt_huid,
link_mbox_reg,
TRACE_ERR_ARGS(err));
break;
}


} while( 0 );

TRACUCOMP( g_trac_nc,EXIT_MRK"ncddRead: "
Expand Down Expand Up @@ -299,7 +365,7 @@ errlHndl_t ncddWrite (node_comm_args_t & i_args)
ctrl_reg_cmd.link_id = i_args.linkId;


err = ncddRegisterOp( DeviceFW::WRITE,
err = ncddRegisterOp( DeviceFW::WRITE,
&ctrl_reg_cmd.value,
NCDD_REG_CTRL,
i_args );
Expand All @@ -316,7 +382,8 @@ errlHndl_t ncddWrite (node_comm_args_t & i_args)
}

// Wait for command to be complete
err = ncddWaitForCmdComp(i_args);
ctrl_reg_cmd.value = 0;
err = ncddWaitForCmdComp(i_args, ctrl_reg_cmd);
if(err)
{
TRACFCOMP(g_trac_nc,ERR_MRK"ncddWrite: Wait For Cmd Complete "
Expand All @@ -327,8 +394,43 @@ errlHndl_t ncddWrite (node_comm_args_t & i_args)
break;
}

// @TODO RTC 191008 - have ncddWaitForCmdComp return status to
// check for 'write' bit being set
// Check for 'sent' bit being set
if (ctrl_reg_cmd.sent != 1)
{
TRACFCOMP( g_trac_nc,
ERR_MRK"ncddWrite: 'Sent' bit not set in status reg: "
"0x%.16llX (Target 0x%X)",
ctrl_reg_cmd.value, i_args.tgt_huid );

/*@
* @errortype
* @reasoncode RC_NCDD_DATA_NOT_SENT
* @moduleid MOD_NCDD_WRITE
* @userdata1 Status Register Value
* @userdata2 Target HUID
* @devdesc Sent bit not set in Node Comm status/ctrl register
* @custdesc Secure Boot failure
*/
err = new ERRORLOG::ErrlEntry( ERRORLOG::ERRL_SEV_UNRECOVERABLE,
MOD_NCDD_WRITE,
RC_NCDD_DATA_NOT_SENT,
ctrl_reg_cmd.value,
i_args.tgt_huid);

// Likely an issue with Processor or its bus
err->addHwCallout( i_args.tgt,
HWAS::SRCI_PRIORITY_HIGH,
HWAS::DELAYED_DECONFIG,
HWAS::GARD_NULL );

// Or HB code failed to do the procedure correctly
err->addProcedureCallout(HWAS::EPUB_PRC_HB_CODE,
HWAS::SRCI_PRIORITY_MED);

// @TODO RTC 184518 - Look into bus callouts

break;
}

} while( 0 );

Expand All @@ -342,7 +444,7 @@ errlHndl_t ncddWrite (node_comm_args_t & i_args)


errlHndl_t ncddCheckStatus (node_comm_args_t & i_args,
const ctrl_reg_t i_statusVal )
const ctrl_reg_t & i_statusVal )
{
errlHndl_t err = nullptr;
bool errorFound = false;
Expand Down Expand Up @@ -436,14 +538,14 @@ errlHndl_t ncddCheckStatus (node_comm_args_t & i_args,
} // end ncddCheckStatus


errlHndl_t ncddWaitForCmdComp (node_comm_args_t & i_args)
errlHndl_t ncddWaitForCmdComp (node_comm_args_t & i_args,
ctrl_reg_t & o_statusVal )
{
errlHndl_t err = nullptr;
uint64_t interval_ns = NODE_COMM_POLL_DELAY_NS;
int timeout_ns = NODE_COMM_POLL_DELAY_TOTAL_NS;
ctrl_reg_t ctrl_reg_status;


TRACUCOMP(g_trac_nc, "ncddWaitForCmdComp(): timeout_ns=%d, "
"interval_ns=%d", timeout_ns, interval_ns);

Expand Down Expand Up @@ -537,6 +639,8 @@ errlHndl_t ncddWaitForCmdComp (node_comm_args_t & i_args)

} while (0);

o_statusVal = ctrl_reg_status;

TRACUCOMP( g_trac_nc,EXIT_MRK"ncddWaitForCmdComp: "
TRACE_ERR_FMT,
TRACE_ERR_ARGS(err));
Expand All @@ -553,9 +657,37 @@ void ncddHandleError( errlHndl_t & io_err,
TRACE_ERR_FMT,
TRACE_ERR_ARGS(io_err));

errlHndl_t l_err = nullptr;

do
{
// @TODO RTC:191008 Implement simple reset functionality
// On a fail issue a cmd to the control reg with just the 'reset' bit set
ctrl_reg_t ctrl_reg_cmd;
ctrl_reg_cmd.value = 0x0;
ctrl_reg_cmd.reset = 1;


l_err = ncddRegisterOp( DeviceFW::WRITE,
&ctrl_reg_cmd.value,
NCDD_REG_CTRL,
i_args );

if(l_err)
{
TRACFCOMP(g_trac_nc,ERR_MRK"ncddHandleError: SCOM deviceWrite call "
"failed for Target HUID 0x%08X and address 0x%016llX: "
TRACE_ERR_FMT
" . Committing after chaining to existing error: "
TRACE_ERR_FMT,
i_args.tgt_huid,
NCDD_REG_CTRL,
TRACE_ERR_ARGS(l_err),
TRACE_ERR_ARGS(io_err));

l_err->plid(io_err->plid());
errlCommit( l_err, SECURE_COMP_ID );
break;
}

} while (0);

Expand Down
11 changes: 7 additions & 4 deletions src/usr/secureboot/node_comm/node_comm_dd.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,13 +155,13 @@ errlHndl_t ncddWrite (node_comm_args_t & i_args);
* @param[in] i_args - Structure containing arguments needed for a command
* transaction.
*
* @param[in] i_statusVal - Control Register value to be checked
* @param[in] i_statusVal - Control/Status Register value to be checked
*
* @return errlHndl_t - nullptr if successful and the command is complete;
* otherwise a pointer to the error log.
*/
errlHndl_t ncddCheckForErrors (node_comm_args_t & i_args,
ctrl_reg_t i_statusVal );
errlHndl_t ncddCheckStatus (node_comm_args_t & i_args,
const ctrl_reg_t & i_statusVal );

/**
* @brief Waits for the operation to complete or timeout while
Expand All @@ -170,10 +170,13 @@ errlHndl_t ncddCheckForErrors (node_comm_args_t & i_args,
* @param[in] i_args - Structure containing arguments needed for a command
* transaction.
*
* @param[in] o_statusVal - Last Control/Status Register value that was checked
*
* @return errlHndl_t - nullptr if successful and the command is complete;
* otherwise a pointer to the error log.
*/
errlHndl_t ncddWaitForCmdComp (node_comm_args_t & i_args);
errlHndl_t ncddWaitForCmdComp (node_comm_args_t & i_args,
ctrl_reg_t & o_statusVal);

/**
* @brief Analyzes an error handle object and performs any
Expand Down
2 changes: 1 addition & 1 deletion src/usr/secureboot/node_comm/node_comm_test.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ errlHndl_t nodeCommXbus2ProcTest(void)


// 3) Read message on proc with Link Mailbox found above
TRACFCOMP(g_trac_nc,"nodeCommXbus2ProcTest: Attention Found on"
TRACFCOMP(g_trac_nc,"nodeCommXbus2ProcTest: Attention Found on "
"proc=0x%X for L%d/M%d ",
TARGETING::get_huid(read_tgt), linkId, mboxId);

Expand Down
26 changes: 26 additions & 0 deletions src/usr/targeting/common/xmltohb/attribute_types_hb.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1155,4 +1155,30 @@
<writeable />
</attribute>

<attribute>
<id>HB_NODE_COMM_XBUS_MUTEX</id>
<description>Mutex to guard Node Comm XBUS register access</description>
<simpleType>
<hbmutex>
</hbmutex>
</simpleType>
<persistency>volatile-zeroed</persistency>
<readable/>
<writeable/>
<hbOnly/>
</attribute>

<attribute>
<id>HB_NODE_COMM_ABUS_MUTEX</id>
<description>Mutex to guard Node Comm ABUS register access</description>
<simpleType>
<hbmutex>
</hbmutex>
</simpleType>
<persistency>volatile-zeroed</persistency>
<readable/>
<writeable/>
<hbOnly/>
</attribute>

</attributes>

0 comments on commit 26d9aed

Please sign in to comment.