Skip to content

Commit

Permalink
Send "recollect dma buffers" message in TI path
Browse files Browse the repository at this point in the history 
Change-Id: Ice352f333cc3cfca7393b335414958612834b3c8
RTC:170428
CQ:SW359697
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/44706
Reviewed-by: Martin Gloff <mgloff@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
  • Loading branch information
@brianebakke @dcrowell77
brianebakke authored and dcrowell77 committed Sep 19, 2017
1 parent 88342b0 commit 046a265
Show file tree
Hide file tree
Showing 6 changed files with 282 additions and 45 deletions.
18 changes: 12 additions & 6 deletions src/include/usr/mbox/mboxif.H
View file
Expand Up @@ -5,7 +5,7 @@
/* */
/* OpenPOWER HostBoot Project */
/* */
/* Contributors Listed Below - COPYRIGHT 2012,2014 */
/* Contributors Listed Below - COPYRIGHT 2012,2017 */
/* [+] International Business Machines Corp. */
/* */
/* */
Expand Down Expand Up @@ -59,7 +59,7 @@ namespace MBOX
* Send message asynchronously
* @param[in] i_q_id id, of the FSP message queue to send the msg to.
* @param[in] i_msg. The message to send.
* @param[in] i_node The destination node [0-7] for IPC messages,
* @param[in] i_node The destination node [0-7] for IPC messages,
* otherwise the default is FSP
*
* @return errlHndl_t on error.
Expand All @@ -71,8 +71,8 @@ namespace MBOX
* @pre For FSP mbox messages, i_msg->extra_data must be NULL If there is
* no payload associated with the message.
*
* @pre For FSP mbox messages,
* i_msg->extra_data = malloc(size); i_msg->data[1] = size;
* @pre For FSP mbox messages,
* i_msg->extra_data = malloc(size); i_msg->data[1] = size;
* Any extra data associated with i_msg was obtained from the heap
* using malloc and i_msg->data[1] contains the length of that data.
*
Expand All @@ -93,7 +93,7 @@ namespace MBOX
* @pre io_msg->extra_data == NULL If there is no extra data is associated
* with the message.
*
* @pre io_msg->extra_data = malloc(size); io_msg->data[1] = size;
* @pre io_msg->extra_data = malloc(size); io_msg->data[1] = size;
* Any extra data associated with io_msg was obtained from the heap
* using malloc and io_msg->data[1] contains the length of that data.
*
Expand All @@ -114,7 +114,7 @@ namespace MBOX
bool mailbox_enabled();

/**
* Suspend the mailbox.
* Suspend the mailbox.
*
* @param[in] i_disable_hw_int, disable interrupts from hw along
* with SW suspend. Defaults to SW suspend only
Expand Down Expand Up @@ -162,6 +162,12 @@ namespace MBOX
*/
void deallocate(void * i_ptr);

/**
* Reclaim any DMA buffers owned by the FSP
* @return errlHndl_t on error
*/
errlHndl_t reclaimDmaBfrsFromFsp( void );

}; // end namespace MBOX

#endif
25 changes: 22 additions & 3 deletions src/usr/isteps/istep16/call_host_activate_master.C
View file
Expand Up @@ -166,9 +166,28 @@ void* call_host_activate_master (void *io_pArgs)
//Because of a bug in how the SBE injects the IPI used to wake
//up the master core, need to ensure no mailbox traffic
//or even an interrupt in the interrupt presenter
// 1) suspend the mailbox with interrupt disable
// 2) tell the SBE to start the deadman timer
// 3) ensure that interrupt presenter is drained
// 1) Reclaim all DMA bfrs from the FSP
// 2) suspend the mailbox with interrupt disable
// 3) tell the SBE to start the deadman timer
// 4) ensure that interrupt presenter is drained
l_errl = MBOX::reclaimDmaBfrsFromFsp();
if (l_errl)
{
TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
"call_host_activate_master ERROR : "
"MBOX::reclaimDmaBfrsFromFsp");

// if it not complete then thats okay, but we want to store the
// log away somewhere. Since we didn't get all the DMA buffers
// back its not a big deal to commit a log, even if we lose a
// DMA buffer because of it it doesn't matter that much.
// this will generate more traffic to the FSP
l_errl->setSev(ERRORLOG::ERRL_SEV_INFORMATIONAL);
errlCommit( l_errl, HWPF_COMP_ID );

// (do not break. keep going to suspend)
}

l_errl = MBOX::suspend(true, true);
if (l_errl)
{
Expand Down
207 changes: 185 additions & 22 deletions src/usr/mbox/mailboxsp.C
View file
Expand Up @@ -49,6 +49,7 @@
#include <kernel/console.H>
#include <arch/pirformat.H>
#include <sbeio/sbeioif.H>
#include <sys/time.h>

// Local functions
namespace MBOX
Expand Down Expand Up @@ -90,7 +91,9 @@ MailboxSp::MailboxSp()
iv_allow_blk_resp(false),
iv_sum_alloc(0),
iv_pend_alloc(),
iv_allocAddrs()
iv_allocAddrs(),
iv_reclaim_sent_cnt(0),
iv_reclaim_rsp_cnt(0)
{
// mailbox target
TARGETING::targetService().masterProcChipTargetHandle(iv_trgt);
Expand Down Expand Up @@ -280,27 +283,55 @@ void MailboxSp::msgHandler()
crit_assert(0);
}

// note : an outstanding req dma bfrs msg
// will cause quiesce to be false
if(iv_shutdown_msg && quiesced())
{
// If all DMA buffers still not owned
// try once to get them all back.
if(!iv_dmaBuffer.ownsAllBlocks() &&
!iv_dmaBuffer.shutdownDmaRequestSent())
if // all DMA buffers have been reclaimed
( iv_dmaBuffer.ownsAllBlocks() )
{
iv_dmaBuffer.setShutdownDmaRequestSent(true);
mbox_msg_t dma_request_msg;
dma_request_msg.msg_queue_id = FSP_MAILBOX_MSGQ;
dma_request_msg.msg_payload.type =
MSG_REQUEST_DMA_BUFFERS;
dma_request_msg.msg_payload.__reserved__async = 1;

send_msg(&dma_request_msg);
// continue with shutdown
TRACFCOMP(g_trac_mbox,
INFO_MRK"MBOXSP DMA bfrs reclaimed "
"on shutdown");

handleShutdown();
}

else if // a "reclaim bfr" msg is outstanding
( isDmaReqBfrMsgOutstanding() )
{
// (need to wait for the msg(s) to complete
// before sending another msg)
TRACFCOMP(g_trac_mbox,
INFO_MRK
"MailboxSp::msgHandler - "
"Wait for Reclaim Msg Completion");
}

else if // more "reclaim bfr" msgs can be sent
( iv_dmaBuffer.maxShutdownDmaRequestSent() == false )
{
TRACFCOMP(g_trac_mbox,
INFO_MRK
"MailboxSp::msgHandler - "
"Send Reclaim Msg to FSP");

// send a "reclaim bfr" msg
iv_dmaBuffer.incrementShutdownDmaRequestSentCnt();
sendReclaimDmaBfrsMsg();
}

else
{
// continue with shutdown
TRACFCOMP(g_trac_mbox,
INFO_MRK"MBOXSP DMA bfrs not reclaimed "
"on shutdown");

handleShutdown();
}
}
} // end shutdown msg & quiesced

if(iv_suspended && quiesced())
{
Expand Down Expand Up @@ -364,15 +395,11 @@ void MailboxSp::msgHandler()
// is not pending, but may not get them all back
// at this time.
//
if(!iv_dmaBuffer.ownsAllBlocks() && !iv_dma_pend)
if( !iv_dmaBuffer.ownsAllBlocks() &&
!iv_dma_pend &&
!isDmaReqBfrMsgOutstanding() )
{
mbox_msg_t dma_request_msg;
dma_request_msg.msg_queue_id = FSP_MAILBOX_MSGQ;
dma_request_msg.msg_payload.type =
MSG_REQUEST_DMA_BUFFERS;
dma_request_msg.msg_payload.__reserved__async = 1;

send_msg(&dma_request_msg);
sendReclaimDmaBfrsMsg();
}

if(quiesced()) //already in shutdown state
Expand Down Expand Up @@ -1146,6 +1173,9 @@ void MailboxSp::handle_hbmbox_resp(mbox_msg_t & i_mbox_msg)
iv_dmaBuffer.addBuffers
(i_mbox_msg.msg_payload.data[0]);

// track response received
iv_reclaim_rsp_cnt++;

iv_dma_pend = false;

send_msg(); // send next message, if there is one
Expand Down Expand Up @@ -1241,6 +1271,121 @@ errlHndl_t MailboxSp::send(queue_id_t i_q_id,
return err;
}

/**
* Reclaim any DMA buffers owned by the FSP
*/
errlHndl_t MailboxSp::reclaimDmaBfrsFromFsp( void )
{
errlHndl_t err = NULL;

// locate the FSP mailbox
MailboxSp & fspMbox = Singleton<MailboxSp>::instance();

// reclaim the dma bfrs
err = fspMbox._reclaimDmaBfrsFromFsp();

return( err );
}

errlHndl_t MailboxSp::_reclaimDmaBfrsFromFsp( void )
{
errlHndl_t err = NULL;
int msgSentCnt = 0;
int maxDmaBfrs = iv_dmaBuffer.maxDmaBfrs();

TRACFBIN(g_trac_mbox,
INFO_MRK
"MailboxSp::_reclaimDmaBfrsFromFsp - Start."
" DmaBuffer = ",
&iv_dmaBuffer,
sizeof(iv_dmaBuffer) );

while // bfrs still need to be reclaimed
( iv_dmaBuffer.ownsAllBlocks() == false )
{
if // request dma bfrs msg is outstanding
( isDmaReqBfrMsgOutstanding() == true )
{
// (wait for msg to complete)
nanosleep( 0, 1000000 ); // 1ms to avoid tight busy loop
task_yield();
}

else if // can send another request dma bfrs msg
(msgSentCnt < maxDmaBfrs)
{
// send the message
msgSentCnt++;
sendReclaimDmaBfrsMsg();
}

else
{
// (sent max number of reclaims and bfrs still not free)
// (something real bad is happening, exit so we don't hang)

// create a snapshot of DMA buffer control object for tracing
char dmyArray[sizeof(iv_dmaBuffer)];
memcpy( &dmyArray[0],
(void *)&iv_dmaBuffer,
sizeof(dmyArray) );

TRACFBIN(g_trac_mbox,
ERR_MRK
"MailboxSp::_reclaimDmaBfrsFromFsp -"
"Reclaim Did Not Complete. "
"DmaBuffer = ",
&dmyArray[0],
sizeof(dmyArray) );

break;
}
} // end wait for bfrs to be reclaimed

return( err );
}


void MailboxSp::sendReclaimDmaBfrsMsg( void )
{
// allocate local msg bfr on the stack
mbox_msg_t local_msg_bfr;

sendReclaimDmaBfrsMsg( local_msg_bfr );

return;
}


void MailboxSp::sendReclaimDmaBfrsMsg( mbox_msg_t & i_mbox_msg )
{
// isolate all occurrences of this message to this routine
// so the total number of outstanding Request DMA Bfr
// messages can be tracked in one place. This allows
// a mechanism to determine if any of these messages are
// on either the message Q or have been sent on HW to FSP.
// iv_dma_pend only tracks the msg from load on HW to
// response received. Does not consider Queue.
TRACFCOMP(g_trac_mbox,
INFO_MRK
"MailboxSp::sendReclaimDmaBfrsMsg - "
"Send Reclaim Msg to FSP");

// send a request dma bfrs msg to reclaim from fsp
new (&i_mbox_msg) mbox_msg_t();

i_mbox_msg.msg_queue_id = FSP_MAILBOX_MSGQ;
i_mbox_msg.msg_payload.type = MSG_REQUEST_DMA_BUFFERS;
i_mbox_msg.msg_payload.__reserved__async = 1;

// track the msg until completion;
iv_reclaim_sent_cnt++;

send_msg(&i_mbox_msg);

return;
}


errlHndl_t MailboxSp::msgq_register(queue_id_t i_queue_id, msg_q_t i_msgQ)
{
Expand Down Expand Up @@ -2279,3 +2424,21 @@ void MBOX::deallocate(void * i_ptr)
}
}

errlHndl_t MBOX::reclaimDmaBfrsFromFsp( void )
{
errlHndl_t err = NULL;

msg_q_t mboxQ = msg_q_resolve(VFS_ROOT_MSG_MBOX);
if(mboxQ)
{
// reclaim the dma bfrs
err = MailboxSp::reclaimDmaBfrsFromFsp();
}
else
{
TRACFCOMP(g_trac_mbox, ERR_MRK"MBOX::reclaimDmaBfrsFromFsp - "
"Mailbox Service not available");
}

return( err );
}

0 comments on commit 046a265

Please sign in to comment.