Skip to content

Commit

Permalink
Close and Disable TCEs on Non-Master Nodes
Browse files Browse the repository at this point in the history 
During istep21.1 this commit sends a message to all non-Master nodes
to close and disable their TCEs using the ipc mailbox interface to
the FSP.

Change-Id: I5575228c9f237025a48e0aeca23e741724ec7886
RTC:187335
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/56269
CI-Ready: Michael Baiocchi <mbaiocch@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com>
Reviewed-by: ILYA SMIRNOV <ismirno@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>
  • Loading branch information
@mabaiocchi @wghoffa
mabaiocchi authored and wghoffa committed Mar 29, 2018
1 parent 42e4c42 commit 95c1dd7
Show file tree
Hide file tree
Showing 5 changed files with 201 additions and 6 deletions.
1 change: 1 addition & 0 deletions src/include/usr/mbox/ipc_msg_types.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ namespace IPC
IPC_START_PAYLOAD,
IPC_QUERY_CHIPINFO,
IPC_SET_SBE_CHIPINFO,
IPC_CLOSE_TCES,
};

}; // namespace IPC
Expand Down
1 change: 1 addition & 0 deletions src/include/usr/mbox/mbox_queues.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ namespace MBOX
HB_POP_ATTR_MSGQ = 10, // populate Attribute response
HB_COALESCE_MSGQ = 11, //host_coalesce response
HB_SBE_SYSCONFIG_MSGQ = 12, //For SBE System Config response
HB_CLOSE_TCES_MSGQ = 13, // close/disable TCEs

// Add HB mbox msg queue ids (services) before this line
HB_LAST_VALID_MSGQ, // end of valid HB mbox msgQ ids
Expand Down
126 changes: 123 additions & 3 deletions src/usr/isteps/istep21/call_host_runtime_setup.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,13 @@
#include <runtime/runtime.H>
#include <runtime/customize_attrs_for_payload.H>
#include <targeting/common/util.H>
#include <targeting/targplatutil.H>
#include <vpd/vpd_if.H>
#include <util/utiltce.H>
#include <util/utilmclmgr.H>
#include <map>
#include <sys/internode.h>
#include <mbox/ipc_msg_types.H>

#include <secureboot/service.H>
#include <secureboot/containerheader.H>
Expand All @@ -63,6 +66,117 @@ using namespace TARGETING;
namespace ISTEP_21
{


// Direct non-master nodes to close and disable their TCEs
errlHndl_t closeNonMasterTces(void)
{
errlHndl_t l_err = nullptr;
uint64_t nodeid = TARGETING::UTIL::getCurrentNodePhysId();

TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
ENTER_MRK"closeNonMasterTces(): nodeid=%d", nodeid);

// keep track of the number of messages we send so we
// know how many responses to expect
uint64_t msg_count = 0;

do{

TARGETING::Target * sys = nullptr;
TARGETING::targetService().getTopLevelTarget( sys );
assert(sys != nullptr, "closeNonMasterTces() system target is nullptr");

TARGETING::ATTR_HB_EXISTING_IMAGE_type hb_images =
sys->getAttr<TARGETING::ATTR_HB_EXISTING_IMAGE>();
// This msgQ catches the node responses from the commands
msg_q_t msgQ = msg_q_create();
l_err = MBOX::msgq_register(MBOX::HB_CLOSE_TCES_MSGQ,msgQ);

if(l_err)
{
TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
"closeNonMasterTces(): MBOX::msgq_register failed!" );
break;
}

// loop thru rest all nodes -- sending msg to each
TARGETING::ATTR_HB_EXISTING_IMAGE_type mask = 0x1 <<
((sizeof(TARGETING::ATTR_HB_EXISTING_IMAGE_type) * 8) -1);

TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
"closeNonMasterTces(): HB_EXISTING_IMAGE (mask) = 0x%X, "
"(hb_images=0x%X)",
mask, hb_images);

for (uint64_t l_node=0; (l_node < MAX_NODES_PER_SYS); l_node++ )
{
if (l_node == nodeid)
{
TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
"closeNonMasterTces(): don't send IPC_CLOSE_TCES "
"message to master node %d",
nodeid );
continue;
}

if( 0 != ((mask >> l_node) & hb_images ) )
{
TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
"closeNonMasterTces(): send IPC_CLOSE_TCES message "
"to node %d",
l_node );

msg_t * msg = msg_allocate();
msg->type = IPC::IPC_CLOSE_TCES;
msg->data[0] = l_node; // destination node
msg->data[1] = nodeid; // respond to this node

// send the message to the slave hb instance
l_err = MBOX::send(MBOX::HB_IPC_MSGQ, msg, l_node);

if( l_err )
{
TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
"closeNonMasterTces(): MBOX::send to node %d "
"failed",
l_node);
break;
}

++msg_count;

} // end of node to process
} // end for loop on nodes

// wait for a response to each message we sent
if( l_err == nullptr )
{
//$TODO RTC:189356 - need timeout here
while(msg_count)
{
msg_t * response = msg_wait(msgQ);
TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
"closeNonMasterTces(): IPC_CLOSE_TCES : node %d "
"completed",
response->data[0]);
msg_free(response);
--msg_count;
}
}

MBOX::msgq_unregister(MBOX::HB_CLOSE_TCES_MSGQ);
msg_q_destroy(msgQ);

} while(0);

TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
EXIT_MRK"closeNonMasterTces(): l_err rc = 0x%X, msg_count=%d",
ERRL_GETRC_SAFE(l_err), msg_count );

return l_err;
}


// Verify PAYLOAD and Move PAYLOAD+HDAT from Temporary TCE-related
// memory region to the proper location
errlHndl_t verifyAndMovePayload(void)
Expand Down Expand Up @@ -446,7 +560,6 @@ void* call_host_runtime_setup (void *io_pArgs)
// Close PAYLOAD TCEs
if (TCE::utilUseTcesForDmas())
{

l_err = TCE::utilClosePayloadTces();
if ( l_err )
{
Expand All @@ -456,8 +569,15 @@ void* call_host_runtime_setup (void *io_pArgs)
break;
}

// @TODO RTC 187335 Close TCEs on non-master nodes

// Close TCEs on non-master nodes
l_err = closeNonMasterTces();
if ( l_err )
{
TRACFCOMP( ISTEPS_TRACE::g_trac_isteps_trace,
"Failed closeNonMasterTces" );
// break from do loop if error occurred
break;
}
}

// Need to load up the runtime module if it isn't already loaded
Expand Down
72 changes: 70 additions & 2 deletions src/usr/mbox/ipcSp.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,9 @@
#include <mbox/mbox_reasoncodes.H>
#include <intr/interrupt.H>
#include <initservice/initserviceif.H>
#include <sbeio/sbeioif.H>

#include <sbeio/sbeioif.H>
#include <util/utiltce.H>
#include <targeting/targplatutil.H>

namespace ISTEP_21
{
Expand Down Expand Up @@ -321,6 +322,73 @@ void IpcSp::msgHandler()

break;
}

case IPC_CLOSE_TCES:
{
TRACFCOMP( g_trac_ipc,
"Closing TCEs on this node (%d)",
TARGETING::UTIL::getCurrentNodePhysId());

// make sure util module is loaded
const char * libutil = "libutil.so";
if ( !VFS::module_is_loaded( libutil ) )
{
err = VFS::module_load( libutil );

if ( err )
{
TRACFCOMP( g_trac_ipc,
"Could not load util module" );
}
else
{
mod_loaded = true;
}
}

if(!err)
{
err = TCE::utilClosePayloadTces();
}

if(!err)
{
err = TCE::utilDisableTces();
}

if(err)
{
uint32_t l_errPlid = err->plid();
errlCommit(err, IPC_COMP_ID);
INITSERVICE::doShutdown(l_errPlid, true);
}

if(mod_loaded)
{
err = VFS::module_unload( libutil );

if (err)
{
errlCommit(err, IPC_COMP_ID);
}
else
{
mod_loaded = false;
}
}

// Respond
err = MBOX::send(MBOX::HB_CLOSE_TCES_MSGQ, msg, msg->data[1] );
if (err)
{
uint32_t l_errPlid = err->plid();
errlCommit(err,IPC_COMP_ID);
INITSERVICE::doShutdown(l_errPlid, true);
}

break;
}

default:

TRACFCOMP( g_trac_ipc,
Expand Down
7 changes: 6 additions & 1 deletion src/usr/util/utiltcemgr.C
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,12 @@ errlHndl_t utilClosePayloadTces(void)
// Close the Unsecure Memory Region that was opened for the FSP to run
// PSI Diagnostics Test using the PAYLOAD section
// -- addr is a constant for PAYLOAD
errl = SBEIO::closeUnsecureMemRegion(MCL_TMP_ADDR,
// -- Address must be HRMOR-specific
uint64_t hrmorVal = cpu_spr_value(CPU_SPR_HRMOR);
uint64_t addr = hrmorVal - VMM_HRMOR_OFFSET + MCL_TMP_ADDR;
TRACUCOMP(g_trac_tce,"utilClosePayloadTces(): addr=0x%.16llX, hrmor=0x%.16llX", addr, hrmorVal);

errl = SBEIO::closeUnsecureMemRegion(addr,
nullptr); //Master Processor
if(errl)
{
Expand Down

0 comments on commit 95c1dd7

Please sign in to comment.