Setup Node-Level Attributes for Multinode TCE Support

This commit establishes a TCE Table and allocates the necessary TCEs for each node in a multinode (ie, Fleetwood) system. It then sets the corresponding node-level TCE attributes such that the FSP can use these TCEs for PSI Diagnostics and DMAing-via-TCEs the PAYLOAD and HDAT content. Change-Id: Ibd06675ea68a9c630987765b51fdd9db631f6425 RTC:187335 Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/56129 CI-Ready: Michael Baiocchi <mbaiocch@us.ibm.com> Reviewed-by: ILYA SMIRNOV <ismirno@us.ibm.com> Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com> Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com> Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com> Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com> Reviewed-by: Nicholas E. Bofferding <bofferdn@us.ibm.com> Reviewed-by: Marshall J. Wilks <mjwilks@us.ibm.com> Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
open-power · Mar 26, 2018 · fe61cf0 · fe61cf0
1 parent a7decd2
commit fe61cf0
Show file tree

Hide file tree

Showing 5 changed files with 105 additions and 28 deletions.
diff --git a/src/include/usr/targeting/targplatutil.H b/src/include/usr/targeting/targplatutil.H
@@ -198,6 +198,34 @@ inline bool isThisMasterNodeTarget(const Target* const i_pTarget)
  */
 bool isCurrentMasterNode();
 
+#ifndef __HOSTBOOT_RUNTIME
+/* @brief - returns the Target of the current Node
+ *
+ * @par Detailed Description:
+ * Hostboot only runs on one node at a time and this interface returns
+ * the target of this node.
+ *
+ * NOTE: Assert if more or less than 1 node is found in targeting information.
+ * NOTE: Assert if Node Target is nullptr.
+ *
+ * @return Pointer to Target of current Node
+ */
+Target* getCurrentNodeTarget(void);
+
+/* @brief - returns the physical Node Id of the current Node
+ *
+ * @par Detailed Description:
+ * Hostboot only runs on one node at a time and this interface returns
+ * the Node Id of this node as defined in its entity path.
+ *
+ * NOTE: Assert if more or less than 1 node is found in targeting information.
+ * NOTE: Assert if Node Target is nullptr.
+ *
+ * @return unit8_t indicating the Node Id of the current Node
+ */
+uint8_t getCurrentNodePhysId(void);
+#endif
+
 /* @brief - Syncs the master system target's attribute with non-master system
  * targets.
  *

diff --git a/src/usr/isteps/istep16/call_host_ipl_complete.C b/src/usr/isteps/istep16/call_host_ipl_complete.C
@@ -111,18 +111,8 @@ void* call_host_ipl_complete (void *io_pArgs)
             l_err = nullptr;
         }
 #endif
-        //@TODO RTC:187335
-        //TCE setup is broken on multinode.  This is a quick hack to only do it
-        //on drawer 0 (and if you don't have a drawer 0 this is busted)
-        TARGETING::Target* mproc = nullptr;
-        TARGETING::targetService().masterProcChipTargetHandle(mproc);
-        TARGETING::EntityPath epath = mproc->getAttr<TARGETING::ATTR_PHYS_PATH>();
-        const TARGETING::EntityPath::PathElement pe =
-          epath.pathElementOfType(TARGETING::TYPE_NODE);
-        uint64_t nodeid = pe.instance;
-
-        // Setup the TCEs needed for the FSP to DMA the PAYLOAD
-        if (TCE::utilUseTcesForDmas() && (0 == nodeid))
+
+        if (TCE::utilUseTcesForDmas())
         {
             l_err = TCE::utilSetupPayloadTces();
 

diff --git a/src/usr/isteps/istep21/call_host_runtime_setup.C b/src/usr/isteps/istep21/call_host_runtime_setup.C
@@ -455,6 +455,9 @@ void* call_host_runtime_setup (void *io_pArgs)
                 // break from do loop if error occurred
                 break;
             }
+
+            // @TODO RTC 187335 Close TCEs on non-master nodes
+
         }
 
         // Need to load up the runtime module if it isn't already loaded

diff --git a/src/usr/targeting/targplatutil.C b/src/usr/targeting/targplatutil.C
@@ -123,6 +123,31 @@ bool isCurrentMasterNode()
     return (isMaster == 1);
 }
 
+#ifndef __HOSTBOOT_RUNTIME
+Target* getCurrentNodeTarget(void)
+{
+    // Get node target
+    TargetHandleList l_nodelist;
+    getEncResources(l_nodelist, TARGETING::TYPE_NODE,
+                    TARGETING::UTIL_FILTER_FUNCTIONAL);
+    assert(l_nodelist.size() == 1, "ERROR, only expect one node.");
+
+    Target* pTgt =  l_nodelist[0];
+    assert(pTgt != nullptr, "getCurrentNodeTarget found nullptr");
+
+    return pTgt;
+}
+
+uint8_t getCurrentNodePhysId(void)
+{
+    Target* pNodeTgt = getCurrentNodeTarget();
+    EntityPath epath = pNodeTgt->getAttr<TARGETING::ATTR_PHYS_PATH>();
+    const TARGETING::EntityPath::PathElement pe =
+              epath.pathElementOfType(TARGETING::TYPE_NODE);
+    return pe.instance;
+}
+#endif
+
 // return the sensor number from the passed in target
 uint32_t getSensorNumber( const TARGETING::Target* i_pTarget,
                           TARGETING::SENSOR_NAME i_name )

diff --git a/src/usr/util/utiltcemgr.C b/src/usr/util/utiltcemgr.C
@@ -32,10 +32,12 @@
 #include <util/align.H>
 #include <sys/mmio.h>
 #include <sys/mm.h>
+#include <sys/misc.h>
 #include <arch/ppc.H>
 #include <errno.h>
 #include <targeting/common/commontargeting.H>
 #include <targeting/common/utilFilter.H>
+#include <targeting/targplatutil.H>
 #include <kernel/console.H>
 #include "utiltcemgr.H"
 #include <util/util_reasoncodes.H>
@@ -128,14 +130,19 @@ errlHndl_t utilSetupPayloadTces(void)
     uint64_t addr=0x0;
     size_t   size=0x0;
     uint32_t token=0x0;
+    uint8_t  nodeId = TARGETING::UTIL::getCurrentNodePhysId();
 
     do{
 
-    TRACFCOMP(g_trac_tce,ENTER_MRK"utilSetupPayloadTces()");
+    TRACFCOMP(g_trac_tce,ENTER_MRK"utilSetupPayloadTces(): nodeId=0x%X", nodeId);
 
     // Allocate TCEs for PAYLOAD to Temporary Space
-    addr = MCL_TMP_ADDR;
+    // -- Address must be HRMOR-specific
+    uint64_t hrmorVal = cpu_spr_value(CPU_SPR_HRMOR);
+    addr = hrmorVal - VMM_HRMOR_OFFSET + MCL_TMP_ADDR;
     size = MCL_TMP_SIZE;
+    TRACUCOMP(g_trac_tce,"utilSetupPayloadTces(): addr=0x%.16llX, hrmor=0x%.16llX, size=0x%X", addr, hrmorVal, size);
+
     errl = utilAllocateTces(addr, size, token);
     if (errl)
     {
@@ -148,24 +155,32 @@ errlHndl_t utilSetupPayloadTces(void)
     }
 
     // Set attribute to tell FSP what the PAYLOAD token is
+    TARGETING::Target* pNodeTgt = TARGETING::UTIL::getCurrentNodeTarget();
+    pNodeTgt->setAttr<TARGETING::ATTR_TCE_START_TOKEN_FOR_PAYLOAD>(token);
+
+    // For PSI Diagnostics Test the FSP writes and reads back patterns to the
+    // PAYLOAD section via PSI and FSI so it needs to know the starting memory
+    // address of this section and have an unsecure read-write memory region
+    // opened for it
+    pNodeTgt->setAttr<TARGETING::ATTR_START_MEM_ADDRESS_FOR_PAYLOAD_TCE_TOKEN>(addr);
+
+    // Legacy Support: Also set System Level Attributes on Node 0
+    // @TODO RTC 190014 Can Be Removed Once FSP Switches To Node Attributes
     // Get Target Service and the system target to set attributes
     TARGETING::TargetService& tS = TARGETING::targetService();
     TARGETING::Target* sys = nullptr;
     (void) tS.getTopLevelTarget( sys );
     assert(sys, "utilSetupPayloadTces() system target is NULL");
+    if (nodeId == 0)
+    {
+        sys->setAttr<TARGETING::ATTR_TCE_START_TOKEN_FOR_PAYLOAD>(token);
+        sys->setAttr<TARGETING::ATTR_START_MEM_ADDRESS_FOR_PAYLOAD_TCE_TOKEN>(addr);
+    }
 
-    sys->setAttr<TARGETING::ATTR_TCE_START_TOKEN_FOR_PAYLOAD>(token);
-
-    // Save for internal use since we can't trust FSP won't change the attribute
+    // Save for internal use since can't trust FSP won't change attribute
     Singleton<UtilTceMgr>::instance().setToken(UtilTceMgr::PAYLOAD_TOKEN,
                                                token);
 
-    // For PSI Diagnostics Test the FSP writes and reads back patterns to the
-    // PAYLOAD section via PSI and FSI so it needs to know the starting memory
-    // address of this section and have an unsecure read-write memory region
-    // opened for it
-    sys->setAttr<TARGETING::ATTR_START_MEM_ADDRESS_FOR_PAYLOAD_TCE_TOKEN>(addr);
-
     // Open Read-Write Unsecure Memory Region
     errl = SBEIO::openUnsecureMemRegion(addr,
                                         size,
@@ -179,8 +194,10 @@ errlHndl_t utilSetupPayloadTces(void)
     }
 
     // Allocate TCEs for HDAT
-    addr = HDAT_TMP_ADDR;
+    // -- Address must be HRMOR-specific
+    addr = hrmorVal - VMM_HRMOR_OFFSET + HDAT_TMP_ADDR;
     size = HDAT_TMP_SIZE;
+
     errl = utilAllocateTces(addr, size, token);
     if (errl)
     {
@@ -193,7 +210,14 @@ errlHndl_t utilSetupPayloadTces(void)
     }
 
     // Set attribute to tell FSP what the HDAT token is
-    sys->setAttr<TARGETING::ATTR_TCE_START_TOKEN_FOR_HDAT>(token);
+    pNodeTgt->setAttr<TARGETING::ATTR_TCE_START_TOKEN_FOR_HDAT>(token);
+
+    // Legacy Support: Also set System Level Attributes on Node 0
+    // @TODO RTC 190014 Can Be Removed Once FSP Switches To Node Attributes
+    if (nodeId == 0)
+    {
+         sys->setAttr<TARGETING::ATTR_TCE_START_TOKEN_FOR_HDAT>(token);
+    }
 
     // Save for internal use since we can't trust FSP won't change the attribute
     Singleton<UtilTceMgr>::instance().setToken(UtilTceMgr::HDAT_TOKEN,
@@ -211,10 +235,11 @@ errlHndl_t utilClosePayloadTces(void)
     errlHndl_t errl = nullptr;
 
     uint32_t token=0x0;
+    uint8_t  nodeId = TARGETING::UTIL::getCurrentNodePhysId();
 
     do{
 
-    TRACFCOMP(g_trac_tce,ENTER_MRK"utilClosePayloadTces()");
+    TRACFCOMP(g_trac_tce,ENTER_MRK"utilClosePayloadTces(): nodeId=0x%X", nodeId);
 
     // Close PAYLOAD TCEs
     token = Singleton<UtilTceMgr>::instance().getToken(UtilTceMgr::PAYLOAD_TOKEN);
@@ -270,9 +295,14 @@ UtilTceMgr::UtilTceMgr(const uint64_t i_tableAddr, const size_t i_tableSize)
   ,iv_payloadToken(INVALID_TOKEN_VALUE)
   ,iv_hdatToken(INVALID_TOKEN_VALUE)
 {
+
+    // Need to set up TCE Table with HRMOR-specific Address
+    uint64_t hrmorVal = cpu_spr_value(CPU_SPR_HRMOR);
+    iv_tceTablePhysAddr = hrmorVal - VMM_HRMOR_OFFSET + TCE_TABLE_ADDR;
+
     // Table Address must be 4MB Aligned and default input is TCE_TABLE_ADDR
     static_assert( TCE_TABLE_ADDR % TCE_TABLE_ADDRESS_ALIGNMENT == 0,"TCE Table must align on 4 MB boundary");
-    assert( i_tableAddr % TCE_TABLE_ADDRESS_ALIGNMENT == 0,"TCE Table must align on 4 MB boundary: 0x%.16llX", i_tableAddr);
+    assert( iv_tceTablePhysAddr % TCE_TABLE_ADDRESS_ALIGNMENT == 0,"TCE Table must align on 4 MB boundary: 0x%.16llX", iv_tceTablePhysAddr);
 
     // TCE Entry counts are based on the following assumption
     static_assert((sizeof(uint64_t) == sizeof(TceEntry_t)), "TceEntry_t struct must be size of uint64_t)");
@@ -282,7 +312,8 @@ UtilTceMgr::UtilTceMgr(const uint64_t i_tableAddr, const size_t i_tableSize)
 
     iv_tceEntryCount = iv_tceTableSize/(sizeof (uint64_t));
 
-    TRACUCOMP(g_trac_tce,"UtilTceMgr::UtilTceMgr: iv_tceTableVaAddr=0x%.16llX, iv_tceTablePhysAddr=0x%.16llX, iv_tceTableSize=0x%llX, iv_tceEntryCount=0x%X, iv_allocatedAddrs,size=%d", iv_tceTableVaAddr, iv_tceTablePhysAddr, iv_tceTableSize, iv_tceEntryCount, iv_allocatedAddrs.size());
+
+    TRACUCOMP(g_trac_tce,"UtilTceMgr::UtilTceMgr: iv_tceTableVaAddr=0x%.16llX, iv_tceTablePhysAddr=0x%.16llX, iv_tceTableSize=0x%llX, iv_tceEntryCount=0x%X, iv_allocatedAddrs,size=%d, hrmorVal=0x%.16llX", iv_tceTableVaAddr, iv_tceTablePhysAddr, iv_tceTableSize, iv_tceEntryCount, iv_allocatedAddrs.size(), hrmorVal);
 
     // Initialize HW without Initializing Table so that FSP cannot DMA
     // to memory without Hostboot control