Skip to content

Commit

Permalink
Validate OMI INBAND BAR offset attributes against calculated values
Browse files Browse the repository at this point in the history 
While setting up the virtual memory mapped IO to the OCMB chips we
make some assumptions that the OCMB MMIO spaces will be contiguous.
The p9a_omi_setup_bars HWP uses OMI_INBAND_BAR_BASE_ADDR_OFFSET to
set the scom registers that determine the physical offset mapped
to the IO. When setting up the Virtual addresses hostboot uses to
represent the physical mmio address, we must validate that the attribute
matches with what we calculated. While doing this we found that the
virtual address attribute was being calculated incorrectly. It was
not localizing the OCMB position relative to the MC which is required
when calculating the offset into the MC bar.

Change-Id: I0ebbcd38e19a238e2cc16791bb0595536788bb7f
RTC: 201493
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/75631
Reviewed-by: Matthew Raybuck <matthew.raybuck@ibm.com>
Reviewed-by: Michael Baiocchi <mbaiocch@us.ibm.com>
Reviewed-by: Roland Veloz <rveloz@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>
  • Loading branch information
@crgeddes @dcrowell77
crgeddes authored and dcrowell77 committed Apr 18, 2019
1 parent 9d139bd commit e583424
Show file tree
Hide file tree
Showing 5 changed files with 122 additions and 115 deletions.
3 changes: 2 additions & 1 deletion src/include/arch/memorymap.H
View file
Expand Up @@ -5,7 +5,7 @@
/* */
/* OpenPOWER HostBoot Project */
/* */
/* Contributors Listed Below - COPYRIGHT 2017,2018 */
/* Contributors Listed Below - COPYRIGHT 2017,2019 */
/* [+] International Business Machines Corp. */
/* */
/* */
Expand Down Expand Up @@ -38,6 +38,7 @@
constexpr uint64_t MMIO_OFFSET_PER_CHIP = (4*TERABYTE); //0x40000000000
constexpr uint64_t MMIO_OFFSET_PER_GROUP = (32*TERABYTE); //0x200000000000

constexpr uint64_t MMIO_BASE = 0x6000000000000;
/**
* @brief Compute MMIO value for a given chip and base value
*/
Expand Down
3 changes: 2 additions & 1 deletion src/include/usr/mmio/mmio_reasoncodes.H
View file
Expand Up @@ -5,7 +5,7 @@
/* */
/* OpenPOWER HostBoot Project */
/* */
/* Contributors Listed Below - COPYRIGHT 2011,2018 */
/* Contributors Listed Below - COPYRIGHT 2011,2019 */
/* [+] International Business Machines Corp. */
/* */
/* */
Expand Down Expand Up @@ -51,6 +51,7 @@ namespace MMIO
RC_BAD_MMIO_READ = MMIO_COMP_ID | 0x08,
RC_BAD_MMIO_WRITE = MMIO_COMP_ID | 0x09,
RC_PROC_NOT_FOUND = MMIO_COMP_ID | 0x0A,
RC_BAR_OFFSET_MISMATCH = MMIO_COMP_ID | 0x0B,
};
};

Expand Down
5 changes: 2 additions & 3 deletions src/usr/ibscom/ibscom.C
View file
Expand Up @@ -5,7 +5,7 @@
/* */
/* OpenPOWER HostBoot Project */
/* */
/* Contributors Listed Below - COPYRIGHT 2012,2018 */
/* Contributors Listed Below - COPYRIGHT 2012,2019 */
/* [+] International Business Machines Corp. */
/* */
/* */
Expand Down Expand Up @@ -66,7 +66,6 @@ namespace IBSCOM
// SCOM Register addresses
const uint32_t MBS_FIR = 0x02011400;
const uint32_t MBSIBERR0 = 0x0201141B;
const uint64_t IBSCOM_BASE = 0x0006000000000000;
const uint64_t BIT_18_MASK = 0x0000000000002000;

// Register XSCcom access functions to DD framework
Expand Down Expand Up @@ -217,7 +216,7 @@ errlHndl_t getTargetVirtualAddress(Target* i_target,
uint8_t l_chipId =
l_parentChip->getAttr<ATTR_PROC_EFF_FABRIC_CHIP_ID>();

l_IBScomAddr = computeMemoryMapOffset( IBSCOM_BASE,
l_IBScomAddr = computeMemoryMapOffset( MMIO_BASE,
l_groupId,
l_chipId );
l_IBScomAddr = l_IBScomAddr + l_IBScomBaseAddrOffset;
Expand Down
214 changes: 105 additions & 109 deletions src/usr/mmio/mmio.C
View file
Expand Up @@ -5,7 +5,7 @@
/* */
/* OpenPOWER HostBoot Project */
/* */
/* Contributors Listed Below - COPYRIGHT 2018 */
/* Contributors Listed Below - COPYRIGHT 2018,2019 */
/* [+] International Business Machines Corp. */
/* */
/* */
Expand All @@ -26,9 +26,11 @@
#include <devicefw/driverif.H>
#include <errl/errlentry.H>
#include <errl/errlmanager.H>
#include <errl/errludtarget.H>
#include <targeting/common/predicates/predicates.H>
#include <targeting/common/utilFilter.H>
#include <targeting/common/targetservice.H>
#include <arch/memorymap.H>
#include <arch/ppc.H>

#include "mmio.H"
Expand All @@ -43,6 +45,8 @@
trace_desc_t* g_trac_mmio = NULL;
TRAC_INIT(&g_trac_mmio, MMIO_COMP_NAME, 2*KILOBYTE, TRACE::BUFFER_SLOW);

#define OMI_PER_MC 8

namespace MMIO
{

Expand All @@ -66,128 +70,118 @@ errlHndl_t mmioSetup()
errlHndl_t l_err = nullptr;

TRACFCOMP(g_trac_mmio, ENTER_MRK"mmioSetup");
// called from istep 12.3

// called after OMI bars have been written to HW registers
do
{
// Get the base BAR address for an OCMB and use it to calculate the base
// BAR address for OCMB0 on PROC0 (beginning of reserved physical memory
// for all OCMBs).
// Each pair of OCMBs uses 8GB of interleaved memory,
// the second OCMB's memory starts 2GB after the first's.
TARGETING::TargetHandleList l_omiTargetList;

getAllChiplets(l_omiTargetList, TARGETING::TYPE_OMI);
if (l_omiTargetList.size() == 0)
{
TRACFCOMP(g_trac_mmio,
INFO_MRK"mmioSetup: Exiting, non-OMI system");
break;
}
// map 8 OCMBs at a time, set MMIO_VM_ADDR on each OCMB
//
// loop through all the Memory Channels (MC Targets)
// call allocate of 32 GB virtual memory space with mmio_dev_map() for each MC
TARGETING::TargetHandleList l_mcTargetList;
getAllChiplets(l_mcTargetList, TARGETING::TYPE_MC);

auto l_omi = l_omiTargetList[0];
auto l_omiParentProc = getParentProc(l_omi);
if (l_omiParentProc == nullptr)
for (auto & l_mcTarget: l_mcTargetList)
{
TRACFCOMP(g_trac_mmio, ERR_MRK
"mmioSetup: Unable to find the parent processor for an"
" OMI(0x%X).",
l_omi->getAttr<TARGETING::ATTR_HUID>());
/*@
* @errortype
* @moduleid MMIO::MOD_MMIO_SETUP
* @reasoncode MMIO::RC_PROC_NOT_FOUND
* @userdata1 Target huid
* @userdata2 None
* @devdesc mmioSetup> Unable to find parent processor for OMI.
* @custdesc Unexpected memory subsystem firmware error.
*/
l_err = new ERRORLOG::ErrlEntry(
ERRORLOG::ERRL_SEV_UNRECOVERABLE,
MMIO::MOD_MMIO_SETUP,
MMIO::RC_PROC_NOT_FOUND,
l_omi->getAttr<TARGETING::ATTR_HUID>(),
0,
ERRORLOG::ErrlEntry::ADD_SW_CALLOUT);

break;
}

// There's a 1:1 relationship between OMIs and OCMBs, so we can directly
// relate the OMI position and BAR base addr to its associated OCMB.

// Get the position of the random OCMB. (OCMBs 0-15 will be on proc0,
// 16-31 on proc1, etc)
auto l_ocmbPos = l_omi->getAttr<TARGETING::ATTR_CHIP_UNIT>();
l_ocmbPos += l_omiParentProc->getAttr<TARGETING::ATTR_POSITION>() *
fapi2::MAX_OMI_PER_PROC;
uint32_t l_mcChipUnit =
l_mcTarget->getAttr<TARGETING::ATTR_CHIP_UNIT>();

// Get the base BAR address of the OCMB.
auto l_ocmbBaseAddr =
l_omi->getAttr<TARGETING::ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET>();
// Get the base BAR address for OpenCapi Memory Interfaces (OMIs) of the this Memory Channel (MC)
auto l_omiBaseAddr =
l_mcTarget->getAttr<TARGETING::ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET>();

// Calculate the base BAR address of OCMB0 on PROC0 by subtracting 8GB
// for every pair of OCMBs beyond the first pair, and subtract an
// additional 2GB if the initial OCMB is the second in a pair.
l_ocmbBaseAddr -= ((l_ocmbPos / 2) * 8 * GIGABYTE) +
((l_ocmbPos % 2) * 2 * GIGABYTE);
// Apply the MMIO base offset so we get the real address
uint64_t l_realAddr = ( l_omiBaseAddr | MMIO_BASE );

// map 8 OCMBs at a time, set MMIO_VM_ADDR on each OCMB
//
// loop through all the procs
// call mmio_dev_map() on OCMBs 0-7 and 8-15
// set VM_ADDR on each OCMB
// each pair of OCMBs has their memories interleaved with their
// 2GB config sections together and their 2GB mmio sections
// together, we will be setting VM_ADDR to point to the cfg
// section of each ocmb
// Example
// 0GB ocmb0 cfg
// 2GB ocmb1 cfg
// 4GB ocmb0 mmio
// 6GB ocmb1 mmio
TARGETING::TargetHandleList l_procTargetList;

getAllChips(l_procTargetList, TARGETING::TYPE_PROC);
for (auto & l_procTarget: l_procTargetList)
{
// map all 16 OCMBs, 8 OCMBs (32GB) at a time
uint64_t *l_virtAddr[2] = {nullptr};
uint32_t l_procNum =
l_procTarget->getAttr<TARGETING::ATTR_POSITION>();
uint64_t l_realAddr =
l_ocmbBaseAddr + (l_procNum * 2 * THIRTYTWO_GB);

l_virtAddr[0] = static_cast<uint64_t *>
(mmio_dev_map(reinterpret_cast<void *>(l_realAddr),
THIRTYTWO_GB));
l_realAddr += THIRTYTWO_GB;
l_virtAddr[1] = static_cast<uint64_t *>
// Map the device with a kernal call, each device, the MC, is 32 GB
uint64_t l_virtAddr = reinterpret_cast<uint64_t>
(mmio_dev_map(reinterpret_cast<void *>(l_realAddr),
THIRTYTWO_GB));

TRACFCOMP ( g_trac_mmio, "MC%.02x (0x%.08X) MMIO BAR PHYSICAL ADDR = 0x%lx VIRTUAL ADDR = 0x%lx" ,
l_mcChipUnit ? 0x23 : 0x01, TARGETING::get_huid(l_mcTarget),
l_realAddr, l_virtAddr);

// set VM_ADDR on each OCMB
TARGETING::TargetHandleList l_ocmbTargetList;
l_ocmbTargetList.clear();
getChildAffinityTargets(l_ocmbTargetList, l_procTarget,
TARGETING::CLASS_CHIP, TARGETING::TYPE_OCMB_CHIP);
for (auto & l_ocmbTarget: l_ocmbTargetList)
TARGETING::TargetHandleList l_omiTargetList;
getChildChiplets(l_omiTargetList, l_mcTarget, TARGETING::TYPE_OMI);

for (auto & l_omiTarget: l_omiTargetList)
{
uint64_t l_ocmbVmAddr = 0;
uint32_t l_ocmbNum =
l_ocmbTarget->getAttr<TARGETING::ATTR_POSITION>();
// ATTR_CHIP_UNIT is relative to other OMI under this PROC
uint32_t l_omiChipUnit =
l_omiTarget->getAttr<TARGETING::ATTR_CHIP_UNIT>();

// Get the OMI position relative to other OMIs under its parent MC chiplet
uint32_t l_omiPosRelativeToMc = l_omiChipUnit % OMI_PER_MC;

// Calculate what we think the real address for this OCMB should be. This should
// match what the ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET attribute is set to.

// Each Memory Controller Channel (MCC) uses 8 GB of Memory Mapped IO, 4 GB for each of its child OCMBs.
// Each OCMB has 2 MMIO distinct spaces that get mapped. The CONFIG space, and the MMIO space. The CONFIG
// Space is always before the MMIO space we will treat that as the BAR for the OCMB target. These
// paired OCMB spaces get interleaved as follows :
// ocmb | BAR ATTRIBUTE | Type | Base reg - end addr | size | sub-ch
// +-----+--------------------+------+-----------------------------------------+------+-------
// ocmb0 | 0x0006030200000000 | cnfg | 0x0006030200000000 - 0x000603027FFFFFFF | 2GB | 0
// ocmb1 | 0x0006030280000000 | cnfg | 0x0006030280000000 - 0x00060302FFFFFFFF | 2GB | 1
// ocmb0 | N/A | mmio | 0x0006030300000000 - 0x000603037FFFFFFF | 2GB | 0
// ocmb1 | N/A | mmio | 0x0006030380000000 - 0x00060303FFFFFFFF | 2GB | 1
// +-----+--------------------+------+-----------------------------------------+------+-------

// Calculate CNFG space BAR to write to OCMB attribute
uint64_t l_currentOmiOffset = (( l_omiPosRelativeToMc / 2) * 8 * GIGABYTE) +
(( l_omiPosRelativeToMc % 2) * 2 * GIGABYTE);

// Calculated real address for this OMI is (BAR from MC attribute) + (currentOmiOffset)
uint64_t l_calulatedRealAddr = l_omiBaseAddr + l_currentOmiOffset;

// Grab bar value from attribute to verify it matches our calculations
auto l_omiBarAttrVal = l_omiTarget->getAttr<TARGETING::ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET>();

if(l_omiBarAttrVal != l_calulatedRealAddr)
{
TRACFCOMP(g_trac_mmio,
"Discrepancy found between calculated OMI MMIO bar offset and what we found in ATTR_OMI_INBAND_BAR_BASE_ADDR_OFFSET");
TRACFCOMP(g_trac_mmio, "Calculated Offset: 0x%lx, Attribute Value : 0x%lx", l_calulatedRealAddr, l_omiBarAttrVal);

/*@
* @errortype ERRORLOG::ERRL_SEV_UNRECOVERABLE
* @moduleid MMIO::MOD_MMIO_SETUP
* @reasoncode MMIO::RC_BAR_OFFSET_MISMATCH
* @userdata1 Calculated Bar Offset
* @userdata2 Bar offset from attribute
* @devdesc mmioSetup> Mismatch between calculated map value
* and what is in attribute xml
* @custdesc Unexpected memory subsystem firmware error.
*/
l_err = new ERRORLOG::ErrlEntry(
ERRORLOG::ERRL_SEV_UNRECOVERABLE,
MMIO::MOD_MMIO_SETUP,
MMIO::RC_BAR_OFFSET_MISMATCH,
l_calulatedRealAddr,
l_omiBarAttrVal,
ERRORLOG::ErrlEntry::ADD_SW_CALLOUT);
l_err->collectTrace( MMIO_COMP_NAME);
ERRORLOG::ErrlUserDetailsTarget(l_omiTarget).addToLog(l_err);

break;
}

// OCMBs 0-7 in first map, 8-15 in second map
l_ocmbVmAddr =
reinterpret_cast<uint64_t>(l_virtAddr[l_ocmbNum / 8]);

// Each pair of OCMBs uses 8GB of interleaved memory,
// the second OCMB's memory starts 2GB after the first's
l_ocmbVmAddr += ((l_ocmbNum / 2) * 8 * GIGABYTE) +
((l_ocmbNum % 2) * 2 * GIGABYTE);
uint64_t l_currentOmiVirtAddr = l_virtAddr + l_currentOmiOffset;

l_ocmbTarget->
setAttr<TARGETING::ATTR_MMIO_VM_ADDR>(l_ocmbVmAddr);
// set VM_ADDR the associated OCMB
TARGETING::TargetHandleList l_ocmbTargetList;
getChildAffinityTargets(l_ocmbTargetList, l_omiTarget,
TARGETING::CLASS_CHIP, TARGETING::TYPE_OCMB_CHIP);

assert(l_ocmbTargetList.size() == 1 , "OCMB chips list found for a given OMI != 1 as expected");

TRACFCOMP(g_trac_mmio, "Setting HUID 0x%.08X MMIO vm addr to be 0x%lx , real address is 0x%lx", TARGETING::get_huid(l_ocmbTargetList[0]),
l_currentOmiVirtAddr, l_calulatedRealAddr | MMIO_BASE );

l_ocmbTargetList[0]->setAttr<TARGETING::ATTR_MMIO_VM_ADDR>(l_currentOmiVirtAddr);
}
}
} while(0);
Expand Down Expand Up @@ -216,7 +210,7 @@ errlHndl_t ocmbMmioPerformOp(DeviceFW::OperationType i_opType,

TRACDCOMP(g_trac_mmio, ENTER_MRK"ocmbMmioPerformOp");
TRACDCOMP(g_trac_mmio, INFO_MRK"op=%d, target=0x%.8X",
i_opType, i_target);
i_opType, TARGETING::get_huid(i_target));
TRACDCOMP(g_trac_mmio, INFO_MRK"buffer=%p, length=%d, accessType=%ld",
io_buffer, io_buflen, i_accessType);
TRACDCOMP(g_trac_mmio, INFO_MRK"offset=0x%lX, accessLimit=%ld",
Expand All @@ -226,6 +220,8 @@ errlHndl_t ocmbMmioPerformOp(DeviceFW::OperationType i_opType,
{
uint64_t l_addr = i_target->getAttr<TARGETING::ATTR_MMIO_VM_ADDR>();

TRACDCOMP(g_trac_mmio, INFO_MRK"MMIO Op l_addr=0x%lX ", l_addr);

if (l_addr == 0)
{
TRACFCOMP(g_trac_mmio, ERR_MRK
Expand Down
12 changes: 11 additions & 1 deletion src/usr/targeting/common/xmltohb/simics_AXONE.system.xml
View file
Expand Up @@ -5,7 +5,7 @@
<!-- -->
<!-- OpenPOWER HostBoot Project -->
<!-- -->
<!-- Contributors Listed Below - COPYRIGHT 2018 -->
<!-- Contributors Listed Below - COPYRIGHT 2018,2019 -->
<!-- [+] International Business Machines Corp. -->
<!-- -->
<!-- -->
Expand Down Expand Up @@ -6512,6 +6512,11 @@
<id>TYPE</id>
<default>MC</default>
</attribute>
<attribute>
<id>OMI_INBAND_BAR_BASE_ADDR_OFFSET</id>
<!-- 3TB + 16GB -->
<default>0x30400000000</default>
</attribute>
</targetInstance>

<targetInstance>
Expand Down Expand Up @@ -6574,6 +6579,11 @@
<id>TYPE</id>
<default>MC</default>
</attribute>
<attribute>
<id>OMI_INBAND_BAR_BASE_ADDR_OFFSET</id>
<!-- 3TB + 48GB -->
<default>0x30C00000000</default>
</attribute>
</targetInstance>

<!-- ===================================================================== -->
Expand Down

0 comments on commit e583424

Please sign in to comment.