Force checkstops for unhandled machine checks

Default MSR[ME]=0 during initial boot for bootloader and
 hostboot kernel
Once the xscom address range has been mapped in, enable the
 machine check handler to force a checkstop and set MSR[ME]=1
 to allow regular machine check handling

CQ: SW401402
Change-Id: I104e39465e61b3b19d5c073e71271102711ae54f
Reviewed-on: http://ralgit01.raleigh.ibm.com/gerrit1/47179
Reviewed-by: Christian R. Geddes <crgeddes@us.ibm.com>
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: William G. Hoffa <wghoffa@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

src/bootloader/bl_start.S

@@ -5,7 +5,7 @@
 #
 # OpenPOWER HostBoot Project
 #
-# Contributors Listed Below - COPYRIGHT 2015,2017
+# Contributors Listed Below - COPYRIGHT 2015,2018
 # [+] Google Inc.
 # [+] International Business Machines Corp.
 #
@@ -74,11 +74,10 @@ _start:
     ;// Set thread priority high.
     or 2,2,2
 
-    ;// Clear MSR[TA] (bit 1) and enable MSR[ME] (bit 51).
+    ;// Clear MSR[TA] (bit 1)
     mfmsr r2
     rldicl r2,r2,1,1    ;// Clear bit 1 - result [1-63,0]
     rotrdi r2,r2,1      ;// Rotate right 1 - result [0,63]
-    ori r2,r2,4096      ;// Set bit 51
     ;// Set up SRR0 / SRR1 to enable new MSR.
     mtsrr1 r2
     li r2, _start_postmsr@l

src/build/tools/listdeps.pl

@@ -223,6 +223,7 @@
     "libsecureboot_trusted.so"  => '1',
     "libsecureboot_base.so"     => '1',
     "libscom.so"                => '1',
+    "libxscom.so"               => '1',
 };
 
 # A list of the dependent libraries in each istep.

src/include/kernel/machchk.H

@@ -5,7 +5,9 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* COPYRIGHT International Business Machines Corp. 2013,2014              */
+/* Contributors Listed Below - COPYRIGHT 2013,2018                        */
+/* [+] International Business Machines Corp.                              */
+/*                                                                        */
 /*                                                                        */
 /* Licensed under the Apache License, Version 2.0 (the "License");        */
 /* you may not use this file except in compliance with the License.       */
@@ -43,6 +45,34 @@ namespace Kernel
          *  @return bool - True if MC successfully handled, false otherwise.
          */
         bool handleSLB(task_t* t);
+
+        /**
+         * Constants to define the FIR bit to use to force a checkstop
+         *  for an unhandled machine check.
+         */
+#ifdef CONFIG_P9_SYSTEM
+        constexpr uint64_t MCHK_XSTOP_FIR_SCOM_ADDR = 0x05012000;
+        constexpr uint64_t MCHK_XSTOP_FIR_VALUE = 0x0000000100000000ull;//31
+#endif
+
+        /** @fn setCheckstopData
+         *  @brief Tells the kernel how to force a checkstop for unrecoverable
+         *         machine checks
+         *  @param[in] i_xstopAddr - XSCOM MMIO address of FIR to write
+         *  @param[in] i_xstopData - Data to write into FIR to trigger xstop
+         *
+         *  @return none
+         */
+        void setCheckstopData(uint64_t i_xstopAddr,
+                              uint64_t i_xstopData);
+
+        /** @fn forceCheckstop
+         *  @brief Force a checkstop if we know how in order to get better
+         *         error isolation for cache/memory UEs
+         *
+         *  @return none
+         */
+        void forceCheckstop();
     }
 }
 

src/include/kernel/syscalls.H

@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2010,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2010,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -123,6 +123,9 @@ namespace Systemcalls
             /** critassert() */
         MISC_CRITASSERT,
 
+            /** set_mchk_data() */
+        MISC_SETMCHKDATA,
+
 	SYSCALL_MAX
     };
 

src/include/sys/misc.h

@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2011,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2011,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -248,6 +248,16 @@ int cpu_all_winkle();
  */
 void cpu_crit_assert(uint64_t i_failAddr);
 
+/** @fn set_mchk_data
+ *  @brief Tells the kernel how to force a checkstop for unrecoverable
+ *         machine checks
+ *  @param[in] i_xstopAddr - XSCOM MMIO address of FIR to write
+ *  @param[in] i_xstopData - Data to write into FIR to trigger xstop
+ *
+ *  @return none
+ */
+void set_mchk_data(uint64_t i_xstopAddr, uint64_t i_xstopData);
+
 #ifdef __cplusplus
 }
 #endif

src/include/usr/xscom/xscomif.H

@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2017                             */
+/* Contributors Listed Below - COPYRIGHT 2017,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -34,6 +34,15 @@ namespace XSCOM
  */
 uint64_t get_master_bar( void );
 
+/**
+ * @brief Generate a fully-qualified MMIO address for a physical scom
+ *        address, relative to the given processor target
+ *  @param[in] i_proc - Processor
+ *  @param[in] i_scomAddr - Physical scom address to convert
+ *  @return uint64_t - MMIO address
+ */
+uint64_t generate_mmio_addr( TARGETING::Target* i_proc,
+                             uint64_t i_scomAddr );
 
 };  // namespace XSCOM
 

src/kernel/bltohbdatamgr.C

@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2017                             */
+/* Contributors Listed Below - COPYRIGHT 2017,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -145,7 +145,6 @@ printk("Version=%lX\n",i_data.version);
     //       were set correctly, instead use BLTOHB_SECURE_OVERRIDES version.
     if( iv_data.version >= Bootloader::BLTOHB_SECURE_OVERRIDES )
     {
-printk("lpc=%lX, xscom=%lX\n", i_data.lpcBAR, i_data.xscomBAR );
         kassert(i_data.lpcBAR>0);
         kassert(i_data.xscomBAR>0);
         iv_data.lpcBAR = i_data.lpcBAR;
@@ -158,9 +157,10 @@ printk("lpc=%lX, xscom=%lX\n", i_data.lpcBAR, i_data.xscomBAR );
         iv_data.xscomBAR = MMIO_GROUP0_CHIP0_XSCOM_BASE_ADDR;
 
     }
+    printk("lpc=%lX, xscom=%lX\n", i_data.lpcBAR, i_data.xscomBAR );
 
-printk("lpc=%lX, xscom=%lX, iv_data=%p\n", iv_data.lpcBAR, iv_data.xscomBAR,
-       static_cast<void *>(&iv_data) );
+    printk("iv_lpc=%lX, iv_xscom=%lX, iv_data=%p\n",
+            iv_data.lpcBAR, iv_data.xscomBAR, static_cast<void *>(&iv_data) );
 
     // Check if bootloader advertised the size of the structure it saw;
     // otherwise use the default padded size

src/kernel/exception.C

@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2010,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2010,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -328,6 +328,7 @@ void kernel_execute_machine_check()
                t->tid, getPIR(),
                getSRR0(), getSRR1(), getDSISR(), getDAR());
         MAGIC_INSTRUCTION(MAGIC_BREAK_ON_ERROR);
+        Kernel::MachineCheck::forceCheckstop();
         kassert(false);
     }
 
@@ -388,6 +389,7 @@ void kernel_execute_machine_check()
                 t->tid, getPIR(),
                 getSRR0(), getSRR1(), getDSISR(), getDAR());
         MAGIC_INSTRUCTION(MAGIC_BREAK_ON_ERROR);
+        Kernel::MachineCheck::forceCheckstop();
         TaskManager::endTask(t, NULL, TASK_STATUS_CRASHED);
     }
 }
@@ -425,3 +427,4 @@ void kernel_execute_unhandled_exception()
     termWriteSRC(TI_UNHANDLED_EX, KERNEL::RC_UNHANDLED_EX, exception);
     terminateExecuteTI();
 }
+

src/kernel/machchk.C

@@ -27,12 +27,20 @@
 #include <kernel/vmmmgr.H>
 #include <sys/mmio.h>
 #include <arch/memorymap.H>
+#include <arch/ppc.H>
 
 namespace Kernel
 {
 namespace MachineCheck
 {
 
+//Keep track of the MMIO address that we can use to force a checkstop
+static uint64_t* g_xstopRegPtr = nullptr;
+
+//Keep track of the data to write into the xstop reg
+static uint64_t g_xstopRegValue = 0;
+
+
 bool handleLoadUE(task_t* t)
 {
     bool handled = false;
@@ -132,5 +140,42 @@ bool handleSLB(task_t* t)
 }
 
 
+/**
+ *  @brief Tells the kernel how to force a checkstop for unrecoverable
+ *         machine checks
+ */
+void setCheckstopData(uint64_t i_xstopAddr, uint64_t i_xstopData)
+{
+    g_xstopRegPtr = reinterpret_cast<uint64_t*>(i_xstopAddr
+                                                |VmmManager::FORCE_PHYS_ADDR);
+    g_xstopRegValue = i_xstopData;
+    printk( "Set MchChk Xstop: %p=%.16lX\n", g_xstopRegPtr, g_xstopRegValue );
+
+    // Now that the machine check handler can do the xscom we
+    //  can set MSR[ME]=1 to enable the regular machine check
+    //  handling
+    uint64_t l_msr = getMSR();
+    l_msr |= 0x0000000000001000; //set bit 51
+    setMSR(l_msr);
+}
+
+/**
+ *  @brief Force a checkstop if we know how in order to get better
+ *         error isolation for cache/memory UEs
+ */
+void forceCheckstop()
+{
+    if( g_xstopRegPtr != nullptr )
+    {
+        printk( "Forcing a xstop with %p = %.16lX\n",
+                g_xstopRegPtr, g_xstopRegValue );
+        *g_xstopRegPtr = g_xstopRegValue;
+    }
+    else
+    {
+        printk( "Unable to force checkstop, No xstop reg set\n" );
+    }
+}
+
 }
 }

src/kernel/start.S

@@ -33,11 +33,10 @@ _start:
     ;// Set thread priority high.
     or 2,2,2
 
-    ;// Clear MSR[TA] (bit 1) and enable MSR[ME] (bit 51).
+    ;// Clear MSR[TA] (bit 1)
     mfmsr r2
     rldicl r2,r2,1,1    ;// Clear bit 1 - result [1-63,0]
     rotrdi r2,r2,1      ;// Rotate right 1 - result [0,63]
-    ori r2,r2,4096      ;// Set bit 51
     ;// Set up SRR0 / SRR1 to enable new MSR.
     mtsrr1 r2
     li r2, _start_postmsr@l
@@ -473,7 +472,7 @@ kernel_dispatch_task:
     stdcx. r0, TASK_CPUPTR, r1  ;// the CPU pointer in the task.
 
     mfmsr r2            ;// Get current MSR
-    ori r2,r2, 0xD030   ;// Enable MSR[EE,ME,PR,IR,DR].
+    ori r2,r2, 0xC030   ;// Enable MSR[EE,PR,IR,DR].
     rldicl r2,r2,50,1   ;// Clear ...
     rotldi r2,r2,14     ;//     MSR[FP]
     ld r3, TASK_MSR_MASK(r1) ;// Load MSR mask.

src/kernel/syscall.C

@@ -45,6 +45,7 @@
 #include <kernel/doorbell.H>
 #include <sys/sync.h>
 #include <errno.h>
+#include <kernel/machchk.H>
 
 namespace KernelIpc
 {
@@ -142,6 +143,7 @@ namespace Systemcalls
     void MmExtend(task_t *t);
     void MmLinearMap(task_t *t);
     void CritAssert(task_t *t);
+    void SetMchkData(task_t *t);
 
 
     syscall syscalls[] =
@@ -185,6 +187,7 @@ namespace Systemcalls
         &MmExtend,        // MM_EXTEND
         &MmLinearMap,     // MM_LINEAR_MAP
         &CritAssert,  // MISC_CRITASSERT
+        &SetMchkData,  // MISC_SETMCHKDATA
 
         };
 };
@@ -988,6 +991,18 @@ namespace Systemcalls
         CpuManager::critAssert(i_failAddr);
     }
 
+    /**
+     *  @brief Tells the kernel how to force a checkstop for unrecoverable
+     *         machine checks
+     * @param[in] t: the task calling the critical assert
+     */
+    void SetMchkData(task_t* t)
+    {
+        uint64_t i_xstopAddr = (uint64_t)(TASK_GETARG0(t));
+        uint64_t i_xstopData = (uint64_t)(TASK_GETARG1(t));
+
+        Kernel::MachineCheck::setCheckstopData(i_xstopAddr,i_xstopData);
+    }
 
 };
 

src/lib/syscall_misc.C

@@ -5,7 +5,7 @@
 /*                                                                        */
 /* OpenPOWER HostBoot Project                                             */
 /*                                                                        */
-/* Contributors Listed Below - COPYRIGHT 2011,2017                        */
+/* Contributors Listed Below - COPYRIGHT 2011,2018                        */
 /* [+] International Business Machines Corp.                              */
 /*                                                                        */
 /*                                                                        */
@@ -137,3 +137,11 @@ void cpu_crit_assert(uint64_t i_failAddr)
 {
    _syscall1(MISC_CRITASSERT, reinterpret_cast<void*>(i_failAddr));
 }
+
+
+void set_mchk_data(uint64_t i_xstopAddr, uint64_t i_xstopData)
+{
+    _syscall2(MISC_SETMCHKDATA,
+              reinterpret_cast<void*>(i_xstopAddr),
+              reinterpret_cast<void*>(i_xstopData));
+}