Add support for MPIPL/FADUMP tests

This moves the existing OpTestKernel into PowerNVDump file and also
adds support for fadump/MPIPL test scenarios.

1. So we have three scenarios here after kernel crash.

Enable both fadump, kdump ==> Test fadump/MPIPL
Enable only kdump         ==> Test kdump
Disable both              ==> Test normal IPL

2. Add OPAL assert based MPIPL test scenario

This scenario uses pdbg tool to trigger OPAL assert from the BMC,
so that MPIPL boot starts after the assert finishes.

Signed-off-by: Srikanth Aithal <sraithal@linux.vnet.ibm.com>
Signed-off-by: Pridhiviraj Paidipeddi <ppaidipe@linux.vnet.ibm.com>

OpTestConfiguration.py

@@ -309,6 +309,8 @@ def get_parser():
                             help="pflash to copy to BMC (if needed)")
     imagegroup.add_argument("--pupdate",
                             help="pupdate to flash PNOR for Supermicro systems")
+    imagegroup.add_argument("--pdbg",
+                            help="pdbg binary to be executed on BMC")
 
     stbgroup = parser.add_argument_group('STB', 'Secure and Trusted boot parameters')
     stbgroup.add_argument("--un-signed-pnor", help="Unsigned or improperly signed PNOR")

common/Exceptions.py

@@ -166,6 +166,15 @@ def __str__(self):
         return "Kernel Kdump (machine in state '{}'): {}".format(
             self.state, self.log)
 
+class KernelFADUMP(Exception):
+    '''
+    We observe a kernel crash and follwed by MPIPL boot to dump opalcore/vmcore for debug.
+    '''
+    def __init__(self, state, log):
+        self.log = log
+        self.state = state
+    def __str__(self):
+        return "Kernel FADUMP/MPIPL (machine in state '{}'): {}".format(self.state, self.log)
 
 class KernelCrashUnknown(Exception):
     '''

common/OPexpect.py

@@ -36,9 +36,10 @@
 
 import pexpect
 from Exceptions import *
-import OpTestSystem
+
 
 class spawn(pexpect.spawn):
+
     def __init__(self, command, args=[], maxread=8000,
                  searchwindowsize=None, logfile=None, cwd=None, env=None,
                  ignore_sighup=False, echo=True, preexec_fn=None,
@@ -73,17 +74,17 @@ def expect(self, pattern, timeout=-1, searchwindowsize=-1, async=False):
                        "OPAL exiting with locks held",
                        "LOCK ERROR: Releasing lock we don't hold",
                        "OPAL: Reboot requested due to Platform error."
-        ]
+                       ]
 
-        patterns = list(op_patterns) # we want a *copy*
+        patterns = list(op_patterns)  # we want a *copy*
         if isinstance(pattern, list):
             patterns = patterns + pattern
         else:
             patterns.append(pattern)
 
-        r = super(spawn,self).expect(patterns,
-                                     timeout=timeout,
-                                     searchwindowsize=searchwindowsize)
+        r = super(spawn, self).expect(patterns,
+                                      timeout=timeout,
+                                      searchwindowsize=searchwindowsize)
 
         if r in [pexpect.EOF, pexpect.TIMEOUT]:
             return r
@@ -93,53 +94,59 @@ def expect(self, pattern, timeout=-1, searchwindowsize=-1, async=False):
 
         state = None
 
-        if r in [1,2,3,4,5,6,7,8,9,10,11,12,13]:
+        if r in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]:
             # We set the system state to UNKNOWN_BAD as we want to have a path
             # to recover and run the next test, which is going to be to IPL
             # the box again.
             # We do this via a callback rather than any other method as that's
             # just a *lot* easier with current code structure
             if self.failure_callback:
                 state = self.failure_callback(self.failure_callback_data)
-        if r in [1,2,3,4,5,6,7,8]:
+        if r in [1, 2, 3, 4, 5, 6, 7, 8]:
             log = str(self.after)
             l = 0
 
-            while l != 7:
-                l = super(spawn,self).expect(["INFO: rcu_sched self-detected stall on CPU",
-                                              "Watchdog .* Hard LOCKUP",
-                                              "Sending IPI to other CPUs",
-                                              ":mon>",
-                                              "Rebooting in \d+ seconds",
-                                              "Kernel panic - not syncing: Fatal exception",
-                                              "Kernel panic - not syncing: Hard LOCKUP", pexpect.TIMEOUT],
-                                             timeout=15)
+            while l != 8:
+                l = super(spawn, self).expect(["INFO: rcu_sched self-detected stall on CPU",
+                                               "Watchdog .* Hard LOCKUP",
+                                               "Sending IPI to other CPUs",
+                                               ":mon>",
+                                               "Rebooting in \d+ seconds",
+                                               "Kernel panic - not syncing: Fatal exception",
+                                               "Kernel panic - not syncing: Hard LOCKUP",
+                                               "opal_cec_reboot2", pexpect.TIMEOUT],
+                                              timeout=15)
                 log = log + str(self.before) + str(self.after)
-                if l in [2,3,4]:
-                    # We know we have the end of the error message, so let's stop here.
+                if l in [2, 3, 4, 7]:
+                    # We know we have the end of the error message, so let's
+                    # stop here.
                     break
 
             if r == 1 or r == 8:
                 raise KernelSoftLockup(state, log)
             if r == 2:
                 raise KernelBug(state, log)
-            if r == 3:
-                raise KernelPanic(state, log)
             if r == 4 or r == 6 or r == 7:
                 raise KernelHardLockup(state, log)
-            if r == 5 and l == 2:
+            if r == 3 and l == 2:
                 raise KernelKdump(state, log)
+            if r == 3 and l == 7:
+                raise KernelFADUMP(state, log)
+            if r == 3:
+                raise KernelPanic(state, log)
             if r == 5:
                 raise KernelOOPS(state, log)
-            if l == 7:
+            if l == 8:
                 raise KernelCrashUnknown(state, log)
 
-        if r in [9,10,11,12]:
+        if r in [9, 10, 11, 12]:
             l = 0
             log = self.after
-            l = super(spawn,self).expect("boot_entry.*\r\n", timeout=10)
+            l = super(spawn, self).expect(["boot_entry.*\r\n",
+                                           "Initiated MPIPL", pexpect.TIMEOUT],
+                                          timeout=10)
             log = log + self.before + self.after
-            if r in [9,11,12]:
+            if r in [9, 11, 12]:
                 raise SkibootAssert(state, log)
             if r == 10:
                 raise SkibootException(state, log)
@@ -149,15 +156,16 @@ def expect(self, pattern, timeout=-1, searchwindowsize=-1, async=False):
             # Let's attempt to capture Hostboot output
             log = self.before + self.after
             try:
-                l = super(spawn,self).expect("================================================",
-                                             timeout=120)
+                l = super(spawn, self).expect("================================================",
+                                              timeout=120)
                 log = log + self.before + self.after
-                l = super(spawn,self).expect("System checkstop occurred during runtime on previous boot", timeout=30)
+                l = super(spawn, self).expect(
+                    "System checkstop occurred during runtime on previous boot", timeout=30)
                 log = log + self.before + self.after
-                l = super(spawn,self).expect("================================================",
-                                             timeout=60)
+                l = super(spawn, self).expect("================================================",
+                                              timeout=60)
                 log = log + self.before + self.after
-                l = super(spawn,self).expect("ISTEP", timeout=20)
+                l = super(spawn, self).expect("ISTEP", timeout=20)
                 log = log + self.before + self.after
             except pexpect.TIMEOUT as t:
                 pass

common/OpTestConstants.py

@@ -105,6 +105,8 @@ class OpTestConstants():
     CLEAR_GARD_CMD = '/gard clear all'
     LIST_GARD_CMD = '/gard list'
     OPAL_MSG_LOG = "cat /sys/firmware/opal/msglog"
+    PROC_CMDLINE = "cat /proc/cmdline"
+    OPAL_DUMP_NODE = "/proc/device-tree/ibm,opal/dump/"
     NVRAM_PRINT_CFG = "nvram --print-config"
     NVRAM_UPDATE_CONFIG_TEST_DATA = "nvram --update-config test-name=test-value"
     NVRAM_TEST_DATA = "test-name=test-value"

op-test

@@ -96,7 +96,7 @@ from testcases import OpalErrorLog
 from testcases import LightPathDiagnostics
 from testcases import DPO
 from testcases import EPOW
-from testcases import OpTestKernel
+from testcases import PowerNVDump
 from testcases import fspresetReload
 from testcases import OpTestPrdDaemon
 from testcases import OpalUtils