Shutdown: Move attribute sync to shutdown handler

Moved synching attributes into an attribute resource provider shutdown path
to avoid race conditions along the shutdown path where a normal error path
coupled with parallel shutdown leads to FSP attribute sync errors.

Change-Id: Ibdb828f8132da1e251f880ef0e7d4fea2e4619a3
CQ: SW443737
Reviewed-on: http://rchgit01.rchland.ibm.com/gerrit1/66431
Tested-by: Jenkins Server <pfd-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP Build CI <op-jenkins+hostboot@us.ibm.com>
Tested-by: Jenkins OP HW <op-hw-jenkins+hostboot@us.ibm.com>
Tested-by: FSP CI Jenkins <fsp-CI-jenkins+hostboot@us.ibm.com>
Reviewed-by: Daniel M. Crowell <dcrowell@us.ibm.com>

src/include/usr/sbeio/sbeioif.H

@@ -101,12 +101,17 @@ namespace SBEIO
     /**
      * @brief Sends a PSU chipOp to quiesce the SBE
      *
-     * @param[in]  i_target  Target with SBE to quiesce
+     * @param[in]  i_pProc  Processor target with SBE to quiesce.
+     *     Throws error if requested target is nullptr or does not
+     *     refer to a processor.
+     *
+     * @note Sets the requested processor's ASSUME_SBE_QUIESCED attribute
+     *   to true to inhibit future quiesce sensitive operations.
      *
      * @return errlHndl_t Error log handle on failure.
      *
      */
-    errlHndl_t sendPsuQuiesceSbe(TARGETING::Target * i_target);
+    errlHndl_t sendPsuQuiesceSbe(TARGETING::Target* i_pProc);
 
      /**
      * @brief Get the capabilities of the SBE via PSU

src/include/usr/sbeio/sbeioreasoncodes.H

@@ -57,6 +57,7 @@ enum sbeioModuleId
     SBEIO_GET_FFDC_HANDLER              = 0x0C,
     SBEIO_GET_SBE_RC                    = 0x0D,
     SBEIO_HANDLE_VITAL_ATTN             = 0x0E,
+    SBEIO_SEND_PSU_QUIESCE_SBE          = 0x0F,
 };
 
 /**
@@ -75,6 +76,7 @@ enum sbeioReasonCode
     SBEIO_PSU_NOT_READY                = SBEIO_COMP_ID | 0x03,
     SBEIO_PSU_FFDC_MISSING             = SBEIO_COMP_ID | 0x04,
     SBEIO_PSU_SEND                     = SBEIO_COMP_ID | 0x05,
+    SBEIO_PSU_INVALID_TARGET           = SBEIO_COMP_ID | 0x06,
 
     // SBE FIFO error codes
     SBEIO_FIFO_UPSTREAM_TIMEOUT        = SBEIO_COMP_ID | 0x10,

src/include/usr/targeting/attrrp.H

@@ -324,6 +324,54 @@ class AttrRP
 
 #ifndef __HOSTBOOT_RUNTIME
 
+        /**
+          * @enum msg_mm_rp_runtime_prep_t
+          *
+          * @brief Message subtypes for the MSG_MM_RP_RUNTIME_PREP message
+          */
+        enum msg_mm_rp_runtime_prep_t
+        {
+            // data[0] = MSG_MM_RP_RUNTIME_PREP_BEGIN or
+            // data[0] = MSG_MM_RP_RUNTIME_PREP_END
+            MSG_MM_RP_RUNTIME_PREP_BEGIN, ///< Force all pages to be validated
+            MSG_MM_RP_RUNTIME_PREP_END,   ///< Disable force validation
+        };
+
+        /**
+         *  @enum ATTRRP_MSG_TYPE
+         *
+         *  @brief Non-kernel message types for the attribute resource provider
+         *      message handler
+         */
+        enum ATTRRP_MSG_TYPE : uint32_t
+        {
+            // Prepare runtime for secure transition of attributes
+            MSG_MM_RP_RUNTIME_PREP        = 0x00000002,
+            // Arm service to synchronize attributes when Hostboot shuts down
+            MSG_PRIME_SHUTDOWN_ATTR_SYNC  = 0x00000003,
+            // Invoke attribute synchronization at shutdown
+            MSG_INVOKE_SHUTDOWN_ATTR_SYNC = 0x00000004,
+        };
+
+        /**
+         *  @enum RESOURCE
+         *
+         *  @brief Enumeration which indicates a specific resource (service,
+         *      condition, etc.)
+         */
+        enum RESOURCE : uint8_t
+        {
+            MAILBOX, ///< FSI mailbox service
+        };
+
+        /**
+         *  @brief Notifies the attribute resource provider that a specific
+         *      resource of interest is ready/available
+         *
+         *  @param[in] i_resource Resource that is ready/available
+         */
+        static void notifyResourceReady(RESOURCE i_resource);
+
         /**
         *  @brief Modifies the memory R/W permissions on VMM pages for a
         *         given type of AttrRP_Section. Valid types can be found
@@ -433,8 +481,9 @@ class AttrRP
          */
         AttrRP()
 #ifndef __HOSTBOOT_RUNTIME
-            : iv_msgQ(NULL), iv_sections(NULL), iv_sectionCount(0),
-              iv_isMpipl(false)
+            : iv_msgQ(nullptr), iv_attrSyncMsgQ(msg_q_create()),
+              iv_sections(nullptr), iv_sectionCount(0),
+              iv_shutdownAttrSyncPrimed(false),iv_isMpipl(false)
 #else
             : iv_isTempInstance(false), iv_isMpipl(false)
 #endif
@@ -567,16 +616,49 @@ class AttrRP
 
 #ifndef __HOSTBOOT_RUNTIME
 
+        /**
+         *  @brief Notifies the attribute resource provider that a specific
+         *      resource of interest is ready/available
+         *
+         *  @param[in] i_resource Resource that is ready/available
+         */
+        void _notifyResourceReady(RESOURCE i_resource) const;
+
+        /**
+         *  @brief Synchronizes attributes to FSP during a user space initiated
+         *      shutdown
+         *
+         *  @par Detailed Description:
+         *      During a user space initiated shutdown, the init service calls
+         *      the resource provider shutdown handler which attempts to
+         *      synchronize attributes down to the FSP.  For this to succeed,
+         *      the FSP must be available, the mailbox must be online (and have
+         *      earlier called notifyResourceReady API to arm the
+         *      synchronization), and SBE must not be quiesced (or mailbox
+         *      traffic going through the SBE FIFO in secure mode will fail)
+         */
+        void invokeShutdownAttrSync() const;
+
         /**
          *  @brief Processes daemon messages
          *
          *  @par Detailed Description:
-         *      Performs a while(1) waiting for messages from the
-         *      kernel/VMM and handles as appropriately.  Reads / writes
-         *      data from / to PNOR for the attribute sections.
+         *      Performs a while(1) waiting for messages from
+         *      kernel/VMM/user space and handles as appropriate.  Reads /
+         *      writes data from / to PNOR for the attribute sections for
+         *      kernel/VMM messages.
          */
         void msgServiceTask() const;
 
+        /**
+         *  @brief Task which processes attribute synchronization requests
+         *
+         *  @par Detailed Description:
+         *      Processes attribute synchronization requests, especially in
+         *      relation to synchronizing attributes when Hostboot terminates.
+         */
+        void attrSyncTask();
+
         /**
          *  @brief Parses the attribute section header in PNOR.
          *
@@ -656,14 +738,35 @@ class AttrRP
          */
         static void* startMsgServiceTask(void* i_pInstance);
 
+        /**
+         *  @brief Starts the attribute provider's attribute synchronization
+         *      task
+         *
+         *  @par Detailed Description:
+         *      task_create should call this static function to enter the
+         *      daemonized attribute synchronization task which handles
+         *      attribute synchronization requests
+         *
+         *  @param[in] i_pInstance The AttrRP to call attrSyncTask on.  Must not
+         *      be nullptr (or asserts);
+         */
+        static void* startAttrSyncTask(void* i_pInstance);
+
         // Message Queue for VMM requests
         msg_q_t iv_msgQ;
 
+        // Message Queue for attribute sync requests
+        msg_q_t iv_attrSyncMsgQ;
+
         // Parsed structures of the attribute sections.
         AttrRP_Section* iv_sections;
 
         // Count of attribute sections.
         size_t iv_sectionCount;
+
+        // Whether service is primed to invoke attribute synchronization when
+        // Hostboot shuts down under user space control
+        bool iv_shutdownAttrSyncPrimed;
 #else
         // Indicator that AttrRP instance is a temporary one, not the singleton
         bool iv_isTempInstance;
@@ -681,21 +784,8 @@ class AttrRP
 TARG_DECLARE_SINGLETON(TARGETING::AttrRP,theAttrRP);
 
 extern const char* ATTRRP_MSG_Q;
+extern const char* ATTRRP_ATTR_SYNC_MSG_Q;
 
-// user-defined message subtype for MSG_MM_RP_RUNTIME_PREP
-/**
-  * @enum msg_mm_rp_runtime_prep_t
-  * @brief Message type and subtypes for the MSG_MM_RP_RUNTIME_PREP message
-  */
-enum msg_mm_rp_runtime_prep_t
-{
-    MSG_MM_RP_RUNTIME_PREP = 0x2, // prepare runtime for secure transition
-                                  // of attrs
-                                  // data[0] = MSG_MM_RP_RUNTIME_PREP_BEGIN or
-                                  // data[0] = MSG_MM_RP_RUNTIME_PREP_END
-    MSG_MM_RP_RUNTIME_PREP_BEGIN,
-    MSG_MM_RP_RUNTIME_PREP_END,
-};
 
 } // End namespace TARGETING
 

src/include/usr/targeting/common/targreasoncodes.H

@@ -51,6 +51,8 @@ enum TargetingModuleId
     TARG_HANDLE_ENUM_CHECK_FAILURE  = 0x0B,
     TARG_HANDLE_RANGE_CHECK_FAILURE = 0x0C,
     TARG_EDIT_PAGE_PERMISSIONS      = 0x0D,
+    TARG_NOTIFY_RESOURCE_READY      = 0x0E,
+    TARG_ATTR_SYNC_TASK             = 0x0F,
 };
 
 enum TargetingReasonCode
@@ -80,6 +82,7 @@ enum TargetingReasonCode
     TARG_RC_ATTRIBUTE_RANGE_CHECK_FAIL      = TARG_COMP_ID | 0x17,
     TARG_RC_CONCURRENT_CODE_UPDATE_FAIL     = TARG_COMP_ID | 0x18,
     TARG_RC_ATTR_OVER_FAPI_TANK_NOT_SUPPORTED = TARG_COMP_ID | 0x19,
+    TARG_RC_UNSUPPORTED_ATTR_SYNC_MSG       = TARG_COMP_ID | 0x1A,
 };
 
 }; // End TARGETING namespace

src/usr/initservice/istepdispatcher/istepdispatcher.C

@@ -1028,21 +1028,6 @@ errlHndl_t IStepDispatcher::doIstep(uint32_t i_istep,
         {
             TRACFCOMP(g_trac_initsvc, ERR_MRK"doIstep: Istep failed, plid 0x%x",
                       err->plid());
-
-            // istep fails, sync attributes to FSP
-            if( INITSERVICE::spBaseServicesEnabled() )
-            {
-                TRACFCOMP(g_trac_initsvc, ERR_MRK"doIstep, Sync attributes to FSP");
-                errlHndl_t l_errl = TARGETING::syncAllAttributesToFsp();
-
-                if(l_errl)
-                {
-                    TRACFCOMP(g_trac_initsvc, ERR_MRK"doIstep: Attribute syncing"
-                         " failed see 0x%08X for details", l_errl->eid());
-                    l_errl->setSev(ERRORLOG::ERRL_SEV_INFORMATIONAL);
-                    errlCommit(l_errl, INITSVC_COMP_ID);
-                }
-            }
         }
 
         // Check for any attentions and invoke PRD for analysis

src/usr/isteps/istep21/call_host_start_payload.C

@@ -449,6 +449,14 @@ errlHndl_t callShutdown ( uint64_t i_masterInstance,
         // Create a task to handle the messages
         task_create(ISTEP_21::msg_handler, l_msgQ);
 
+        // Unregister the AttrRP shutdown handler which synchronizes all
+        // attributes at shutdown, as closing the SBE memory regions below will
+        // cause all DMAs (and thus attribute sync) to fail.  Intentionally
+        // ignore the response from unregister.  This API will get called on
+        // all nodes.
+        auto pAttrMsgQ = msg_q_resolve(TARGETING::ATTRRP_ATTR_SYNC_MSG_Q);
+        INITSERVICE::unregisterShutdownEvent(pAttrMsgQ);
+
         // Tell SBE to Close All Unsecure Memory Regions
         err = SBEIO::closeAllUnsecureMemRegions();
         if (err)

src/usr/mbox/mailboxsp.C

@@ -52,6 +52,7 @@
 #include <sbeio/sbeioif.H>
 #include <sys/time.h>
 #include <intr/interrupt.H>
+#include <targeting/attrrp.H>
 
 // Local functions
 namespace MBOX
@@ -222,8 +223,17 @@ errlHndl_t MailboxSp::_init()
 #ifndef CONFIG_VPO_COMPILE
     // Start the the interprocessor communications message handler
     IPC::IpcSp::init(err);
-    // call ErrlManager function - tell him that MBOX is ready!
-    ERRORLOG::ErrlManager::errlResourceReady(ERRORLOG::MBOX);
+
+    // On error VFS won't initialize the mailbox address space, opening up
+    // the chance of downstream task crashes later.
+    if(!err)
+    {
+        // call ErrlManager function - tell him that MBOX is ready!
+        ERRORLOG::ErrlManager::errlResourceReady(ERRORLOG::MBOX);
+        TARGETING::AttrRP::notifyResourceReady(
+            TARGETING::AttrRP::RESOURCE::MAILBOX);
+    }
+
 #endif
 
     return err;

src/usr/runtime/populate_hbruntime.C

@@ -3497,9 +3497,9 @@ errlHndl_t persistent_rwAttrRuntimeCheck( void )
 
     assert(l_msg != nullptr, "Bug! Message allocation failed!");
 
-    l_msg->type = TARGETING::MSG_MM_RP_RUNTIME_PREP;
+    l_msg->type = TARGETING::AttrRP::MSG_MM_RP_RUNTIME_PREP;
 
-    l_msg->data[0] = TARGETING::MSG_MM_RP_RUNTIME_PREP_BEGIN;
+    l_msg->data[0] = TARGETING::AttrRP::MSG_MM_RP_RUNTIME_PREP_BEGIN;
 
     int rc = msg_sendrecv(l_msgQ, l_msg);
 
@@ -3539,8 +3539,8 @@ errlHndl_t persistent_rwAttrRuntimeCheck( void )
             validateAllRwNvAttr( *targets );
         }
 
-        l_msg->type = TARGETING::MSG_MM_RP_RUNTIME_PREP;
-        l_msg->data[0] = TARGETING::MSG_MM_RP_RUNTIME_PREP_END;
+        l_msg->type = TARGETING::AttrRP::MSG_MM_RP_RUNTIME_PREP;
+        l_msg->data[0] = TARGETING::AttrRP::MSG_MM_RP_RUNTIME_PREP_END;
 
         int rc = msg_sendrecv(l_msgQ, l_msg);