Baremetal: Be more patient with IPMI and BMC

Before we called 'power status; power on' in a loop which made the
IPMI/BMCs not behave well.  Also the total time we would wait (2.5
seconds wasn't always enough).  So make sure power on/off is only called
once and wait up to 10 seconds for the power state change to go into
effect.

This patch has been tested on real baremetal using
https://wiki.openstack.org/wiki/TripleO/TripleOCloud

This bug is also linked to ironic so the change will be made there as
well.

Change-Id: I5a4d7c84ebdf9c1f7d8d0570dbc31764c31f1fc6
Closes-Bug: #1234479

etc/nova/nova.conf.sample

@@ -2968,7 +2968,7 @@
 
 # maximal number of retries for IPMI operations (integer
 # value)
-#ipmi_power_retry=5
+#ipmi_power_retry=10
 
 
 #

nova/tests/virt/baremetal/test_ipmi.py

@@ -159,10 +159,8 @@ def test_power_on_max_retries(self):
         self.ipmi._exec_ipmitool("power on").AndReturn([])
         self.ipmi._exec_ipmitool("power status").AndReturn(
                 ["Chassis Power is off\n"])
-        self.ipmi._exec_ipmitool("power on").AndReturn([])
         self.ipmi._exec_ipmitool("power status").AndReturn(
                 ["Chassis Power is off\n"])
-        self.ipmi._exec_ipmitool("power on").AndReturn([])
         self.ipmi._exec_ipmitool("power status").AndReturn(
                 ["Chassis Power is off\n"])
         self.mox.ReplayAll()

nova/virt/baremetal/ipmi.py

@@ -47,7 +47,7 @@
                default=paths.state_path_def('baremetal/console'),
                help='path to directory stores pidfiles of baremetal_terminal'),
     cfg.IntOpt('ipmi_power_retry',
-               default=5,
+               default=10,
                help='maximal number of retries for IPMI operations'),
     ]
 
@@ -154,13 +154,16 @@ def _wait_for_power_on():
                 raise loopingcall.LoopingCallDone()
             try:
                 self.retries += 1
-                self._exec_ipmitool("power on")
+                if not self.power_on_called:
+                    self._exec_ipmitool("power on")
+                    self.power_on_called = True
             except Exception:
                 LOG.exception(_("IPMI power on failed"))
 
         self.retries = 0
+        self.power_on_called = False
         timer = loopingcall.FixedIntervalLoopingCall(_wait_for_power_on)
-        timer.start(interval=0.5).wait()
+        timer.start(interval=1.0).wait()
 
     def _power_off(self):
         """Turn the power to this node OFF."""
@@ -178,13 +181,16 @@ def _wait_for_power_off():
                 raise loopingcall.LoopingCallDone()
             try:
                 self.retries += 1
-                self._exec_ipmitool("power off")
+                if not self.power_off_called:
+                    self._exec_ipmitool("power off")
+                    self.power_off_called = True
             except Exception:
                 LOG.exception(_("IPMI power off failed"))
 
         self.retries = 0
+        self.power_off_called = False
         timer = loopingcall.FixedIntervalLoopingCall(_wait_for_power_off)
-        timer.start(interval=0.5).wait()
+        timer.start(interval=1.0).wait()
 
     def _set_pxe_for_next_boot(self):
         try: