BZ#1184630 - deployments run forever

https://bugzilla.redhat.com/show_bug.cgi?id=1184630

Implement timeouts for actions which do polling. Note that if DynFlow
executor stops working, the timeout checking will stop working too,
and the deployment will still run forever. This patch mitigates issues
like Puppet not being triggered, provisioning failures etc., but not
DynFlow failures.

app/lib/actions/staypuft/host/report_wait.rb

@@ -16,6 +16,9 @@ module Host
 
       class ReportWait < Actions::Base
 
+        TIMEOUT = 9000
+
+        middleware.use Actions::Staypuft::Middleware::Timeout
         middleware.use Actions::Staypuft::Middleware::AsCurrentUser
         include Dynflow::Action::Polling
 

app/lib/actions/staypuft/host/wait_until_provisioned.rb

@@ -16,7 +16,13 @@ module Host
 
       class WaitUntilProvisioned < Actions::Base
 
+        TIMEOUT = 7200
+
         middleware.use Actions::Staypuft::Middleware::AsCurrentUser
+        # Using the Timeout middleware to make sure input[:timeout] is
+        # set. The actual timeout check is performed separately for
+        # the WaitUntilProvisioned because it's not a polling action.
+        middleware.use Actions::Staypuft::Middleware::Timeout
 
         def plan(host)
           plan_self host_id: host.id
@@ -26,11 +32,22 @@ def run(event = nil)
           case event
           when nil
             suspend do |suspended_action|
+              # schedule timeout
+              world.clock.ping suspended_action, input[:timeout], "timeout"
+
+              # wake up when provisioning is finished
               Rails.cache.write(
                   ::Staypuft::Concerns::HostOrchestrationBuildHook.cache_id(input[:host_id]),
                   { execution_plan_id: suspended_action.execution_plan_id,
                     step_id:           suspended_action.step_id })
             end
+          when "timeout"
+            # clear timeout_start so that the action can be resumed/skipped
+            output[:timeout_start] = nil
+            fail(::Staypuft::Exception,
+                 "You've reached the timeout set for this action. If the " +
+                 "action is still ongoing, you can click on the " +
+                 "\"Resume Deployment\" button to continue.")
           when Hash
             output[:installed_at] = event.fetch(:installed_at).to_s
           when Dynflow::Action::Skip

app/lib/actions/staypuft/host/wait_until_ready.rb

@@ -17,7 +17,9 @@ module Host
       class WaitUntilReady < Actions::Base
 
         STARTUP_GRACE_PERIOD = 60
+        TIMEOUT = 1800
 
+        middleware.use Actions::Staypuft::Middleware::Timeout
         middleware.use Actions::Staypuft::Middleware::AsCurrentUser
         include Dynflow::Action::Polling
 

app/lib/actions/staypuft/middleware/timeout.rb

@@ -0,0 +1,39 @@
+module Actions
+  module Staypuft
+    module Middleware
+      class Timeout < Dynflow::Middleware
+
+        def plan(*args)
+          pass(*args).tap do
+            action.input[:timeout] ||= action.class::TIMEOUT
+          end
+        end
+
+        def run(*args)
+          assert_timeout_not_reached
+          pass(*args)
+        end
+
+        private
+
+        def assert_timeout_not_reached
+          action.output[:timeout_start] ||= Time.now.to_i
+
+          timeout_start = action.output[:timeout_start]
+          now = Time.now.to_i
+          timeout = action.input[:timeout]
+
+          if now - timeout_start > timeout
+            # clear timeout_start so that the action can be resumed/skipped
+            action.output[:timeout_start] = nil
+            fail(::Staypuft::Exception,
+                 "You've reached the timeout set for this action. If the " +
+                 "action is still ongoing, you can click on the " +
+                 "\"Resume Deployment\" button to continue.")
+          end
+        end
+
+      end
+    end
+  end
+end