Delay for empty/timeout retries

CHANGELOG.md

@@ -4,6 +4,7 @@
 - Change HerokuRun#call to use exceptions for empty output retries in preparation of related work
 - Refactor run_shell! to use Open3.popen3 in preparation for timeout handling
 - Timeout support for 'heroku run' (in case of networking, hanging or "server boot" tests etc), defaults to 60 s, global override `$HATCHET_DEFAULT_RUN_TIMEOUT`, per-test option `:timeout`
+- Delay for empty/timeout retries if run_multi is off, defaults to 1 s, global override `$HATCHET_RUN_RETRY_DELAY`
 
 ## 7.4.0
 

lib/hatchet/app.rb

@@ -193,6 +193,7 @@ def run(cmd_type, command = DefaultCommand, options = {}, &block)
         command,
         app: self,
         retry_on_empty: options.fetch(:retry_on_empty, !ENV["HATCHET_DISABLE_EMPTY_RUN_RETRY"]),
+        retry_delay: @run_multi ? 0 : (ENV["HATCHET_RUN_RETRY_DELAY"] || 1).to_i,
         heroku: options[:heroku],
         raw: options[:raw],
         timeout: options.fetch(:timeout, (ENV["HATCHET_DEFAULT_RUN_TIMEOUT"] || 60).to_i)
@@ -248,6 +249,7 @@ def run_multi(command, options = {}, &block)
           command,
           app: self,
           retry_on_empty: options.fetch(:retry_on_empty, !ENV["HATCHET_DISABLE_EMPTY_RUN_RETRY"]),
+          retry_delay: @run_multi ? 0 : (ENV["HATCHET_RUN_RETRY_DELAY"] || 1).to_i,
           heroku: options[:heroku],
           raw: options[:raw],
           timeout: options.fetch(:timeout, (ENV["HATCHET_DEFAULT_RUN_TIMEOUT"] || 60).to_i)

lib/hatchet/heroku_run.rb

@@ -72,6 +72,7 @@ def initialize(
       app: ,
       heroku: {},
       retry_on_empty: !ENV["HATCHET_DISABLE_EMPTY_RUN_RETRY"],
+      retry_delay: 0,
       raw: false,
       stderr: $stderr,
       timeout: 0)
@@ -81,6 +82,7 @@ def initialize(
       @timeout = timeout
       @command = build_heroku_command(command, heroku || {})
       @retry_on_empty = retry_on_empty
+      @retry_delay = retry_delay
       @stderr = stderr
       @result = nil
       @dyno_id = nil
@@ -114,6 +116,7 @@ def call
           message << "\nreleases: #{@app.releases}"
           message << "\n#{caller.join("\n")}"
           @stderr.puts message
+          sleep(@retry_delay) # without run_multi, this will prevent occasional "can only run one free dyno" errors
           retry
         end
       rescue HerokuRunTimeoutError => e
@@ -127,6 +130,7 @@ def call
           message << "\nreleases: #{@app.releases}"
           message << "\n#{caller.join("\n")}"
           @stderr.puts message
+          sleep(@retry_delay) # without run_multi, this will prevent occasional "can only run one free dyno" errors
           retry
         end
       end