Merge pull request #216 from kaushik/CPD-1834

Wait for CI_status and release tag in github build mechanism

lib/moonshot/build_mechanism/github_release.rb

@@ -17,8 +17,10 @@ class GithubRelease # rubocop:disable Metrics/ClassLength
     def_delegator :@build_mechanism, :output_file
 
     # @param build_mechanism Delegates building after GitHub release is created.
-    def initialize(build_mechanism)
+    def initialize(build_mechanism, ci_status_timeout: 600, max_tag_find_timeout: 240)
       @build_mechanism = build_mechanism
+      @ci_status_timeout = ci_status_timeout
+      @max_tag_find_timeout = max_tag_find_timeout
     end
 
     def doctor_hook
@@ -113,7 +115,7 @@ def git_tag_exists(tag, sha)
     def git_push_tag(remote, tag)
       cmd = "git push #{remote} refs/tags/#{tag}:refs/tags/#{tag}"
       sh_step(cmd) do
-        sleep 2 # GitHub needs a moment to register the tag.
+        hub_find_remote_tag(tag)
       end
     end
 
@@ -141,15 +143,23 @@ def hub_release_exists(semver)
       exists
     end
 
+    # Attempt to find the build. Rescue and re-attempt if the build can not
+    # be found on github yet.
+    def hub_find_remote_tag(tag_name)
+      retry_opts = {
+        max_elapsed_time: @max_tag_find_timeout,
+        multiplier: 2
+      }
+      sh_retry("hub ls-remote --exit-code --tags upstream #{tag_name}",
+               opts: retry_opts)
+    end
+
     def validate_commit
       cmd = "git show --stat #{@sha}"
       sh_step(cmd, msg: "Validate commit #{@sha}.") do |_, out|
         @commit_detail = out
       end
-      cmd = "hub ci-status --verbose #{@sha}"
-      sh_step(cmd, msg: "Check CI status for #{@sha}.") do |_, out|
-        @ci_statuses = out
-      end
+      @ci_statuses = check_ci_status(@sha)
     end
 
     def validate_changelog(version)
@@ -171,6 +181,25 @@ def fetch_changes(version)
       end
     end
 
+    # Checks for the commit's CI job status. If its not finished yet,
+    # wait till timeout.
+    #
+    # @param sha [String] Commit sha.
+    #
+    # @return [String] Status and links to the CI jobs
+    def check_ci_status(sha)
+      out = nil
+      retry_opts = {
+        max_elapsed_time: @ci_status_timeout,
+        base_interval: 10
+      }
+      ilog.start_threaded("Check CI status for #{sha}.") do |step|
+        out = sh_retry("hub ci-status --verbose #{sha}", opts: retry_opts)
+        step.success
+      end
+      out
+    end
+
     def releases_url
       `hub browse -u -- releases`.chomp
     end

lib/moonshot/build_mechanism/travis_deploy.rb

@@ -68,32 +68,18 @@ def find_build_and_job(version)
     #
     # @return [String] Job number for the travis build.
     def wait_for_build(version)
+      # Attempt to find the build. Re-attempt if the build can not
+      # be found on travis yet.
+      retry_opts = {
+        tries: MAX_BUILD_FIND_ATTEMPTS,
+        base_interval: 10
+      }
       job_number = nil
-      attempts = 0
-      loop do
-        # Give travis some time to start the build.
-        attempts += 1
-        sleep 10
-
-        # Attempt to find the build. Rescue and re-attempt if the build can not
-        # be found on travis yet.
-        begin
-          build_out = sh_out("bundle exec travis show #{@cli_args} #{version}")
-        rescue RuntimeError => e
-          next unless attempts >= MAX_BUILD_FIND_ATTEMPTS
-          raise e
-        end
-
-        unless (job_number = build_out.match(/^#(\d+\.\d+) .+BUILD=1.+/)[1])
-          next unless attempts >= MAX_BUILD_FIND_ATTEMPTS
-          raise "Build for #{version} not found.\n#{build_out}"
-        end
-
-        # If we've reached this point then everything went smoothly and we can
-        # exit the loop.
-        break
+      sh_retry("bundle exec travis show #{@cli_args} #{version}",
+               opts: retry_opts) do |build_out|
+        raise CommandError, "Build for #{version} not found.\n#{build_out}" \
+          unless (job_number = build_out.match(/^#(\d+\.\d+) .+BUILD=1.+/)[1])
       end
-
       job_number
     end
 

lib/moonshot/shell.rb

@@ -1,7 +1,20 @@
 require 'thor'
+require 'retriable'
 
 # Mixin providing the Thor::Shell methods and other shell execution helpers.
 module Moonshot::Shell
+  CommandError = Class.new(RuntimeError)
+
+  # Retry every 10 seconds for a maximum of 2 minutes.
+  DEFAULT_RETRY_OPTIONS = {
+    on: RuntimeError,
+    tries: 50,
+    multiplier: 1,
+    base_interval: 10,
+    max_elapsed_time: 120,
+    rand_factor: 0
+  }.freeze
+
   # Run a command, returning stdout. Stderr is suppressed unless the command
   # returns non-zero.
   def sh_out(cmd, fail: true, stdin: '')
@@ -22,7 +35,7 @@ def sh_out(cmd, fail: true, stdin: '')
     r_err.close
 
     if fail && $CHILD_STATUS.exitstatus != 0
-      raise "`#{cmd}` exited #{$CHILD_STATUS.exitstatus}\n" \
+      raise CommandError, "`#{cmd}` exited #{$CHILD_STATUS.exitstatus}\n" \
            "stdout:\n" \
            "#{stdout}\n" \
            "stderr:\n" \
@@ -51,4 +64,22 @@ def sh_step(cmd, args = {})
       step.success
     end
   end
+
+  # Retries every second upto maximum of 2 minutes with the default options.
+  #
+  # @param cmd [String] command to execute.
+  # @param fail [Boolean] Raise error when the command exits with non-zero.
+  # @param stdin [String] Input to the command.
+  # @param opts [Hash] Options for retriable.
+  #
+  # @return [String] Stdout form the command.
+  def sh_retry(cmd, fail: true, stdin: '', opts: {})
+    Retriable.retriable(DEFAULT_RETRY_OPTIONS.merge(opts)) do
+      out = sh_out(cmd, stdin: stdin)
+      yield out if block_given?
+      out
+    end
+  rescue CommandError => e
+    raise e if fail
+  end
 end

spec/moonshot/build_mechanism/github_release_spec.rb

@@ -125,5 +125,69 @@ module Moonshot # rubocop:disable ModuleLength
         expect(subject.send(:hub_release_exists, tag)).to eq(false)
       end
     end
+
+    describe '#hub_find_remote_tag' do
+      let(:tag) { '1.23.4' }
+
+      before(:each) do
+        Retriable.configure do |c|
+          c.sleep_disabled = true
+        end
+      end
+      it 'passes if the github release exists' do
+        expect(subject).to receive(:sh_out)
+          .with("hub ls-remote --exit-code --tags upstream #{tag}", stdin: '')
+          .and_return(tag)
+        expect(subject.send(:hub_find_remote_tag, tag)).to eq(tag)
+      end
+
+      it 'retries and passes if the github release exists' do
+        expect(subject).to receive(:sh_out)
+          .with("hub ls-remote --exit-code --tags upstream #{tag}", stdin: '')
+          .and_raise.twice
+        expect(subject).to receive(:sh_out)
+          .with("hub ls-remote --exit-code --tags upstream #{tag}", stdin: '')
+          .and_return(tag)
+        expect(subject.send(:hub_find_remote_tag, tag)).to eq(tag)
+      end
+
+      it 'retries and fails if the github release does not exist' do
+        expect(subject).to receive(:sh_out)
+          .with("hub ls-remote --exit-code --tags upstream #{tag}", stdin: '')
+          .and_raise(Shell::CommandError).at_least(:twice)
+
+        expect do
+          subject.send(:hub_find_remote_tag, tag)
+        end.to raise_error(Shell::CommandError)
+      end
+    end
+
+    describe '#check_ci_status' do
+      let(:sha) { '485173e66f42e0685d1fa9dd853027a583116e3d' }
+      let(:ci_statuses) { "Job 1: link\nJob 2: link\n" }
+      let(:step) do
+        instance_double(InteractiveLogger::Step)
+      end
+
+      before(:each) do
+        allow(resources.ilog).to receive(:start_threaded).and_yield(step)
+        Retriable.configure do |c|
+          c.sleep_disabled = true
+        end
+      end
+
+      it 'passes if the ci status returns with non-zero status' do
+        expect(subject).to receive(:sh_out).with("hub ci-status --verbose #{sha}", anything)
+          .and_return(ci_statuses)
+        expect(step).to receive(:success)
+        expect(subject.send(:check_ci_status, sha)).to eq(ci_statuses)
+      end
+
+      it 'should fail if status does not exit non zero within time limit' do
+        expect(subject).to receive(:sh_out).with("hub ci-status --verbose #{sha}", anything)
+          .and_raise(Shell::CommandError).at_least(:twice)
+        expect { subject.send(:check_ci_status, sha) }.to raise_error(Shell::CommandError)
+      end
+    end
   end
 end

spec/moonshot/build_mechanism/travis_deploy_spec.rb

@@ -63,23 +63,26 @@ module Moonshot # rubocop:disable Metrics/ModuleLength
         'gemfile: .travis.build.Gemfile, os: linux '
       end
 
-      it 'should only make one attempt if the job is found' do
-        expect(subject).to receive(:sleep).once
-        expect(subject).to receive(:sh_out).and_return(output)
+      before(:each) do
+        Retriable.configure do |c|
+          c.sleep_disabled = true
+        end
+      end
+
+      it 'should return the right job number' do
+        expect(subject).to receive(:sh_out).and_return(output).once
 
         expect(subject.send(:wait_for_build, tag)).to eq(job_number)
       end
 
       it 'should only make the max number of attempts before failing' do
-        expect(subject).to receive(:sleep).exactly(10).times
         expect(subject).to receive(:sh_out).and_raise.exactly(10).times
 
         expect { subject.send(:wait_for_build, tag) }.to \
           raise_error(RuntimeError)
       end
 
       it 'should make attempts until the build is found' do
-        expect(subject).to receive(:sleep).exactly(3).times
         expect(subject).to receive(:sh_out).and_raise.twice
         expect(subject).to receive(:sh_out).and_return(output)