Merge pull request #238 from Shopify/parallelize_debug

Fetch debug info in parallel
Shopify · Jan 12, 2018 · 3bad9f1 · 3bad9f1
2 parents 3ef4fe8 + 0623b71
commit 3bad9f1
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 10 deletions.
diff --git a/lib/kubernetes-deploy/kubernetes_resource.rb b/lib/kubernetes-deploy/kubernetes_resource.rb
@@ -140,7 +140,15 @@ def deploy_method
       :apply
     end
 
+    def sync_debug_info
+      @events = fetch_events
+      @logs = fetch_logs if supports_logs?
+      @debug_info_synced = true
+    end
+
     def debug_message
+      sync_debug_info unless @debug_info_synced
+
       helpful_info = []
       if deploy_failed?
         helpful_info << ColorizedString.new("#{id}: FAILED").red
@@ -156,22 +164,20 @@ def debug_message
       end
       helpful_info << "  - Final status: #{status}"
 
-      events = fetch_events
-      if events.present?
+      if @events.present?
         helpful_info << "  - Events (common success events excluded):"
-        events.each do |identifier, event_hashes|
+        @events.each do |identifier, event_hashes|
           event_hashes.each { |event| helpful_info << "      [#{identifier}]\t#{event}" }
         end
       else
         helpful_info << "  - Events: #{DEBUG_RESOURCE_NOT_FOUND_MESSAGE}"
       end
 
-      if respond_to?(:fetch_logs)
-        container_logs = fetch_logs
-        if container_logs.blank? || container_logs.values.all?(&:blank?)
+      if supports_logs?
+        if @logs.blank? || @logs.values.all?(&:blank?)
           helpful_info << "  - Logs: #{DEBUG_RESOURCE_NOT_FOUND_MESSAGE}"
         else
-          sorted_logs = container_logs.sort_by { |_, log_lines| log_lines.length }
+          sorted_logs = @logs.sort_by { |_, log_lines| log_lines.length }
           sorted_logs.each do |identifier, log_lines|
             if log_lines.empty?
               helpful_info << "  - Logs from container '#{identifier}': #{DEBUG_RESOURCE_NOT_FOUND_MESSAGE}"
@@ -322,6 +328,10 @@ def create_definition_tempfile
       file&.close
     end
 
+    def supports_logs?
+      respond_to?(:fetch_logs)
+    end
+
     def statsd_tags
       status = if deploy_failed?
         "failure"

diff --git a/lib/kubernetes-deploy/resource_watcher.rb b/lib/kubernetes-deploy/resource_watcher.rb
@@ -82,6 +82,7 @@ def record_statuses_for_summary(resources)
 
       if fail_count > 0
         @logger.summary.add_action("failed to #{@operation_name} #{fail_count} #{'resource'.pluralize(fail_count)}")
+        KubernetesDeploy::Concurrency.split_across_threads(failed_resources, &:sync_debug_info)
         failed_resources.each { |r| @logger.summary.add_paragraph(r.debug_message) }
       end
     end

diff --git a/test/unit/kubernetes-deploy/resource_watcher_test.rb b/test/unit/kubernetes-deploy/resource_watcher_test.rb
@@ -42,8 +42,14 @@ def test_failure_with_mock_resource
 
     watcher = KubernetesDeploy::ResourceWatcher.new([resource], logger: logger)
     watcher.run(delay_sync: 0.1)
+    logger.print_summary(false)
 
-    assert_logs_match(/web-pod failed to deploy after \d\.\ds/)
+    assert_logs_match_all([
+      /web-pod failed to deploy after \d\.\ds/,
+      "Result: FAILURE",
+      "Failed to deploy 1 resource",
+      "Something went wrong"
+    ], in_order: true)
   end
 
   def test_timeout_from_resource
@@ -94,6 +100,8 @@ def test_reminder_logged_at_interval_even_when_nothing_happened
   private
 
   MockResource = Struct.new(:id, :hits_to_complete, :status) do
+    attr_reader :debug_message
+
     def sync
       @hits ||= 0
       @hits += 1
@@ -115,8 +123,8 @@ def timeout
       hits_to_complete
     end
 
-    def debug_message
-      "Something went wrong"
+    def sync_debug_info
+      @debug_message = "Something went wrong"
     end
 
     def pretty_status