[scan] Reduce run time by removing report generation command as separ…

…ate process (#7365)

* [scan] Increase speed of test report generation

* [scan] Move xcpretty reporter code to separate file

* Fix some bugs with output_files option

* [scan] bug fixes and formatting

* [scan] add tests for XCPrettyReporterOptionsGenerator

* Rubocop fixes

* [scan] Add backwards compatibility for option :custom_report_file_name

In this way, it should now behave the same way as :output_files, but is
marked as deprecated because the option name is misleading/confusing.

* [scan] Add runtime deprecation warning message

* [scan] Use fastlane-provided deprecation mechanism

* [scan] Remove unneccesary comment

* add mocks to fix unit tests

* Slight refactoring to remove unit test workarounds

* Fix trailing whitespace

* Make changes for code review comments

* Add a unit test for deprecated Scan option custom_report_file_name

* Update tests to take order of values into account

* Forgot to add check for warning message

* Re-add line, accidentally removed

* run rubocop

scan/lib/scan.rb

@@ -3,8 +3,8 @@
 require 'scan/options'
 require 'scan/runner'
 require 'scan/detect_values'
-require 'scan/report_collector'
 require 'scan/test_command_generator'
+require 'scan/xcpretty_reporter_options_generator.rb'
 require 'scan/test_result_parser'
 require 'scan/error_handler'
 require 'scan/slack_poster'

scan/lib/scan/options.rb

@@ -113,8 +113,14 @@ def self.available_options
         FastlaneCore::ConfigItem.new(key: :output_types,
                                      short_option: "-f",
                                      env_name: "SCAN_OUTPUT_TYPES",
-                                     description: "Comma separated list of the output types (e.g. html, junit)",
+                                     description: "Comma separated list of the output types (e.g. html, junit, json-compilation-database)",
                                      default_value: "html,junit"),
+        FastlaneCore::ConfigItem.new(key: :output_files,
+                                     env_name: "SCAN_OUTPUT_FILES",
+                                     description: "Comma separated list of the output files, corresponding to the types provided by :output_types (order should match). If specifying an output type of json-compilation-database with :use_clang_report_name enabled, that option will take precedence",
+                                     conflicting_options: [:custom_report_file_name],
+                                     optional: true,
+                                     default_value: nil),
         FastlaneCore::ConfigItem.new(key: :buildlog_path,
                                      short_option: "-l",
                                      env_name: "SCAN_BUILDLOG_PATH",
@@ -251,7 +257,9 @@ def self.available_options
                                     default_value: false),
         FastlaneCore::ConfigItem.new(key: :custom_report_file_name,
                                      env_name: "SCAN_CUSTOM_REPORT_FILE_NAME",
-                                     description: "Sets custom full report file name",
+                                     description: "Sets custom full report file name when generating a single report",
+                                     deprecated: "Use --output_files",
+                                     conflicting_options: [:output_files],
                                      optional: true,
                                      is_string: true)
       ]

scan/lib/scan/runner.rb

@@ -46,22 +46,7 @@ def test_app
     end
 
     def handle_results(tests_exit_status)
-      # First, generate a JUnit report to get the number of tests
-      require 'tempfile'
-      output_file = Tempfile.new("junit_report")
-
-      report_collector = ReportCollector.new(Scan.config[:open_report],
-                                             Scan.config[:output_types],
-                                             Scan.config[:output_directory],
-                                             Scan.config[:use_clang_report_name],
-                                             Scan.config[:custom_report_file_name])
-
-      cmd = report_collector.generate_commands(TestCommandGenerator.xcodebuild_log_path,
-                                               types: 'junit',
-                                               output_file_name: output_file.path).values.last
-      system(cmd)
-
-      result = TestResultParser.new.parse_result(output_file.read)
+      result = TestResultParser.new.parse_result(test_results)
       SlackPoster.new.run(result)
 
       if result[:failures] > 0
@@ -81,15 +66,22 @@ def handle_results(tests_exit_status)
 
       copy_simulator_logs
 
-      report_collector.parse_raw_file(TestCommandGenerator.xcodebuild_log_path)
-
       if result[:failures] > 0
         UI.test_failure!("Tests have failed")
       end
 
       unless tests_exit_status == 0
         UI.test_failure!("Test execution failed. Exit status: #{tests_exit_status}")
       end
+
+      if !Helper.is_ci? && Scan.cache[:open_html_report_path]
+        `open --hide '#{Scan.cache[:open_html_report_path]}'`
+      end
+    end
+
+    def test_results
+      return "" unless Scan.cache[:temp_junit_report]
+      File.read(Scan.cache[:temp_junit_report])
     end
 
     def copy_simulator_logs

scan/lib/scan/test_command_generator.rb

@@ -75,8 +75,6 @@ def suffix
       end
 
       def pipe
-        # During building we just show the output in the terminal
-        # Check out the ReportCollector class for more xcpretty things
         pipe = ["| tee '#{xcodebuild_log_path}'"]
 
         if Scan.config[:output_style] == 'raw'
@@ -103,7 +101,13 @@ def pipe
           formatter << "--test"
         end
 
-        return pipe << ["| xcpretty #{formatter.join(' ')}"]
+        reporter_options_generator = XCPrettyReporterOptionsGenerator.new(Scan.config[:open_report],
+                                                                          Scan.config[:output_types],
+                                                                          Scan.config[:output_files] || Scan.config[:custom_report_file_name],
+                                                                          Scan.config[:output_directory],
+                                                                          Scan.config[:use_clang_report_name])
+        reporter_options = reporter_options_generator.generate_reporter_options
+        return pipe << "| xcpretty #{formatter.join(' ')} #{reporter_options.join(' ')}"
       end
 
       # Store the raw file

scan/lib/scan/xcpretty_reporter_options_generator.rb

@@ -0,0 +1,76 @@
+module Scan
+  class XCPrettyReporterOptionsGenerator
+    SUPPORTED_REPORT_TYPES = %w(html junit json-compilation-database)
+
+    def self.generate_from_scan_config
+      self.new(Scan.config[:open_report],
+               Scan.config[:output_types],
+               Scan.config[:output_files] || Scan.config[:custom_report_file_name],
+               Scan.config[:output_directory],
+               Scan.config[:use_clang_report_name])
+    end
+
+    # Intialize with values from Scan.config matching these param names
+    def initialize(open_report, output_types, output_files, output_directory, use_clang_report_name)
+      @open_report = open_report
+      @output_types = output_types
+      @output_files = output_files
+      @output_directory = output_directory
+      @use_clang_report_name = use_clang_report_name
+
+      # might already be an array when passed via fastlane
+      @output_types = @output_types.split(',') if @output_types.kind_of?(String)
+
+      if @output_files.nil?
+        @output_files = @output_types.map { |type| "report.#{type}" }
+      elsif @output_files.kind_of?(String)
+        # might already be an array when passed via fastlane
+        @output_files = @output_files.split(',')
+      end
+
+      unless @output_types.length == @output_files.length
+        UI.important("WARNING: output_types and output_files do not have the same number of items. Default values will be substituted as needed.")
+      end
+
+      (@output_types - SUPPORTED_REPORT_TYPES).each do |type|
+        UI.error("Couldn't find reporter '#{type}', available #{SUPPORTED_REPORT_TYPES.join(', ')}")
+      end
+    end
+
+    def generate_reporter_options
+      reporter = []
+
+      valid_types = @output_types & SUPPORTED_REPORT_TYPES
+      valid_types.each do |raw_type|
+        type = raw_type.strip
+        output_path = File.join(File.expand_path(@output_directory), determine_output_file_name(type))
+        reporter << "--report #{type}"
+        reporter << "--output #{output_path}"
+
+        if type == "html" && @open_report
+          Scan.cache[:open_html_report_path] = output_path
+        end
+      end
+
+      # adds another junit reporter in case the user does not specify one
+      # this will be used to generate a results table and then discarded
+      require 'tempfile'
+      Scan.cache[:temp_junit_report] = Tempfile.new("junit_report").path
+      reporter << "--report junit"
+      reporter << "--output #{Scan.cache[:temp_junit_report]}"
+      return reporter
+    end
+
+    private
+
+    def determine_output_file_name(type)
+      if @use_clang_report_name && type == "json-compilation-database"
+        return "compile_commands.json"
+      end
+
+      index = @output_types.index(type)
+      file = @output_files[index]
+      file || "report.#{type}"
+    end
+  end
+end

scan/spec/runner_spec.rb

@@ -8,6 +8,11 @@
         allow(Scan::SlackPoster).to receive(:new).and_return(mock_slack_poster)
         allow(mock_slack_poster).to receive(:run)
         allow(Scan::TestCommandGenerator).to receive(:xcodebuild_log_path).and_return('./scan/spec/fixtures/boring.log')
+
+        mock_test_result_parser = Object.new
+        allow(Scan::TestResultParser).to receive(:new).and_return(mock_test_result_parser)
+        allow(mock_test_result_parser).to receive(:parse_result).and_return({ tests: 100, failures: 0 })
+
         @scan = Scan::Runner.new
       end
 
@@ -20,6 +25,7 @@
             project: './scan/examples/standard/app.xcodeproj',
             include_simulator_logs: false
           })
+
           expect(FastlaneCore::Simulator).not_to receive(:copy_logs)
           @scan.handle_results(0)
         end
@@ -34,6 +40,7 @@
             project: './scan/examples/standard/app.xcodeproj',
             include_simulator_logs: true
           })
+
           expect(FastlaneCore::Simulator).to receive(:copy_logs)
           @scan.handle_results(0)
         end

scan/spec/test_command_generator_spec.rb

@@ -218,6 +218,7 @@
             devices: ["iPhone 6s", "iPad Air"],
             project: './scan/examples/standard/app.xcodeproj'
           })
+          Scan.cache[:temp_junit_report] = './scan/spec/fixtures/boring.log'
 
           expect(FastlaneCore::CommandExecutor).
             to receive(:execute).
@@ -227,6 +228,10 @@
             to receive(:execute).
             with(command: "xcrun simctl spawn 2ABEAF08-E480-4617-894F-6BAB587E7963 log collect --output /tmp/scan_results/system_logs-iPad\\ Air_iOS_10.0.logarchive 2>/dev/null", print_all: false, print_command: true)
 
+          mock_test_result_parser = Object.new
+          allow(Scan::TestResultParser).to receive(:new).and_return(mock_test_result_parser)
+          allow(mock_test_result_parser).to receive(:parse_result).and_return({ tests: 100, failures: 0 })
+
           mock_slack_poster = Object.new
           allow(Scan::SlackPoster).to receive(:new).and_return(mock_slack_poster)
           allow(mock_slack_poster).to receive(:run)