diff --git a/.buildkite/log-results.sh b/.buildkite/log-results.sh
index 913c2a7277..85963fa432 100755
--- a/.buildkite/log-results.sh
+++ b/.buildkite/log-results.sh
@@ -5,33 +5,18 @@ buildkite-agent annotate --style info "## :rspec: Tests summary :rspec:
 "
 buildkite-agent artifact download "elasticsearch-api/tmp/*" .
 
-# Test result summary:
-files="elasticsearch-api/tmp/*.html"
+files="elasticsearch-api/tmp/*.log"
 for f in $files; do
-  TEST_SUITE=`echo $f | grep -o "\(free\|platinum\)"`
-  RUBY_VERSION=`echo $f | grep -Po "(jruby-|\d+\.)+\d+"`
-  EXAMPLES=`cat $f | grep -o "[0-9]\+ examples\?" | tail -1`
-  FAILURES=`cat $f | grep -o "[0-9]\+ failures\?" | tail -1`
-  PENDING=`cat $f | grep -o "[0-9]\+ pending" | tail -1`
+  RUBY_VERSION=`echo $f | grep -Po "(j?ruby-|\d+\.)+\d+" | tail -1`
+  TRANSPORT_VERSION=`echo $f | grep -Po "transport-([\d.]+|main)"`
   buildkite-agent annotate --append "
-:ruby: $RUBY_VERSION :test_tube: $TEST_SUITE :rspec: $EXAMPLES - :x: $FAILURES - :suspect: $PENDING
+:ruby: $RUBY_VERSION :phone: $TRANSPORT_VERSION `tail --lines=2 $f | awk -F "-- :" '{print $2}'`
+
 "
-done
 
-# Failed tests:
-files="elasticsearch-api/tmp/*.log"
-for f in $files; do
-  FAILED_TESTS=`awk 'BEGIN { FS = " | "}; /\| failed \|/{ print $1 }' $f | uniq`
+  FAILED_TESTS=`grep "E," $f`
   if [[ -n "$FAILED_TESTS" ]]; then
-    buildkite-agent annotate --append "
-#### Failures in $f
-"
-    FAILURES_ARRAY=($(echo $FAILED_TESTS | tr ' ' "\n"))
-    for f in "${FAILURES_ARRAY[@]}"
-    do
-      buildkite-agent annotate --append "
-- $f
-"
-    done
+    buildkite-agent annotate --append "#### Failures in $f "
+    buildkite-agent annotate --append `grep "E," $f | awk -F '-- :' '{print $2}'`
   fi
 done
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 7458881c7d..47724d0f5f 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -33,7 +33,9 @@ steps:
       TRANSPORT_VERSION: "{{ matrix.transport }}"
       RUBY_SOURCE: "{{ matrix.ruby_source }}"
       TEST_SUITE: "platinum"
+      DEBUG: true
     command: ./.buildkite/run-yaml-tests.sh
+    artifact_paths: "elasticsearch-api/tmp/*"
   - wait: ~
     continue_on_failure: true
   - label: "Log Results"
diff --git a/.buildkite/run-yaml-tests.sh b/.buildkite/run-yaml-tests.sh
index ba04b78cad..a463972037 100644
--- a/.buildkite/run-yaml-tests.sh
+++ b/.buildkite/run-yaml-tests.sh
@@ -37,6 +37,7 @@ docker run \
        --env "TRANSPORT_VERSION=${TRANSPORT_VERSION}" \
        --env "STACK_VERSION=${STACK_VERSION}" \
        --env "ES_YAML_TESTS_BRANCH=${ES_YAML_TESTS_BRANCH}" \
+       --env "DEBUG=${DEBUG}" \
        --volume $repo:/usr/src/app \
        --name elasticsearch-ruby \
        --rm \
diff --git a/elasticsearch-api/spec/yaml-test-runner/run.rb b/elasticsearch-api/spec/yaml-test-runner/run.rb
index c25c3a71c3..53ea8ea134 100644
--- a/elasticsearch-api/spec/yaml-test-runner/run.rb
+++ b/elasticsearch-api/spec/yaml-test-runner/run.rb
@@ -56,12 +56,18 @@
          end
 
 tests_path = File.expand_path('./tmp', __dir__)
-
-logger = Logger.new($stdout)
-logger.level = Logger::WARN unless ENV['DEBUG']
+ruby_version = if defined? JRUBY_VERSION
+                 "jruby-#{JRUBY_VERSION}"
+               else
+                 "ruby-#{RUBY_VERSION}"
+               end
+log_filename = "es-#{Elasticsearch::VERSION}-transport-#{ENV['TRANSPORT_VERSION']}-#{ruby_version}.log"
+logfile = File.expand_path "../../tmp/#{log_filename}", __dir__
+logger = Logger.new(File.open(logfile, 'w'))
+logger.level = ENV['DEBUG'] ? Logger::DEBUG : Logger::WARN
 
 # If we're running in a release branch, download the corresponding branch for tests
 current_branch = `git rev-parse --abbrev-ref HEAD`.strip
 branch = current_branch.match(/[0-9]\.[0-9]+/)&.[](0) || ENV['ES_YAML_TESTS_BRANCH'] || nil
 Elasticsearch::Tests::Downloader::run(tests_path, branch)
-Elasticsearch::Tests::TestRunner.new(CLIENT, tests_path, logger).run
+Elasticsearch::Tests::TestRunner.new(CLIENT, tests_path, logger).run(ENV['SINGLE_TEST'] || [])