Fix specs to work with Elasticsearch 7.x

elastic#35849 changed the search response
format, which broke a bunch of tests.

Fixes elastic#817

spec/es_spec_helper.rb

@@ -40,6 +40,18 @@ def self.es_version
     RSpec.configuration.filter[:es_version] || ENV['ES_VERSION']
   end
 
+  RSpec::Matchers.define :have_hits do |expected|
+    es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION']
+    match do |actual|
+      if ESHelper.es_version_satisfies?(">=7")
+        expected == actual['hits']['total']['value']
+      else
+        expected == actual['hits']['total']
+      end
+    end
+  end
+
+
   def self.es_version_satisfies?(*requirement)
     es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION']
     if es_version.nil?

spec/integration/outputs/create_spec.rb

@@ -35,7 +35,7 @@ def get_es_output(action, id, version=nil, version_type=nil)
       # Wait or fail until everything's indexed.
       Stud::try(3.times) do
         r = @es.search
-        expect(r["hits"]["total"]).to eq(1)
+        expect(r).to have_hits(1)
       end
     end
 

spec/integration/outputs/ingest_pipeline_spec.rb

@@ -54,13 +54,13 @@
       #Wait or fail until everything's indexed.
       Stud::try(20.times) do
         r = @es.search
-        expect(r["hits"]["total"]).to eq(1)
+        expect(r).to have_hits(1)
       end
     end
 
     it "indexes using the proper pipeline" do
       results = @es.search(:index => 'logstash-*', :q => "message:\"netcat\"")
-      expect(results["hits"]["total"]).to eq(1)
+      expect(results).to have_hits(1)
       expect(results["hits"]["hits"][0]["_source"]["response"]).to eq("200")
       expect(results["hits"]["hits"][0]["_source"]["bytes"]).to eq("182")
       expect(results["hits"]["hits"][0]["_source"]["verb"]).to eq("GET")

spec/integration/outputs/no_es_on_startup_spec.rb

@@ -39,7 +39,7 @@
     subject.multi_receive([event1, event2])
     @es.indices.refresh
     r = @es.search
-    expect(r["hits"]["total"]).to eql(2)
+    expect(r).to have_hits(2)
   end
 
   it 'should ingest events when Elasticsearch recovers after documents are sent' do
@@ -52,7 +52,7 @@
     subject.multi_receive([event1, event2])
     @es.indices.refresh
     r = @es.search
-    expect(r["hits"]["total"]).to eql(2)
+    expect(r).to have_hits(2)
   end
 
 end

spec/integration/outputs/retry_spec.rb

@@ -143,7 +143,7 @@ def mock_actions_with_response(*resp)
 
     @es.indices.refresh
     r = @es.search
-    expect(r["hits"]["total"]).to eql(0)
+    expect(r).to have_hits(0)
   end
 
   it "successful requests should not be appended to retry queue" do
@@ -154,7 +154,7 @@ def mock_actions_with_response(*resp)
     subject.close
     @es.indices.refresh
     r = @es.search
-    expect(r["hits"]["total"]).to eql(1)
+    expect(r).to have_hits(1)
   end
 
   it "should only index proper events" do
@@ -164,6 +164,6 @@ def mock_actions_with_response(*resp)
 
     @es.indices.refresh
     r = @es.search
-    expect(r["hits"]["total"]).to eql(1)
+    expect(r).to have_hits(1)
   end
 end

spec/integration/outputs/templates_5x_spec.rb

@@ -41,19 +41,19 @@
       # Wait or fail until everything's indexed.
       Stud::try(20.times) do
         r = @es.search
-        expect(r["hits"]["total"]).to eq(8)
+        expect(r).to have_hits(8)
       end
     end
 
     it "permits phrase searching on string fields" do
       results = @es.search(:q => "message:\"sample message\"")
-      expect(results["hits"]["total"]).to eq(1)
+      expect(results).to have_hits(1)
       expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
     end
 
     it "numbers dynamically map to a numeric type and permit range queries" do
       results = @es.search(:q => "somevalue:[5 TO 105]")
-      expect(results["hits"]["total"]).to eq(2)
+      expect(results).to have_hits(2)
 
       values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
       expect(values).to include(10)
@@ -63,22 +63,22 @@
 
     it "does not create .keyword field for top-level message field" do
       results = @es.search(:q => "message.keyword:\"sample message here\"")
-      expect(results["hits"]["total"]).to eq(0)
+      expect(results).to have_hits(0)
     end
 
     it "creates .keyword field for nested message fields" do
       results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
-      expect(results["hits"]["total"]).to eq(1)
+      expect(results).to have_hits(1)
     end
 
     it "creates .keyword field from any string field which is not_analyzed" do
       results = @es.search(:q => "country.keyword:\"us\"")
-      expect(results["hits"]["total"]).to eq(1)
+      expect(results).to have_hits(1)
       expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
 
       # partial or terms should not work.
       results = @es.search(:q => "country.keyword:\"u\"")
-      expect(results["hits"]["total"]).to eq(0)
+      expect(results).to have_hits(0)
     end
 
     it "make [geoip][location] a geo_point" do

spec/integration/outputs/templates_spec.rb

@@ -41,19 +41,19 @@
       # Wait or fail until everything's indexed.
       Stud::try(20.times) do
         r = @es.search
-        expect(r["hits"]["total"]).to eq(8)
+        expect(r).to have_hits(8)
       end
     end
 
     it "permits phrase searching on string fields" do
       results = @es.search(:q => "message:\"sample message\"")
-      expect(results["hits"]["total"]).to eq(1)
+      expect(results).to have_hits(1)
       expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
     end
 
     it "numbers dynamically map to a numeric type and permit range queries" do
       results = @es.search(:q => "somevalue:[5 TO 105]")
-      expect(results["hits"]["total"]).to eq(2)
+      expect(results).to have_hits(2)
 
       values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
       expect(values).to include(10)
@@ -63,22 +63,22 @@
 
     it "does not create .raw field for the message field" do
       results = @es.search(:q => "message.raw:\"sample message here\"")
-      expect(results["hits"]["total"]).to eq(0)
+      expect(results).to have_hits(0)
     end
 
     it "creates .raw field for nested message fields" do
       results = @es.search(:q => "somemessage.message.raw:\"sample nested message here\"")
-      expect(results["hits"]["total"]).to eq(1)
+      expect(results).to have_hits(1)
     end
 
     it "creates .raw field from any string field which is not_analyzed" do
       results = @es.search(:q => "country.raw:\"us\"")
-      expect(results["hits"]["total"]).to eq(1)
+      expect(results).to have_hits(1)
       expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
 
       # partial or terms should not work.
       results = @es.search(:q => "country.raw:\"u\"")
-      expect(results["hits"]["total"]).to eq(0)
+      expect(results).to have_hits(0)
     end
 
     it "make [geoip][location] a geo_point" do