Expose sequence number and primary terms in search responses (#37639)

Users may require the sequence number and primary terms to perform optimistic concurrency control operations. Currently, you can get the sequence number via the `docvalues_fields` API but the primary term is not accessible because it is maintained by the `SeqNoFieldMapper` and the infrastructure can't find it. This commit adds a dedicated sub fetch phase to return both numbers that is connected to a new `seq_no_primary_term` parameter.
elastic · Jan 24, 2019 · 4d4f860 · 4d4f860
1 parent 1dde748
commit 4d4f860
Show file tree

Hide file tree

Showing 40 changed files with 611 additions and 71 deletions.
diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
@@ -25,6 +25,7 @@ The top_hits aggregation returns regular search hits, because of this many per h
 * <<search-request-script-fields,Script fields>>
 * <<search-request-docvalue-fields,Doc value fields>>
 * <<search-request-version,Include versions>>
+* <<search-request-seq-no-primary-term,Include Sequence Numbers and Primary Terms>>
 
 ==== Example
 

diff --git a/docs/reference/docs/concurrency-control.asciidoc b/docs/reference/docs/concurrency-control.asciidoc
@@ -87,7 +87,7 @@ returns:
 
 
 Note: The <<search-search,Search API>> can return the `_seq_no` and `_primary_term`
-for each search hit by requesting the `_seq_no` and `_primary_term` <<search-request-docvalue-fields,Doc Value Fields>>.
+for each search hit by setting <<search-request-seq-no-primary-term,`seq_no_primary_term` parameter>>.
 
 The sequence number and the primary term uniquely identify a change. By noting down 
 the sequence number and primary term returned, you can make sure to only change the

diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc
@@ -205,7 +205,7 @@ include::request/preference.asciidoc[]
 
 include::request/explain.asciidoc[]
 
-include::request/version.asciidoc[]
+include::request/version-and-seq-no.asciidoc[]
 
 include::request/index-boost.asciidoc[]
 

diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc
@@ -76,6 +76,7 @@ Inner hits also supports the following per document features:
 * <<search-request-script-fields,Script fields>>
 * <<search-request-docvalue-fields,Doc value fields>>
 * <<search-request-version,Include versions>>
+* <<search-request-seq-no-primary-term,Include Sequence Numbers and Primary Terms>>
 
 [[nested-inner-hits]]
 ==== Nested inner hits

diff --git a/docs/reference/search/request/version-and-seq-no.asciidoc b/docs/reference/search/request/version-and-seq-no.asciidoc
@@ -0,0 +1,34 @@
+[[search-request-seq-no-primary-term]]
+=== Sequence Numbers and Primary Term
+
+Returns the sequence number and primary term of the last modification to each search hit.
+See <<optimistic-concurrency-control>> for more details.
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "seq_no_primary_term": true,
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+[[search-request-version]]
+=== Version
+
+Returns a version for each search hit.
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "version": true,
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+--------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/search/request/version.asciidoc b/docs/reference/search/request/version.asciidoc
diff --git a/...les/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java b/...les/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java
@@ -256,6 +256,7 @@ public void testFromJson() throws IOException {
                 "      \"from\" : 0,\n" +
                 "      \"size\" : 100,\n" +
                 "      \"version\" : false,\n" +
+                "      \"seq_no_primary_term\" : false,\n" +
                 "      \"explain\" : false,\n" +
                 "      \"track_scores\" : false,\n" +
                 "      \"sort\" : [ {\n" +

diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java
@@ -56,6 +56,8 @@
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.index.query.QueryBuilders.nestedQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM;
+import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
 import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery;
 import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -66,6 +68,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -140,9 +143,10 @@ public void testSimpleParentChild() throws Exception {
         assertThat(innerHits.getAt(1).getId(), equalTo("c2"));
         assertThat(innerHits.getAt(1).getType(), equalTo(legacy() ? "comment" : "doc"));
 
+        final boolean seqNoAndTerm = randomBoolean();
         response = client().prepareSearch("articles")
             .setQuery(hasChildQuery("comment", matchQuery("message", "elephant"), ScoreMode.None)
-                .innerHit(new InnerHitBuilder()))
+                .innerHit(new InnerHitBuilder().setSeqNoAndPrimaryTerm(seqNoAndTerm)))
             .get();
         assertNoFailures(response);
         assertHitCount(response, 1);
@@ -159,6 +163,22 @@ public void testSimpleParentChild() throws Exception {
         assertThat(innerHits.getAt(2).getId(), equalTo("c6"));
         assertThat(innerHits.getAt(2).getType(), equalTo(legacy() ? "comment" : "doc"));
 
+        if (seqNoAndTerm) {
+            assertThat(innerHits.getAt(0).getPrimaryTerm(), equalTo(1L));
+            assertThat(innerHits.getAt(1).getPrimaryTerm(), equalTo(1L));
+            assertThat(innerHits.getAt(2).getPrimaryTerm(), equalTo(1L));
+            assertThat(innerHits.getAt(0).getSeqNo(), greaterThanOrEqualTo(0L));
+            assertThat(innerHits.getAt(1).getSeqNo(), greaterThanOrEqualTo(0L));
+            assertThat(innerHits.getAt(2).getSeqNo(), greaterThanOrEqualTo(0L));
+        } else {
+            assertThat(innerHits.getAt(0).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM));
+            assertThat(innerHits.getAt(1).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM));
+            assertThat(innerHits.getAt(2).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM));
+            assertThat(innerHits.getAt(0).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO));
+            assertThat(innerHits.getAt(1).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO));
+            assertThat(innerHits.getAt(2).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO));
+        }
+
         response = client().prepareSearch("articles")
             .setQuery(
                 hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(

diff --git a/...rent-join/src/test/java/org/elasticsearch/join/query/LegacyHasChildQueryBuilderTests.java b/...rent-join/src/test/java/org/elasticsearch/join/query/LegacyHasChildQueryBuilderTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.join.query;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
@@ -240,6 +239,7 @@ public void testFromJson() throws IOException {
                 "      \"from\" : 0,\n" +
                 "      \"size\" : 100,\n" +
                 "      \"version\" : false,\n" +
+                "      \"seq_no_primary_term\" : false,\n" +
                 "      \"explain\" : false,\n" +
                 "      \"track_scores\" : false,\n" +
                 "      \"sort\" : [ {\n" +

diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml
@@ -1,4 +1,7 @@
 setup:
+  - skip:
+      version: " - 5.5.99"
+      reason:  parent-join was added in 5.6.
   - do:
       indices.create:
         index: test
@@ -11,30 +14,30 @@ setup:
                     relations:
                       parent: child
 
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    1
+        body:  {"foo": "bar", "join_field": {"name" : "parent"} }
+
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    2
+        routing: 1
+        body:  {"bar": "baz", "join_field": { "name" : "child", "parent": "1"} }
+
+  - do:
+      indices.refresh: {}
+
 ---
 "Parent/child inner hits":
     - skip:
         version: " - 5.5.99"
         reason:  parent-join was added in 5.6.
 
-    - do:
-        index:
-          index: test
-          type:  doc
-          id:    1
-          body:  {"foo": "bar", "join_field": {"name" : "parent"} }
-
-    - do:
-        index:
-          index: test
-          type:  doc
-          id:    2
-          routing: 1
-          body:  {"bar": "baz", "join_field": { "name" : "child", "parent": "1"} }
-
-    - do:
-        indices.refresh: {}
-
     - do:
         search:
           body: { "query" : { "has_child" : { "type" : "child", "query" : { "match_all" : {} }, "inner_hits" : {} } } }
@@ -44,3 +47,24 @@ setup:
     - match: { hits.hits.0.inner_hits.child.hits.hits.0._index: "test"}
     - match: { hits.hits.0.inner_hits.child.hits.hits.0._id: "2" }
     - is_false: hits.hits.0.inner_hits.child.hits.hits.0._nested
+
+---
+"Parent/child inner hits with seq no":
+  - skip:
+      version: " - 6.6.99"
+      reason: support was added in 6.7
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body: { "query" : { "has_child" :
+            { "type" : "child", "query" : { "match_all" : {} }, "inner_hits" : { "seq_no_primary_term": true} }
+        } }
+  - match: { hits.total: 1 }
+  - match: { hits.hits.0._index: "test" }
+  - match: { hits.hits.0._id: "1" }
+  - match: { hits.hits.0.inner_hits.child.hits.hits.0._index: "test"}
+  - match: { hits.hits.0.inner_hits.child.hits.hits.0._id: "2" }
+  - is_false: hits.hits.0.inner_hits.child.hits.hits.0._nested
+  - gte: { hits.hits.0.inner_hits.child.hits.hits.0._seq_no: 0 }
+  - gte: { hits.hits.0.inner_hits.child.hits.hits.0._primary_term: 1 }
diff --git a/...ndex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java b/...ndex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java
@@ -177,7 +177,7 @@ public void testInitialSearchParamsMisc() {
             fetchVersion = randomBoolean();
             searchRequest.source().version(fetchVersion);
         }
-
+        
         Map<String, String> params = initialSearch(searchRequest, query, remoteVersion).getParameters();
 
         if (scroll == null) {

diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml
@@ -29,13 +29,16 @@
       search:
         index: test_index,my_remote_cluster:test_index
         body:
+          seq_no_primary_term: true
           aggs:
             cluster:
               terms:
                 field: f1.keyword
 
   - match: { _shards.total: 5 }
   - match: { hits.total: 11 }
+  - gte: { hits.hits.0._seq_no: 0 }
+  - gte: { hits.hits.0._primary_term: 1 }
   - length: { aggregations.cluster.buckets: 2 }
   - match: { aggregations.cluster.buckets.0.key: "remote_cluster" }
   - match: { aggregations.cluster.buckets.0.doc_count: 6 }

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
@@ -164,6 +164,10 @@
           "type" : "boolean",
           "description" : "Specify whether to return document version as part of a hit"
         },
+        "seq_no_primary_term": {
+          "type" : "boolean",
+          "description" : "Specify whether to return sequence number and primary term of the last modification of each hit"
+        },
         "request_cache": {
           "type" : "boolean",
           "description" : "Specify if request cache should be used for this request or not, defaults to index level setting"

diff --git a/...api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml b/...api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml
@@ -1,8 +1,4 @@
----
-"top_hits aggregation with nested documents":
-  - skip:
-          version: " - 6.1.99"
-          reason: "<= 6.1 nodes don't always include index or id in nested top hits"
+setup:
   - do:
       indices.create:
         index: my-index
@@ -54,6 +50,12 @@
             ]
           }
 
+---
+"top_hits aggregation with nested documents":
+  - skip:
+          version: " - 6.1.99"
+          reason: "<= 6.1 nodes don't always include index or id in nested top hits"
+
   - do:
       search:
         body:
@@ -80,3 +82,35 @@
   - match: { aggregations.to-users.users.hits.hits.2._index: my-index }
   - match: { aggregations.to-users.users.hits.hits.2._nested.field: users }
   - match: { aggregations.to-users.users.hits.hits.2._nested.offset: 1 }
+
+
+---
+"top_hits aggregation with sequence numbers":
+  - skip:
+      version: " - 6.6.99"
+      reason: support was added in 6.7
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          aggs:
+            groups:
+              terms:
+                field: group.keyword
+              aggs:
+                users:
+                  top_hits:
+                    sort: "users.last.keyword"
+                    seq_no_primary_term: true
+
+  - match: { hits.total: 2 }
+  - length: { aggregations.groups.buckets.0.users.hits.hits: 2 }
+  - match: { aggregations.groups.buckets.0.users.hits.hits.0._id: "1" }
+  - match: { aggregations.groups.buckets.0.users.hits.hits.0._index: my-index }
+  - gte: { aggregations.groups.buckets.0.users.hits.hits.0._seq_no: 0 }
+  - gte: { aggregations.groups.buckets.0.users.hits.hits.0._primary_term: 1 }
+  - match: { aggregations.groups.buckets.0.users.hits.hits.1._id: "2" }
+  - match: { aggregations.groups.buckets.0.users.hits.hits.1._index: my-index }
+  - gte: { aggregations.groups.buckets.0.users.hits.hits.1._seq_no: 0 }
+  - gte: { aggregations.groups.buckets.0.users.hits.hits.1._primary_term: 1 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
@@ -423,3 +423,57 @@ setup:
   - match: { hits.hits.1.inner_hits.sub_hits.hits.total: 3}
   - match: { hits.hits.2.fields.group_alias: [25] }
   - match: { hits.hits.2.inner_hits.sub_hits.hits.total: 2}
+
+---
+"field collapsing, inner_hits and seq_no":
+
+  - skip:
+      version: " - 6.99.0"
+      reason:  "sequence numbers introduced in 7.0.0"
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group, inner_hits: {
+            name: sub_hits, seq_no_primary_term: true, size: 2, sort: [{ sort: asc }]
+          } }
+          sort: [{ sort: desc }]
+
+  - match: { hits.total: 6 }
+  - length: { hits.hits: 3 }
+  - match: { hits.hits.0._index: test }
+  - match: { hits.hits.0.fields.numeric_group: [3] }
+  - match: { hits.hits.0.sort: [36] }
+  - match: { hits.hits.0._id: "6" }
+  - match: { hits.hits.0.inner_hits.sub_hits.hits.total: 1 }
+  - length: { hits.hits.0.inner_hits.sub_hits.hits.hits: 1 }
+  - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" }
+  - gte: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._seq_no: 0 }
+  - gte: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._primary_term: 1 }
+  - match: { hits.hits.1._index: test }
+  - match: { hits.hits.1.fields.numeric_group: [1] }
+  - match: { hits.hits.1.sort: [24] }
+  - match: { hits.hits.1._id: "3" }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.total: 3 }
+  - length: { hits.hits.1.inner_hits.sub_hits.hits.hits: 2 }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "2" }
+  - gte: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._seq_no: 0 }
+  - gte: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._primary_term: 1 }
+  - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" }
+  - gte: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._seq_no: 0 }
+  - gte: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._primary_term: 1 }
+  - match: { hits.hits.2._index: test }
+  - match: { hits.hits.2._type: test }
+  - match: { hits.hits.2.fields.numeric_group: [25] }
+  - match: { hits.hits.2.sort: [10] }
+  - match: { hits.hits.2._id: "4" }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.total: 2 }
+  - length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "5" }
+  - gte: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._seq_no: 0 }
+  - gte: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._primary_term: 1 }
+  - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" }
+  - gte: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._seq_no: 0 }
+  - gte: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._primary_term: 1 }