Sort segments on timestamp in read only engine (#93576)

Ordinary indices support sorting their segments on the timestamp when they have such a field defined
in their mappings (see #75195). This was not initially supported as a directory reader could not be created provding
a leaf sorter, which is now possible in Lucene and we can make use of.

docs/changelog/93576.yaml

@@ -0,0 +1,5 @@
+pr: 93576
+summary: Sort segments on timestamp in read only engine
+area: Engine
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java

@@ -213,13 +213,16 @@ protected final ElasticsearchDirectoryReader wrapReader(
     }
 
     protected DirectoryReader open(IndexCommit commit) throws IOException {
-        // TODO: provide engineConfig.getLeafSorter() when opening a DirectoryReader from a commit
-        // should be available from Lucene v 8.10
         assert Transports.assertNotTransportThread("opening index commit of a read-only engine");
+        DirectoryReader directoryReader = DirectoryReader.open(
+            commit,
+            org.apache.lucene.util.Version.MIN_SUPPORTED_MAJOR,
+            engineConfig.getLeafSorter()
+        );
         if (lazilyLoadSoftDeletes) {
-            return new LazySoftDeletesDirectoryReaderWrapper(DirectoryReader.open(commit), Lucene.SOFT_DELETES_FIELD);
+            return new LazySoftDeletesDirectoryReaderWrapper(directoryReader, Lucene.SOFT_DELETES_FIELD);
         } else {
-            return new SoftDeletesDirectoryReaderWrapper(DirectoryReader.open(commit), Lucene.SOFT_DELETES_FIELD);
+            return new SoftDeletesDirectoryReaderWrapper(directoryReader, Lucene.SOFT_DELETES_FIELD);
         }
     }
 

x-pack/plugin/searchable-snapshots/qa/rest/build.gradle

@@ -10,7 +10,7 @@ final File repoDir = file("$buildDir/testclusters/repo")
 
 restResources {
   restApi {
-    include 'indices', 'search', 'bulk', 'snapshot', 'nodes', '_common', 'searchable_snapshots', 'cluster', 'open_point_in_time', 'close_point_in_time', 'security'
+    include 'indices', 'search', 'bulk', 'index', 'snapshot', 'nodes', '_common', 'searchable_snapshots', 'cluster', 'open_point_in_time', 'close_point_in_time', 'security'
   }
 }
 

x-pack/plugin/searchable-snapshots/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/sort_segments_on_timestamp.yml

@@ -0,0 +1,87 @@
+---
+setup:
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+          settings:
+            number_of_shards: 1
+
+  # 1st segment
+  - do:
+      index:
+        index: test
+        body: { "foo": "bar1", "@timestamp": "2021-08-01" }
+        refresh: true
+
+  # 2nd segment
+  - do:
+      index:
+        index: test
+        body: { "foo": "bar2", "@timestamp": "2021-08-02" }
+        refresh: true
+
+  # 3rd segment missing @timestamp field
+  - do:
+      index:
+        index: test
+        body: { "foo": "bar1"}
+        refresh: true
+
+  - do:
+      snapshot.create_repository:
+        repository: repository-fs
+        body:
+          type: fs
+          settings:
+            location: "repository-fs"
+
+  # Remove the snapshot if a previous test failed to delete it.
+  # Useful for third party tests that runs the test against a real external service.
+  - do:
+      snapshot.delete:
+        repository: repository-fs
+        snapshot: snapshot
+        ignore: 404
+
+  - do:
+      snapshot.create:
+        repository: repository-fs
+        snapshot: snapshot
+        wait_for_completion: true
+
+  - do:
+      indices.delete:
+        index: test
+
+---
+"Test that index segments are sorted on timestamp field if @timestamp field is defined in mapping":
+
+  - do:
+      searchable_snapshots.mount:
+        repository: repository-fs
+        snapshot: snapshot
+        wait_for_completion: true
+        body:
+          index: test
+          renamed_index: test-default-storage
+
+  - match: { snapshot.snapshot: snapshot }
+  - match: { snapshot.shards.failed: 0 }
+  - match: { snapshot.shards.successful: 1 }
+
+  # test that segments are sorted by @timestamp DESC
+  - do:
+      search:
+        index: test-default-storage
+        body:
+          fields: [{ "field":"@timestamp", "format":"yyyy-MM-dd" }]
+  - match: { hits.total.value: 3 }
+  - match: { hits.hits.0.fields.@timestamp: ["2021-08-02"] }
+  - match: { hits.hits.1.fields.@timestamp: ["2021-08-01"] }
+  - is_false: hits.hits.2.fields.@timestamp