Skip to content

Commit

Permalink
Fetch meta fields in FetchFieldsPhase using ValueFetcher (#106325)
Browse files Browse the repository at this point in the history
Here we extract the logic to populate metadata fields such as _ignored, _routing, _size and the deprecated _type into FetchFieldsPhase so that we can use the ValueFetcher interface to retrieve field values. This allows us to fetch values no matter if the Mapper uses stored or doc values.
  • Loading branch information
salvatore-campagna committed Apr 15, 2024
1 parent b9322da commit 4dfcb08
Show file tree
Hide file tree
Showing 7 changed files with 453 additions and 65 deletions.
22 changes: 22 additions & 0 deletions docs/reference/search/profile.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,17 @@ The API returns the following result:
"stored_fields": ["_id", "_routing", "_source"]
},
"children": [
{
"type" : "FetchFieldsPhase",
"description" : "",
"time_in_nanos" : 238762,
"breakdown" : {
"process_count" : 5,
"process" : 227914,
"next_reader" : 10848,
"next_reader_count" : 1
}
},
{
"type": "FetchSourcePhase",
"description": "",
Expand Down Expand Up @@ -1043,6 +1054,17 @@ And here is the fetch profile:
"stored_fields": ["_id", "_routing", "_source"]
},
"children": [
{
"type" : "FetchFieldsPhase",
"description" : "",
"time_in_nanos" : 238762,
"breakdown" : {
"process_count" : 5,
"process" : 227914,
"next_reader" : 10848,
"next_reader_count" : 1
}
},
{
"type": "FetchSourcePhase",
"description": "",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,8 @@ teardown:
---
profile fetch:
- skip:
version: ' - 7.15.99'
reason: fetch profiling implemented in 7.16.0
version: ' - 8.13.99'
reason: fetch fields and stored_fields using ValueFetcher

- do:
search:
Expand All @@ -141,15 +141,20 @@ profile fetch:
- gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 }
- gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 }
- match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] }
- length: { profile.shards.0.fetch.children: 3 }
- match: { profile.shards.0.fetch.children.0.type: FetchSourcePhase }
- length: { profile.shards.0.fetch.children: 4 }
- match: { profile.shards.0.fetch.children.0.type: FetchFieldsPhase }
- gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
- gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
- match: { profile.shards.0.fetch.children.1.type: InnerHitsPhase }
- match: { profile.shards.0.fetch.children.1.type: FetchSourcePhase }
- gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
- gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
- match: { profile.shards.0.fetch.children.2.type: StoredFieldsPhase }
- match: { profile.shards.0.fetch.children.2.type: InnerHitsPhase }
- gt: { profile.shards.0.fetch.children.2.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.2.breakdown.next_reader: 0 }
- gt: { profile.shards.0.fetch.children.2.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.2.breakdown.next_reader: 0 }
- match: { profile.shards.0.fetch.children.3.type: StoredFieldsPhase }
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
---
"_ignored field through get api using stored_fields":
- do:
indices.create:
index: test
body:
mappings:
properties:
keyword:
type: keyword
ignore_above: 5
ip:
type: ip
ignore_malformed: true
value:
type: long
ignore_malformed: true

- do:
index:
index: test
id: 1
refresh: true
body:
keyword: foo
ip: 192.168.0.1
value: 23
- do:
index:
index: test
id: 2
refresh: true
body:
keyword: foobar
ip: garbage
value: missing
- do:
index:
index: test
id: 3
refresh: true
body:
keyword:
- foo
- bar
- foobar
ip:
- 10.10.1.1
- 192.8.1.2
- 199.199.300.999
value:
- 1
- 2
- ops

- do:
get:
index: test
id: 1

- match: {_index: "test"}
- match: {_id: "1"}
- match: {_version: 1}
- match: {found: true}
- match:
_source:
keyword: foo
ip: 192.168.0.1
value: 23

- is_false: fields

- do:
get:
index: test
id: 2
- match: { _index: "test" }
- match: { _id: "2" }
- match: { _version: 1 }
- match: { found: true }
- match:
_source:
ip: garbage
keyword: foobar
value: missing

- is_false: fields

- do:
get:
index: test
id: 3
- match: { _index: "test" }
- match: { _id: "3" }
- match: { _version: 1 }
- match: { found: true }
- match:
_source:
ip:
- 10.10.1.1
- 192.8.1.2
- 199.199.300.999
keyword:
- foo
- bar
- foobar
value:
- 1
- 2
- ops

- is_false: fields

- do:
get:
index: test
id: 1
stored_fields:
- _ignored

- match: { _index: "test" }
- match: { _id: "1" }
- match: { _version: 1 }
- match: { found: true }
- match: { _ignored: null}

- do:
get:
index: test
id: 2
stored_fields:
- _ignored

- match: { _index: "test" }
- match: { _id: "2" }
- match: { _version: 1 }
- match: { found: true }
- match:
_ignored:
- ip
- keyword
- value

- do:
get:
index: test
id: 3
stored_fields:
- _ignored

- match: { _index: "test" }
- match: { _id: "3" }
- match: { _version: 1 }
- match: { found: true }
- match:
_ignored:
- ip
- keyword
- value
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ setup:
---
fetch fields:
- skip:
version: ' - 8.5.99'
reason: stored fields phase added in 8.6
version: ' - 8.13.99'
reason: fetch fields and stored_fields using ValueFetcher

- do:
search:
Expand All @@ -44,17 +44,21 @@ fetch fields:
- match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] }
- length: { profile.shards.0.fetch.children: 2 }
- match: { profile.shards.0.fetch.children.0.type: FetchFieldsPhase }
- match: { profile.shards.0.fetch.children.1.type: StoredFieldsPhase }
- gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
- gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
- match: { profile.shards.0.fetch.children.1.type: StoredFieldsPhase }
- gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
- gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }

---
fetch source:
- skip:
version: ' - 8.5.99'
reason: stored fields phase added in 8.6
version: ' - 8.13.99'
reason: fetch fields and stored_fields using ValueFetcher

- do:
search:
Expand All @@ -71,20 +75,21 @@ fetch source:
- gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 }
- gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 }
- match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] }
- length: { profile.shards.0.fetch.children: 2 }
- match: { profile.shards.0.fetch.children.0.type: FetchSourcePhase }
- gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
- gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
- match: { profile.shards.0.fetch.children.0.debug.fast_path: 1 }
- match: { profile.shards.0.fetch.children.1.type: StoredFieldsPhase }
- length: { profile.shards.0.fetch.children: 3 }
- match: { profile.shards.0.fetch.children.0.type: FetchFieldsPhase }
- match: { profile.shards.0.fetch.children.1.type: FetchSourcePhase }
- gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
- gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
- match: { profile.shards.0.fetch.children.1.debug.fast_path: 1 }
- match: { profile.shards.0.fetch.children.2.type: StoredFieldsPhase }

---
fetch nested source:
- skip:
version: ' - 8.5.99'
reason: stored fields phase added in 8.6
version: ' - 8.13.99'
reason: fetch fields and stored_fields using ValueFetcher

- do:
indices.create:
Expand Down Expand Up @@ -135,24 +140,25 @@ fetch nested source:
- gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 }
- gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 }
- match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] }
- length: { profile.shards.0.fetch.children: 3 }
- match: { profile.shards.0.fetch.children.0.type: FetchSourcePhase }
- gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
- gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.0.breakdown.next_reader: 0 }
- match: { profile.shards.0.fetch.children.1.type: InnerHitsPhase }
- length: { profile.shards.0.fetch.children: 4 }
- match: { profile.shards.0.fetch.children.0.type: FetchFieldsPhase }
- match: { profile.shards.0.fetch.children.1.type: FetchSourcePhase }
- gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
- gt: { profile.shards.0.fetch.children.1.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.1.breakdown.next_reader: 0 }
- match: { profile.shards.0.fetch.children.2.type: StoredFieldsPhase }
- match: { profile.shards.0.fetch.children.2.type: InnerHitsPhase }
- gt: { profile.shards.0.fetch.children.2.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.2.breakdown.next_reader: 0 }
- gt: { profile.shards.0.fetch.children.2.breakdown.next_reader_count: 0 }
- gt: { profile.shards.0.fetch.children.2.breakdown.next_reader: 0 }
- match: { profile.shards.0.fetch.children.3.type: StoredFieldsPhase }

---
disabling stored fields removes fetch sub phases:
- skip:
version: ' - 7.15.99'
reason: fetch profiling implemented in 7.16.0
reason: fetch profiling implemented in 7.16.0

- do:
search:
Expand Down

0 comments on commit 4dfcb08

Please sign in to comment.