From 216c6675e6dc898b491636768840add3f2d99428 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Wed, 18 Dec 2024 15:07:55 +0100 Subject: [PATCH 1/4] Ensure nested field could be used in lookup joins --- .../xpack/esql/CsvTestsDataLoader.java | 2 ++ .../resources/languages_nested_fields.csv | 5 +++++ .../src/main/resources/lookup-join.csv-spec | 22 +++++++++++++++++++ 3 files changed, 29 insertions(+) create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_nested_fields.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 8e81d14b4dfd7..1648b965c6bab 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -64,6 +64,8 @@ public class CsvTestsDataLoader { .withSetting("languages_lookup-settings.json"); private static final TestsDataset LANGUAGES_LOOKUP_NON_UNIQUE_KEY = LANGUAGES_LOOKUP.withIndex("languages_lookup_non_unique_key") .withData("languages_non_unique_key.csv"); + private static final TestsDataset LANGUAGES_NESTED_FIELDS = LANGUAGES_LOOKUP.withIndex("languages_nested_fields") + .withData("languages_nested_fields.csv"); private static final TestsDataset ALERTS = new TestsDataset("alerts"); private static final TestsDataset UL_LOGS = new TestsDataset("ul_logs"); private static final TestsDataset SAMPLE_DATA = new TestsDataset("sample_data"); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_nested_fields.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_nested_fields.csv new file mode 100644 index 0000000000000..f26114245a6d3 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_nested_fields.csv @@ -0,0 +1,5 @@ +_id:integer,language.id:integer,language.name:keyword,language.code:keyword +1,1,English,EN +2,2,French,FR +3,3,Spanish,ES +4,4,German,DE diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 7d4f89ed920a9..8b69d97a43bcd 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -416,6 +416,28 @@ language_code:integer | language_name:keyword | country:keyword 8 | Mv-Lang2 | Mv-Land2 ; +########################################################################### +# nested filed join behavior with languages_nested_fields index +########################################################################### + +joinOnNestedField +required_capability: join_lookup_v8 + +FROM employees +| WHERE 10000 < emp_no AND emp_no < 10005 +| EVAL language.id = emp_no % 10 +| LOOKUP JOIN languages_nested_fields ON language.id +| SORT emp_no +| KEEP emp_no, language.id, language.name +; + +emp_no:integer | language.id:integer | language.name:keyword +10001 | 1 | English +10002 | 2 | French +10003 | 3 | Spanish +10004 | 4 | German +; + ############################################### # Tests with clientips_lookup index ############################################### From 861211e64c3e1ffef32f1822cc42332e33a98072 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Wed, 18 Dec 2024 16:08:34 +0100 Subject: [PATCH 2/4] fix mapping --- .../xpack/esql/CsvTestsDataLoader.java | 8 ++++++-- .../mapping-languages_nested_fields.json | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-languages_nested_fields.json diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 1648b965c6bab..e786c87a9eee5 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -64,8 +64,11 @@ public class CsvTestsDataLoader { .withSetting("languages_lookup-settings.json"); private static final TestsDataset LANGUAGES_LOOKUP_NON_UNIQUE_KEY = LANGUAGES_LOOKUP.withIndex("languages_lookup_non_unique_key") .withData("languages_non_unique_key.csv"); - private static final TestsDataset LANGUAGES_NESTED_FIELDS = LANGUAGES_LOOKUP.withIndex("languages_nested_fields") - .withData("languages_nested_fields.csv"); + private static final TestsDataset LANGUAGES_NESTED_FIELDS = new TestsDataset( + "languages_nested_fields", + "mapping-languages_nested_fields.json", + "languages_nested_fields.csv" + ).withSetting("languages_lookup-settings.json"); private static final TestsDataset ALERTS = new TestsDataset("alerts"); private static final TestsDataset UL_LOGS = new TestsDataset("ul_logs"); private static final TestsDataset SAMPLE_DATA = new TestsDataset("sample_data"); @@ -118,6 +121,7 @@ public class CsvTestsDataLoader { Map.entry(LANGUAGES.indexName, LANGUAGES), Map.entry(LANGUAGES_LOOKUP.indexName, LANGUAGES_LOOKUP), Map.entry(LANGUAGES_LOOKUP_NON_UNIQUE_KEY.indexName, LANGUAGES_LOOKUP_NON_UNIQUE_KEY), + Map.entry(LANGUAGES_NESTED_FIELDS.indexName, LANGUAGES_NESTED_FIELDS), Map.entry(UL_LOGS.indexName, UL_LOGS), Map.entry(SAMPLE_DATA.indexName, SAMPLE_DATA), Map.entry(MV_SAMPLE_DATA.indexName, MV_SAMPLE_DATA), diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-languages_nested_fields.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-languages_nested_fields.json new file mode 100644 index 0000000000000..8655aa80245ca --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-languages_nested_fields.json @@ -0,0 +1,17 @@ +{ + "properties" : { + "language" : { + "properties" : { + "id": { + "type": "integer" + }, + "name": { + "type": "keyword" + }, + "code": { + "type": "keyword" + } + } + } + } +} From 0cfc760996f46354b4688002ad0aad122825d242 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Wed, 18 Dec 2024 16:38:45 +0100 Subject: [PATCH 3/4] add null case --- .../qa/testFixtures/src/main/resources/lookup-join.csv-spec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 8b69d97a43bcd..0c20c70ee589b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -424,7 +424,7 @@ joinOnNestedField required_capability: join_lookup_v8 FROM employees -| WHERE 10000 < emp_no AND emp_no < 10005 +| WHERE 10000 < emp_no AND emp_no < 10006 | EVAL language.id = emp_no % 10 | LOOKUP JOIN languages_nested_fields ON language.id | SORT emp_no @@ -436,6 +436,7 @@ emp_no:integer | language.id:integer | language.name:keyword 10002 | 2 | French 10003 | 3 | Spanish 10004 | 4 | German +10005 | 5 | null ; ############################################### From 50813812dd85c7adce483b0fdf2a15d99b6604ae Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Thu, 19 Dec 2024 11:29:00 +0100 Subject: [PATCH 4/4] add more samples --- .../resources/languages_nested_fields.csv | 2 +- .../src/main/resources/lookup-join.csv-spec | 28 ++++++++++++++++++- .../mapping-languages_nested_fields.json | 7 ++++- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_nested_fields.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_nested_fields.csv index f26114245a6d3..154125cf49304 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_nested_fields.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_nested_fields.csv @@ -1,4 +1,4 @@ -_id:integer,language.id:integer,language.name:keyword,language.code:keyword +_id:integer,language.id:integer,language.name:text,language.code:keyword 1,1,English,EN 2,2,French,FR 3,3,Spanish,ES diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 0c20c70ee589b..618149f2c3dde 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -431,7 +431,7 @@ FROM employees | KEEP emp_no, language.id, language.name ; -emp_no:integer | language.id:integer | language.name:keyword +emp_no:integer | language.id:integer | language.name:text 10001 | 1 | English 10002 | 2 | French 10003 | 3 | Spanish @@ -439,6 +439,32 @@ emp_no:integer | language.id:integer | language.name:keyword 10005 | 5 | null ; + +joinOnNestedFieldRow +required_capability: join_lookup_v8 + +ROW language.code = "EN" +| LOOKUP JOIN languages_nested_fields ON language.code +| KEEP language.id, language.code, language.name.keyword +; + +language.id:integer | language.code:keyword | language.name.keyword:keyword +1 | EN | English +; + + +joinOnNestedNestedFieldRow +required_capability: join_lookup_v8 + +ROW language.name.keyword = "English" +| LOOKUP JOIN languages_nested_fields ON language.name.keyword +| KEEP language.id, language.name, language.name.keyword +; + +language.id:integer | language.name:text | language.name.keyword:keyword +1 | English | English +; + ############################################### # Tests with clientips_lookup index ############################################### diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-languages_nested_fields.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-languages_nested_fields.json index 8655aa80245ca..9b46a85ed8d11 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-languages_nested_fields.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-languages_nested_fields.json @@ -6,7 +6,12 @@ "type": "integer" }, "name": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } }, "code": { "type": "keyword"