From 5a08dad5dee71735061486a349cfb0eb47e84e51 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Thu, 7 Nov 2024 09:23:28 -0500 Subject: [PATCH] [ESQL] Fix Binary Comparisons on Date Nanos (#116346) Relates to #109992 When I implemented binary comparisons for date nanos in #111908 I failed to update the verifyBinaryComparisons rule, which implements another layer of type checking for binary comparisons, external to the type logic in the Expression classes. Since the unit tests for the expressions don't run the full query processing stack, this bug was invisible to them. Only full integration tests, such as the CSV tests I've added here, can catch this kind of error. The initial PR did not include these because we didn't have enough supporting functions (notably TO_DATE_NANOS) to write the tests. This PR fixes the issue by adding DATE_NANOS to the list of allowed types for binary comparisons in the verifier. --- docs/changelog/116346.yaml | 5 ++ .../src/main/resources/date_nanos.csv-spec | 77 +++++++++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 5 ++ .../xpack/esql/analysis/Verifier.java | 1 + 4 files changed, 88 insertions(+) create mode 100644 docs/changelog/116346.yaml diff --git a/docs/changelog/116346.yaml b/docs/changelog/116346.yaml new file mode 100644 index 0000000000000..1dcace88a98c0 --- /dev/null +++ b/docs/changelog/116346.yaml @@ -0,0 +1,5 @@ +pr: 116346 +summary: "[ESQL] Fix Binary Comparisons on Date Nanos" +area: ES|QL +type: bug +issues: [] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index 77883597c9f19..56689718f2366 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -202,6 +202,83 @@ d:date_nanos null ; +date nanos greater than +required_capability: to_date_nanos +required_capability: date_nanos_binary_comparison + +FROM date_nanos | WHERE MV_MIN(nanos) > TO_DATE_NANOS("2023-10-23T12:27:28.948000000Z") | SORT nanos DESC; + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654 +2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878 +2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837 +2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000 +; + +date nanos greater than or equal +required_capability: to_date_nanos +required_capability: date_nanos_binary_comparison + +FROM date_nanos | WHERE MV_MIN(nanos) >= TO_DATE_NANOS("2023-10-23T12:27:28.948000000Z") | SORT nanos DESC; + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654 +2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878 +2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837 +2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000 +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; + +date nanos less than +required_capability: to_date_nanos +required_capability: date_nanos_binary_comparison + +FROM date_nanos | WHERE MV_MIN(nanos) < TO_DATE_NANOS("2023-10-23T12:27:28.948000000Z") AND millis > "2000-01-01" | SORT nanos DESC; + +millis:date | nanos:date_nanos | num:long +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +; + +date nanos less than equal +required_capability: to_date_nanos +required_capability: date_nanos_binary_comparison + +FROM date_nanos | WHERE MV_MIN(nanos) <= TO_DATE_NANOS("2023-10-23T12:27:28.948000000Z") AND millis > "2000-01-01" | SORT nanos DESC; + +millis:date | nanos:date_nanos | num:long +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +; + +date nanos equals +required_capability: to_date_nanos +required_capability: date_nanos_binary_comparison + +FROM date_nanos | WHERE MV_MIN(nanos) == TO_DATE_NANOS("2023-10-23T12:27:28.948000000Z"); + +millis:date | nanos:date_nanos | num:long +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; + +date nanos not equals +required_capability: to_date_nanos +required_capability: date_nanos_binary_comparison + +FROM date_nanos | WHERE MV_MIN(nanos) != TO_DATE_NANOS("2023-10-23T12:27:28.948000000Z") AND millis > "2000-01-01" | SORT nanos DESC; + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T13:53:55.832Z | 2023-10-23T13:53:55.832987654Z | 1698069235832987654 +2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015787878Z | 1698069175015787878 +2023-10-23T13:51:54.732Z | 2023-10-23T13:51:54.732102837Z | 1698069114732102837 +2023-10-23T13:33:34.937Z | 2023-10-23T13:33:34.937193000Z | 1698068014937193000 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +; date nanos to long, index version required_capability: to_date_nanos diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 93d8168228e7b..811b8bad4af04 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -318,6 +318,11 @@ public enum Cap { */ TO_DATE_NANOS(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG), + /** + * Support for date nanos type in binary comparisons + */ + DATE_NANOS_BINARY_COMPARISON(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG), + /** * Support Least and Greatest functions on Date Nanos type */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 0fa2a69438358..632f52d163349 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -507,6 +507,7 @@ public static Failure validateBinaryComparison(BinaryComparison bc) { } allowed.add(DataType.IP); allowed.add(DataType.DATETIME); + allowed.add(DataType.DATE_NANOS); allowed.add(DataType.VERSION); allowed.add(DataType.GEO_POINT); allowed.add(DataType.GEO_SHAPE);