[Feature](ES): Es support sort node pushdown limit and date type

docker/thirdparties/docker-compose/elasticsearch/es.yaml.tpl

@@ -29,7 +29,7 @@ services:
       discovery.type: "single-node"
       ELASTICSEARCH_XPACK_SECURITY_ENABLED: "false"
     volumes:
-      - ./data/es6/:/usr/share/elasticsearch/data
+      - ./data/es6/:/var/lib/elasticsearch/data
     networks:
       - doris--es
     healthcheck:

docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1.json

@@ -2,5 +2,6 @@
   "test1": "string1",
   "test2": "text#1",
   "test3": 3.14,
-  "test4": "2022-08-08"
+  "test4": "2022-08-08",
+  "test5": "2023-02-10 22:00:01"
 }

docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2.json

@@ -2,5 +2,6 @@
   "test1": "string2",
   "test2": "text2",
   "test3": 4,
-  "test4": "2022-09-08"
+  "test4": "2022-09-08",
+  "test5": "2023-02-10 22:02:01"
 }

docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3.json

@@ -2,5 +2,6 @@
   "test1": "string3",
   "test2": "text3_4*5",
   "test3": 5.0,
-  "test4": "2022-08-08"
+  "test4": "2022-08-08",
+  "test5": "2023-02-10 22:03:01"
 }

docker/thirdparties/docker-compose/elasticsearch/scripts/data/data4.json

@@ -0,0 +1,7 @@
+{
+  "test1": "string4",
+  "test2": null,
+  "test3": 5.0,
+  "test4": "2023-02-10",
+  "test5": "2023-02-10 22:04:01"
+}

docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh

@@ -26,9 +26,11 @@ curl "http://${ES_6_HOST}:9200/test2_20220808" -H "Content-Type:application/json
 curl "http://${ES_6_HOST}:9200/test1/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
 curl "http://${ES_6_HOST}:9200/test1/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
 curl "http://${ES_6_HOST}:9200/test1/doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
+curl "http://${ES_6_HOST}:9200/test1/doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
 curl "http://${ES_6_HOST}:9200/test2_20220808/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
 curl "http://${ES_6_HOST}:9200/test2_20220808/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
 curl "http://${ES_6_HOST}:9200/test2_20220808/doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
+curl "http://${ES_6_HOST}:9200/test2_20220808/doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
 
 # es7
 # create index test1
@@ -39,9 +41,11 @@ curl "http://${ES_7_HOST}:9200/test2_20220808" -H "Content-Type:application/json
 curl "http://${ES_7_HOST}:9200/test1/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
 curl "http://${ES_7_HOST}:9200/test1/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
 curl "http://${ES_7_HOST}:9200/test1/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
+curl "http://${ES_7_HOST}:9200/test1/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
 curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
 curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
 curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
+curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
 
 # es8
 # create index test1
@@ -52,6 +56,8 @@ curl "http://${ES_8_HOST}:9200/test2_20220808" -H "Content-Type:application/json
 curl "http://${ES_8_HOST}:9200/test1/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
 curl "http://${ES_8_HOST}:9200/test1/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
 curl "http://${ES_8_HOST}:9200/test1/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
+curl "http://${ES_8_HOST}:9200/test1/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'
 curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json'
 curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json'
 curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json'
+curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json'

docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test1.json

@@ -22,7 +22,12 @@
           "type": "double"
         },
         "test4": {
-          "type": "date"
+          "type": "date",
+          "format": "yyyy-MM-dd"
+        },
+        "test5": {
+          "type": "date",
+          "format": "yyyy-MM-dd HH:mm:ss"
         }
       }
     }

docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test2.json

@@ -25,7 +25,12 @@
           "type": "double"
         },
         "test4": {
-          "type": "date"
+          "type": "date",
+          "format": "yyyy-MM-dd"
+        },
+        "test5": {
+          "type": "date",
+          "format": "yyyy-MM-dd HH:mm:ss"
         }
       }
     }

docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test1.json

@@ -21,7 +21,12 @@
         "type": "double"
       },
       "test4": {
-        "type": "date"
+        "type": "date",
+        "format": "yyyy-MM-dd"
+      },
+      "test5": {
+        "type": "date",
+        "format": "yyyy-MM-dd HH:mm:ss"
       }
     }
   }

docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test2.json

@@ -24,7 +24,12 @@
         "type": "double"
       },
       "test4": {
-        "type": "date"
+        "type": "date",
+        "format": "yyyy-MM-dd"
+      },
+      "test5": {
+        "type": "date",
+        "format": "yyyy-MM-dd HH:mm:ss"
       }
     }
   }

fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java

@@ -446,7 +446,6 @@ public static Type toDorisType(String esType) {
             case "scaled_float":
                 return Type.DOUBLE;
             case "date":
-                return ScalarType.createDateV2Type();
             case "keyword":
             case "text":
             case "ip":

fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java

@@ -1940,6 +1940,8 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s
                 throw new RuntimeException("Broker external table is not supported, try to use table function please");
             case ELASTICSEARCH:
                 scanNode = new EsScanNode(ctx.getNextNodeId(), tblRef.getDesc(), "EsScanNode");
+                // push down limit to elasticsearch
+                scanNode.setLimit(selectStmt.getLimit() + selectStmt.getOffset());
                 break;
             case HIVE:
                 throw new RuntimeException("Hive external table is not supported, try to use hive catalog please");
@@ -1961,6 +1963,8 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s
                 break;
             case ES_EXTERNAL_TABLE:
                 scanNode = new EsScanNode(ctx.getNextNodeId(), tblRef.getDesc(), "EsScanNode", true);
+                // push down limit to elasticsearch
+                scanNode.setLimit(selectStmt.getLimit() + selectStmt.getOffset());
                 break;
             case JDBC_EXTERNAL_TABLE:
                 scanNode = new JdbcScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true);

regression-test/data/es_p0/test_es_query.out

@@ -1,21 +1,28 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !sql62 --
-2022-08-08	text#1	3.14	string1
+2022-08-08T00:00:00.000Z	2023-02-10T22:00:01.000Z	text#1	3.14	string1
 
 -- !sql63 --
-2022-08-08	text#1	3.14	string1
-2022-08-08	text3_4*5	5.0	string3
+2022-08-08T00:00:00.000Z	2023-02-10T22:00:01.000Z	text#1	3.14	string1
+2022-08-08T00:00:00.000Z	2023-02-10T22:03:01.000Z	text3_4*5	5.0	string3
 
 -- !sql64 --
-2022-09-08	text2	4.0	string2
+2022-09-08T00:00:00.000Z	2023-02-10T22:02:01.000Z	text2	4.0	string2
 
 -- !sql72 --
-2022-08-08	text#1	3.14	string1
+2022-08-08	2023-02-10 22:00:01	text#1	3.14	string1
 
 -- !sql73 --
-2022-08-08	text#1	3.14	string1
-2022-08-08	text3_4*5	5.0	string3
+2022-08-08	2023-02-10 22:00:01	text#1	3.14	string1
+2022-08-08	2023-02-10 22:03:01	text3_4*5	5.0	string3
 
 -- !sql74 --
-2022-09-08	text2	4.0	string2
+2022-09-08	2023-02-10 22:02:01	text2	4.0	string2
+
+-- !sql81 --
+2022-08-08	2023-02-10 22:00:01	text#1	3.14	string1
+
+-- !sql82 --
+2022-08-08	2023-02-10 22:00:01	text#1	3.14	string1
+2022-08-08	2023-02-10 22:03:01	text3_4*5	5.0	string3
 

regression-test/suites/es_p0/test_es_query.groovy

@@ -66,16 +66,18 @@ suite("test_es_query", "p0") {
         order_qt_sql72 """select * from test1 where test2='text#1'"""
         order_qt_sql73 """select * from test2_20220808 where test4='2022-08-08'"""
         order_qt_sql74 """select * from test2_20220808 where substring(test2, 2) = 'ext2'"""
-        // es8 has some problem, need fix
-        // sql """switch es8"""
-        // order_qt_sql1 """select * from test1 where test2='text'"""
-        // order_qt_sql2 """select * from test2_20220808 where test4='2022-08-08'"""
-
+        sql """switch es8"""
+        order_qt_sql81 """select * from test1 where test2='text#1'"""
+        order_qt_sql82 """select * from test2_20220808 where test4='2022-08-08'"""
+        explain {
+            sql("select * from test1 order by test1 limit 2,2")
+            contains "limit: 4"
+        }
         sql """drop catalog if exists es6;"""
         sql """drop catalog if exists es7;"""
         sql """drop catalog if exists es8;"""
         sql """drop resource if exists es6_resource;"""
         sql """drop resource if exists es7_resource;"""
         sql """drop resource if exists es8_resource;"""
     }
-}
+}