PromQL Support in ESQL

.gitattributes

@@ -4,12 +4,13 @@ CHANGELOG.asciidoc  merge=union
 # Windows
 build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/*.asciidoc text eol=lf
 
+# ESQL parsing and source generated related assets
 x-pack/plugin/esql/compute/src/main/generated/** linguist-generated=true
 x-pack/plugin/esql/compute/src/main/generated-src/** linguist-generated=true
 x-pack/plugin/esql/src/main/antlr/*.tokens linguist-generated=true
 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/*.interp linguist-generated=true
-x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer*.java linguist-generated=true
-x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser*.java linguist-generated=true
+x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/*BaseLexer*.java linguist-generated=true
+x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/*BaseParser*.java linguist-generated=true
 x-pack/plugin/esql/src/main/generated/** linguist-generated=true
 x-pack/plugin/esql/src/main/generated-src/** linguist-generated=true
 

build-tools-internal/src/main/resources/checkstyle_suppressions.xml

@@ -12,7 +12,7 @@
   <suppress files="modules[/\\]lang-painless[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]painless[/\\]antlr[/\\]SuggestLexer\.java" checks="." />
   <suppress files="plugin[/\\]sql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]sql[/\\]parser[/\\]SqlBase(Base(Listener|Visitor)|Lexer|Listener|Parser|Visitor).java" checks="." />
   <suppress files="plugin[/\\]eql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]eql[/\\]parser[/\\]EqlBase(Base(Listener|Visitor)|Lexer|Listener|Parser|Visitor).java" checks="." />
-  <suppress files="plugin[/\\]esql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]esql[/\\]parser[/\\]EsqlBase(Parser|Lexer).*.java" checks="." />
+  <suppress files="plugin[/\\]esql[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]xpack[/\\]esql[/\\]parser[/\\](EsqlBase|PromqlBase)(Parser|Lexer).*.java" checks="." />
   <suppress files="x-pack[/\\]plugin[/\\]otel-data[/\\]build[/\\]generated[/\\]sources[/\\]" checks="." />
 
   <!-- JNA requires the no-argument constructor on JNAKernel32Library.SizeT to be public-->

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java

@@ -22,6 +22,7 @@
 
 import java.io.IOException;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.Collection;
 import java.util.Objects;
 
@@ -217,6 +218,10 @@ public static Literal timeDuration(Source source, Duration literal) {
         return new Literal(source, literal, DataType.TIME_DURATION);
     }
 
+    public static Literal dateTime(Source source, Instant literal) {
+        return new Literal(source, literal, DataType.DATETIME);
+    }
+
     public static Literal integer(Source source, Integer literal) {
         return new Literal(source, literal, INTEGER);
     }
@@ -229,6 +234,10 @@ public static Literal fromLong(Source source, Long literal) {
         return new Literal(source, literal, LONG);
     }
 
+    public static Expression fromBoolean(Source source, Boolean literal) {
+        return new Literal(source, literal, DataType.BOOLEAN);
+    }
+
     private static BytesRef longAsWKB(DataType dataType, long encoded) {
         return dataType == GEO_POINT ? GEO.longAsWkb(encoded) : CARTESIAN.longAsWkb(encoded);
     }

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Arithmetics.java

@@ -148,7 +148,7 @@ public static Number mod(Number l, Number r) {
         return l.intValue() % r.intValue();
     }
 
-    static Number negate(Number n) {
+    public static Number negate(Number n) {
         if (n == null) {
             return null;
         }

x-pack/plugin/esql/build.gradle

@@ -332,6 +332,8 @@ pluginManager.withPlugin('com.diffplug.spotless') {
       // for some reason "${outputPath}/EsqlBaseParser*.java" does not match the same files...
       targetExclude "src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer*.java",
         "src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser*.java",
+        "src/main/java/org/elasticsearch/xpack/esql/parser/PromqlBaseLexer*.java",
+        "src/main/java/org/elasticsearch/xpack/esql/parser/PromqlBaseParser*.java",
         "src/main/generated/**/*.java",
         "src/main/generated-src/generated/**/*.java"
       toggleOffOn('begin generated imports', 'end generated imports')
@@ -345,6 +347,7 @@ tasks.register("cleanGenerated", Delete) {
   }
   delete fileTree(outputPath) {
     include 'EsqlBase*.java'
+    include 'PromqlBase*.java'
   }
 }
 
@@ -384,52 +387,93 @@ tasks.register("regenParser", JavaExec) {
     "${file(grammarPath)}/EsqlBaseParser.g4"
 }
 
+tasks.register("regenPromqlLexer", JavaExec) {
+  dependsOn "cleanGenerated"
+  mainClass = 'org.antlr.v4.Tool'
+  classpath = configurations.regenerate
+  systemProperty 'file.encoding', 'UTF-8'
+  systemProperty 'user.language', 'en'
+  systemProperty 'user.country', 'US'
+  systemProperty 'user.variant', ''
+  args '-Werror',
+    '-package', 'org.elasticsearch.xpack.esql.parser',
+    '-listener',
+    '-visitor',
+    '-lib', "${file(grammarPath)}",
+    '-o', outputPath,
+    "${file(grammarPath)}/PromqlBaseLexer.g4"
+}
+
+tasks.register("regenPromqlParser", JavaExec) {
+  dependsOn "cleanGenerated"
+  dependsOn "regenPromqlLexer"
+  mainClass = 'org.antlr.v4.Tool'
+  classpath = configurations.regenerate
+  systemProperty 'file.encoding', 'UTF-8'
+  systemProperty 'user.language', 'en'
+  systemProperty 'user.country', 'US'
+  systemProperty 'user.variant', ''
+  args '-Werror',
+    '-package', 'org.elasticsearch.xpack.esql.parser',
+    '-listener',
+    '-visitor',
+    '-lib', outputPath,
+    '-lib', "${file(grammarPath)}",
+    '-o', outputPath,
+    "${file(grammarPath)}/PromqlBaseParser.g4"
+}
+
 tasks.register("regen") {
   dependsOn "regenParser"
+  dependsOn "regenPromqlParser"
   doLast {
     // moves token files to grammar directory for use with IDE's
     ant.move(file: "${outputPath}/EsqlBaseLexer.tokens", toDir: grammarPath)
     ant.move(file: "${outputPath}/EsqlBaseParser.tokens", toDir: grammarPath)
+    ant.move(file: "${outputPath}/PromqlBaseLexer.tokens", toDir: grammarPath)
+    ant.move(file: "${outputPath}/PromqlBaseParser.tokens", toDir: grammarPath)
+
     // make the generated classes package private
     ant.replaceregexp(
-      match: 'public ((interface|class) \\QEsqlBase(Parser|Lexer)\\E\\w+)',
+      match: 'public ((interface|class) \\Q(Es|Prom)qlBase(Parser|Lexer)\\E\\w+)',
       replace: '\\1',
       encoding: 'UTF-8'
     ) {
-      fileset(dir: outputPath, includes: 'EsqlBase*.java')
+      fileset(dir: outputPath, includes: '*qlBase*.java')
     }
     // nuke timestamps/filenames in generated files
     ant.replaceregexp(
       match: '\\Q// Generated from \\E.*',
       replace: '\\/\\/ ANTLR GENERATED CODE: DO NOT EDIT',
       encoding: 'UTF-8'
     ) {
-      fileset(dir: outputPath, includes: 'EsqlBase*.java')
+      fileset(dir: outputPath, includes: '*qlBase*.java')
     }
     // remove tabs in antlr generated files
     ant.replaceregexp(match: '\t', flags: 'g', replace: '  ', encoding: 'UTF-8') {
-      fileset(dir: outputPath, includes: 'EsqlBase*.java')
+      fileset(dir: outputPath, includes: '*qlBase*.java')
     }
     // suppress this-escape warnings on EsqlBaseLexer
     ant.replaceregexp(
-      match: 'public EsqlBaseLexer',
-      replace: '@SuppressWarnings("this-escape")${line.separator}  public EsqlBaseLexer',
+      match: 'public ((Es|Prom)qlBaseLexer)',
+      replace: '@SuppressWarnings("this-escape")${line.separator}  public \\1',
       encoding: 'UTF-8'
     ) {
-      fileset(dir: outputPath, includes: 'EsqlBaseLexer.java')
+      fileset(dir: outputPath, includes: '*qlBaseLexer.java')
     }
+
     // suppress this-escape warnings on all internal EsqlBaseParser class constructores
     ant.replaceregexp(
       match: '([ ]+)public ([A-Z][a-z]+[a-z,A-Z]+\\()',
       flags: 'g',
       replace: '\\1@SuppressWarnings("this-escape")${line.separator}\\1public \\2',
       encoding: 'UTF-8'
     ) {
-      fileset(dir: outputPath, includes: 'EsqlBaseParser.java')
+      fileset(dir: outputPath, includes: '*qlBaseParser.java')
     }
     // fix line endings
     ant.fixcrlf(srcdir: outputPath, eol: 'lf') {
-      patternset(includes: 'EsqlBase*.java')
+      patternset(includes: '*qlBase*.java')
     }
   }
 }

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java

@@ -60,6 +60,8 @@ protected void shouldSkipTest(String testName) throws IOException {
             testCase.requiredCapabilities.contains(SUBQUERY_IN_FROM_COMMAND.capabilityName())
         );
 
+        assumeFalse("Tests using PROMQL are not supported for now", testCase.requiredCapabilities.contains(PROMQL_V0.capabilityName()));
+
         assumeTrue("Cluster needs to support FORK", hasCapabilities(adminClient(), List.of(FORK_V9.capabilityName())));
     }
 }

x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries-avg-over-time.csv-spec

@@ -17,6 +17,25 @@ cost:double       | time_bucket:datetime
 36.375            | 2024-05-10T00:11:00.000Z
 ;
 
+avg_over_time_of_double_no_grouping_single_bucket
+required_capability: ts_command_v0
+required_capability: tbucket
+TS k8s
+| STATS cost=sum(avg_over_time(network.cost)) BY time_bucket = tbucket(1h);
+
+cost:double       | time_bucket:datetime
+56.32254035241995 | 2024-05-10T00:00:00.000Z
+;
+
+avg_over_time_of_double_no_grouping_single_bucket_promql
+required_capability: promql_v0
+TS k8s
+| PROMQL step 1h (sum(avg_over_time(network.cost[1h])));
+
+sum(avg_over_time(network.cost[1h])):double | TBUCKET:date
+56.32254035241995                           | 2024-05-10T00:00:00.000Z
+;
+
 avg_over_time_of_integer
 required_capability: ts_command_v0
 TS k8s | STATS clients = avg(avg_over_time(network.eth0.currently_connected_clients)) BY time_bucket = bucket(@timestamp,1minute) | SORT time_bucket | LIMIT 10;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec

@@ -132,6 +132,25 @@ max(rate(network.total_bytes_in)): double | time_bucket:date
 23.702205882352942                        | 2024-05-10T00:15:00.000Z
 ;
 
+oneRateWithSingleTBucket
+required_capability: ts_command_v0
+required_capability: tbucket
+TS k8s
+| STATS max(rate(network.total_bytes_in)) BY time_bucket = tbucket(1h);
+
+max(rate(network.total_bytes_in)): double | time_bucket:date
+4.379885057471264                         | 2024-05-10T00:00:00.000Z
+;
+
+oneRateWithSingleTBucketPromql
+required_capability: promql_v0
+TS k8s
+| PROMQL step 1h (max(rate(network.total_bytes_in[1h])));
+
+max(rate(network.total_bytes_in[1h])):double | TBUCKET:date
+4.379885057471264                            | 2024-05-10T00:00:00.000Z
+;
+
 twoRatesWithBucket
 required_capability: ts_command_v0
 TS k8s | STATS max(rate(network.total_bytes_in)), sum(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute) | SORT time_bucket DESC | LIMIT 3;