ESQL: Grammar - FROM METADATA no longer require []

docs/changelog/105221.yaml

@@ -0,0 +1,14 @@
+pr: 105221
+summary: "ESQL: Grammar - FROM METADATA no longer requires []"
+area: ES|QL
+type: breaking
+issues: []
+breaking:
+  title: "ESQL: Grammar - FROM METADATA no longer requires []"
+  area: REST API
+  details: "Remove [ ] for METADATA option inside FROM command statements"
+  impact: "Previously to return metadata fields, one had to use square brackets:\
+    \ (eg. 'FROM index [METADATA _index]').\
+    \ This is no longer needed: the [ ] are dropped and do not have to be specified,\
+    \ thus simplifying the command above to:'FROM index METADATA _index'."
+  notable: false

docs/reference/esql/metadata-fields.asciidoc

@@ -5,10 +5,10 @@
 <titleabbrev>Metadata fields</titleabbrev>
 ++++
 
-{esql} can access <<mapping-fields, metadata fields>>. The following are currently
-supported:
+{esql} can access <<mapping-fields, metadata fields>>. The currently
+supported ones are:
 
-  * <<mapping-index-field,`_index`>>: the index the document belongs to.
+  * <<mapping-index-field,`_index`>>: the index to which the document belongs.
   The field is of the type <<keyword, keyword>>.
 
   * <<mapping-id-field,`_id`>>: the source document's ID. The field is of the
@@ -17,18 +17,12 @@ supported:
   * `_version`: the source document's version. The field is of the type
   <<number,long>>.
 
-   * <<mapping-source-field,`_source`>>: the original JSON document body 
-  that was passed at index time (or a reconstructed version if 
-  <<synthetic-source, synthetic `_source_`>> is enabled). The field is 
-  loaded as a special `_source` type. This field is not supported by 
-  functions.
-
-To enable access to these fields, the <<esql-from,`FROM`>> source command 
-requires a dedicated directive:
+To enable the access to these fields, the <<esql-from,`FROM`>> source command needs
+to be provided with a dedicated directive:
 
 [source,esql]
 ----
-FROM index [METADATA _index, _id, _source]
+FROM index METADATA _index, _id
 ----
 
 Metadata fields are only available if the source of the data is an index.

docs/reference/esql/source-commands/from.asciidoc

@@ -66,9 +66,9 @@ or aliases:
 FROM employees-00001,other-employees-*
 ----
 
-Use the `METADATA` directive to enable <<esql-metadata-fields,metadata fields>>:
+Use the optional `METADATA` directive to enable <<esql-metadata-fields,metadata fields>>:
 
 [source,esql]
 ----
-FROM employees [METADATA _id]
+FROM employees METADATA _id
 ----

x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java

@@ -161,16 +161,18 @@ static CsvSpecReader.CsvTestCase convertToRemoteIndices(CsvSpecReader.CsvTestCas
         String query = testCase.query;
         String[] commands = query.split("\\|");
         String first = commands[0].trim();
+
         if (commands[0].toLowerCase(Locale.ROOT).startsWith("from")) {
-            String[] parts = commands[0].split("\\[");
+            String[] parts = commands[0].split("(?i)metadata");
             assert parts.length >= 1 : parts;
             String fromStatement = parts[0];
+
             String[] localIndices = fromStatement.substring("FROM ".length()).split(",");
             String remoteIndices = Arrays.stream(localIndices)
                 .map(index -> "*:" + index.trim() + "," + index.trim())
                 .collect(Collectors.joining(","));
-            var newFrom = "FROM " + remoteIndices + commands[0].substring(fromStatement.length());
-            testCase.query = newFrom + " " + query.substring(first.length());
+            var newFrom = "FROM " + remoteIndices + " " + commands[0].substring(fromStatement.length());
+            testCase.query = newFrom + query.substring(first.length());
         }
         int offset = testCase.query.length() - query.length();
         if (offset != 0) {
@@ -195,7 +197,7 @@ static CsvSpecReader.CsvTestCase convertToRemoteIndices(CsvSpecReader.CsvTestCas
     static boolean hasIndexMetadata(String query) {
         String[] commands = query.split("\\|");
         if (commands[0].trim().toLowerCase(Locale.ROOT).startsWith("from")) {
-            String[] parts = commands[0].split("\\[");
+            String[] parts = commands[0].split("(?i)metadata");
             return parts.length > 1 && parts[1].contains("_index");
         }
         return false;

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java

@@ -441,7 +441,7 @@ public void testMetadataFieldsOnMultipleIndices() throws IOException {
         request.setJsonEntity("{\"a\": 3}");
         assertEquals(201, client().performRequest(request).getStatusLine().getStatusCode());
 
-        var query = fromIndex() + "* [metadata _index, _version, _id] | sort _version";
+        var query = fromIndex() + "* metadata _index, _version, _id | sort _version";
         Map<String, Object> result = runEsql(new RequestObjectBuilder().query(query));
         var columns = List.of(
             Map.of("name", "a", "type", "long"),

x-pack/plugin/esql/qa/testFixtures/src/main/resources/id.csv-spec

@@ -1,9 +1,8 @@
 //
-// Tests for _id fields
 //
 
-selectAll
-FROM apps [metadata _id];
+selectAll#[skip:-8.12.99]
+FROM apps metadata _id;
 ignoreOrder:true
 
 id:integer |name:keyword   |version:version     | _id:keyword
@@ -23,22 +22,23 @@ id:integer |name:keyword   |version:version     | _id:keyword
 14         |mmmmm          |5.2.9               | 14
 ;
 
-filterById
-FROM apps [metadata _id]| WHERE _id == "4";
+filterById#[skip:-8.12.99]
+FROM apps metadata _id | WHERE _id == "4";
 
 id:i |name:k |version:v | _id:k
 4    |ddddd  |2.12.0    | 4
 ;
 
-keepId
-FROM apps [metadata _id] | WHERE id == 3 | KEEP _id;
+keepId#[skip:-8.12.99]
+FROM apps metadata _id | WHERE id == 3 | KEEP _id;
+
 
 _id:k
 3
 ;
 
-idRangeAndSort
-FROM apps [metadata _id] | WHERE _id >= "2" AND _id <= "7" | SORT _id | keep id, name, _id;
+idRangeAndSort#[skip:-8.12.99]
+FROM apps metadata _id | WHERE _id >= "2" AND _id <= "7" | SORT _id | keep id, name, _id;
 
 id:i    |name:k       | _id:k
 2       |bbbbb        | 2
@@ -49,8 +49,8 @@ id:i    |name:k       | _id:k
 7       |ggggg        | 7
 ;
 
-orderById
-FROM apps [metadata _id] | KEEP _id, name | SORT _id;
+orderById#[skip:-8.12.99]
+FROM apps metadata _id | KEEP _id, name | SORT _id;
 
 _id:k  | name:s
 1      | aaaaa
@@ -69,8 +69,8 @@ _id:k  | name:s
 9      | iiiii
 ;
 
-orderByIdDesc
-FROM apps [metadata _id] | KEEP _id, name | SORT _id DESC;
+orderByIdDesc#[skip:-8.12.99]
+FROM apps metadata _id | KEEP _id, name | SORT _id DESC;
 
 _id:k  | name:s
 
@@ -90,8 +90,8 @@ _id:k  | name:s
 1      | aaaaa
 ;
 
-concatId
-FROM apps [metadata _id] | eval c = concat(_id, name) | SORT _id | KEEP c;
+concatId#[skip:-8.12.99]
+FROM apps metadata _id | eval c = concat(_id, name) | SORT _id | KEEP c;
 
 c:k
 1aaaaa
@@ -110,16 +110,16 @@ c:k
 9iiiii
 ;
 
-statsOnId
-FROM apps [metadata _id] | stats c = count(_id), d = count_distinct(_id);
+statsOnId#[skip:-8.12.99]
+FROM apps metadata _id | stats c = count(_id), d = count_distinct(_id);
 
 c:l | d:l
 14  | 14
 ;
 
 
-statsOnIdByGroup
-FROM apps [metadata _id] | stats c = count(_id) by name | sort c desc, name | limit 5;
+statsOnIdByGroup#[skip:-8.12.99]
+FROM apps metadata _id | stats c = count(_id) by name | sort c desc, name | limit 5;
 
 c:l | name:k
 2   | aaaaa

x-pack/plugin/esql/qa/testFixtures/src/main/resources/metadata-IT_tests_only.csv-spec

@@ -1,31 +1,32 @@
 
-simpleKeep
-from employees [metadata _index, _version] | sort emp_no | limit 2 | keep emp_no, _index, _version;
+simpleKeep#[skip:-8.12.99]
+from employees metadata _index, _version | sort emp_no | limit 2 | keep emp_no, _index, _version;
 
 emp_no:integer |_index:keyword |_version:long
 10001          |employees      |1
 10002          |employees      |1
 ;
 
-aliasWithSameName
-from employees [metadata _index, _version] | sort emp_no | limit 2 | eval _index = _index, _version = _version | keep emp_no, _index, _version;
+aliasWithSameName#[skip:-8.12.99]
+from employees metadata _index, _version | sort emp_no | limit 2 | eval _index = _index, _version = _version | keep emp_no, _index, _version;
 
 emp_no:integer |_index:keyword |_version:long
 10001          |employees      |1
 10002          |employees      |1
 ;
 
-inComparison
-from employees [metadata _index, _version] | sort emp_no | where _index == "employees" | where _version == 1 | keep emp_no | limit 2;
+inComparison#[skip:-8.12.99]
+from employees metadata _index, _version | sort emp_no | where _index == "employees" | where _version == 1 | keep emp_no | limit 2;
+
 
 emp_no:integer
 10001
 10002
 ;
 
-metaIndexInAggs
+metaIndexInAggs#[skip:-8.12.99]
 // tag::metaIndexInAggs[]
-FROM employees [METADATA _index, _id]
+FROM employees METADATA _index, _id
 | STATS max = MAX(emp_no) BY _index
 // end::metaIndexInAggs[]
 ;
@@ -36,83 +37,84 @@ max:integer |_index:keyword
 // end::metaIndexInAggs-result[]
 ;
 
-metaIndexAliasedInAggs
-from employees [metadata _index] | eval _i = _index | stats max = max(emp_no) by _i;
+metaIndexAliasedInAggs#[skip:-8.12.99]
+from employees metadata _index | eval _i = _index | stats max = max(emp_no) by _i;
+
 
 max:integer |_i:keyword
 10100       |employees
 ;
 
-metaVersionInAggs
-from employees [metadata _version] | stats min = min(emp_no) by _version;
+metaVersionInAggs#[skip:-8.12.99]
+from employees metadata _version | stats min = min(emp_no) by _version;
 
 min:integer |_version:long
 10001       |1
 ;
 
-metaVersionAliasedInAggs
-from employees [metadata _version] | eval _v = _version | stats min = min(emp_no) by _v;
+metaVersionAliasedInAggs#[skip:-8.12.99]
+from employees metadata _version | eval _v = _version | stats min = min(emp_no) by _v;
 
 min:integer |_v:long
 10001       |1
 ;
 
-inAggsAndAsGroups
-from employees [metadata _index, _version] | stats max = max(_version) by _index;
+inAggsAndAsGroups#[skip:-8.12.99]
+from employees metadata _index, _version | stats max = max(_version) by _index;
 
 max:long |_index:keyword
 1        |employees
 ;
 
-inAggsAndAsGroupsAliased
-from employees [metadata _index, _version] | eval _i = _index, _v = _version | stats max = max(_v) by _i;
+inAggsAndAsGroupsAliased#[skip:-8.12.99]
+from employees metadata _index, _version | eval _i = _index, _v = _version | stats max = max(_v) by _i;
 
 max:long |_i:keyword
 1        |employees
 ;
 
-inFunction
-from employees [metadata _index, _version] | sort emp_no | where length(_index) == length("employees") | where abs(_version) == 1 | keep emp_no | limit 2;
+inFunction#[skip:-8.12.99]
+from employees metadata _index, _version | sort emp_no | where length(_index) == length("employees") | where abs(_version) == 1 | keep emp_no | limit 2;
 
 emp_no:integer
 10001
 10002
 ;
 
-inArithmetics
-from employees [metadata _index, _version] | eval i = _version + 2 | stats min = min(emp_no) by i;
+inArithmetics#[skip:-8.12.99]
+from employees metadata _index, _version | eval i = _version + 2 | stats min = min(emp_no) by i;
 
 min:integer |i:long
 10001       |3
 ;
 
-inSort
-from employees [metadata _index, _version] | sort _version, _index, emp_no | keep emp_no, _version, _index | limit 2;
+inSort#[skip:-8.12.99]
+from employees metadata _index, _version | sort _version, _index, emp_no | keep emp_no, _version, _index | limit 2;
 
 emp_no:integer |_version:long |_index:keyword
 10001          |1             |employees
 10002          |1             |employees
 ;
 
-withMvFunction
-from employees [metadata _version] | eval i = mv_avg(_version) + 2 | stats min = min(emp_no) by i;
+withMvFunction#[skip:-8.12.99]
+from employees metadata _version | eval i = mv_avg(_version) + 2 | stats min = min(emp_no) by i;
 
 min:integer |i:double
 10001       |3.0
 ;
 
-overwritten
-from employees [metadata _index, _version] | sort emp_no | eval _index = 3, _version = "version" | keep emp_no, _index, _version | limit 3;
+overwritten#[skip:-8.12.99]
+from employees metadata _index, _version | sort emp_no | eval _index = 3, _version = "version" | keep emp_no, _index, _version | limit 3;
 
 emp_no:integer |_index:integer |_version:keyword
 10001          |3              |version
 10002          |3              |version
 10003          |3              |version
 ;
 
-multipleIndices
+multipleIndices#[skip:-8.12.99]
 // tag::multipleIndices[]
-FROM ul_logs, apps [METADATA _index, _version]
+FROM ul_logs, apps METADATA _index, _version
 | WHERE id IN (13, 14) AND _version == 1
 | EVAL key = CONCAT(_index, "_", TO_STR(id))
 | SORT id, _index

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java

@@ -114,7 +114,7 @@ public void testSimple() {
     }
 
     public void testMetadataIndex() {
-        try (EsqlQueryResponse resp = runQuery("FROM logs*,*:logs* [METADATA _index] | stats sum(v) by _index | sort _index")) {
+        try (EsqlQueryResponse resp = runQuery("FROM logs*,*:logs* METADATA _index | stats sum(v) by _index | sort _index")) {
             List<List<Object>> values = getValuesList(resp);
             assertThat(values.get(0), equalTo(List.of(285L, "cluster-a:logs-2")));
             assertThat(values.get(1), equalTo(List.of(45L, "logs-1")));

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java

@@ -1222,7 +1222,7 @@ public void testGroupingMultiValueByOrdinals() {
     }
 
     public void testLoadId() {
-        try (EsqlQueryResponse results = run("from test [metadata _id] | keep _id | sort _id ")) {
+        try (EsqlQueryResponse results = run("from test metadata _id | keep _id | sort _id ")) {
             assertThat(results.columns(), equalTo(List.of(new ColumnInfo("_id", "keyword"))));
             ListMatcher values = matchesList();
             for (int i = 10; i < 50; i++) {
@@ -1427,7 +1427,7 @@ public void testQueryOnEmptyMappingIndex() {
 
         assertEmptyIndexQueries(from);
 
-        try (EsqlQueryResponse resp = run(from + "[METADATA _source] | EVAL x = 123")) {
+        try (EsqlQueryResponse resp = run(from + "METADATA _source | EVAL x = 123")) {
             assertFalse(resp.values().hasNext());
             assertThat(resp.columns(), equalTo(List.of(new ColumnInfo("_source", "_source"), new ColumnInfo("x", "integer"))));
         }
@@ -1455,7 +1455,7 @@ public void testQueryOnEmptyDataIndex() {
 
         assertEmptyIndexQueries(from);
 
-        try (EsqlQueryResponse resp = run(from + "[METADATA _source] | EVAL x = 123")) {
+        try (EsqlQueryResponse resp = run(from + "METADATA _source | EVAL x = 123")) {
             assertFalse(resp.values().hasNext());
             assertThat(
                 resp.columns(),
@@ -1470,7 +1470,7 @@ public void testQueryOnEmptyDataIndex() {
     }
 
     private void assertEmptyIndexQueries(String from) {
-        try (EsqlQueryResponse resp = run(from + "[METADATA _source] | KEEP _source | LIMIT 1")) {
+        try (EsqlQueryResponse resp = run(from + "METADATA _source | KEEP _source | LIMIT 1")) {
             assertFalse(resp.values().hasNext());
             assertThat(resp.columns(), equalTo(List.of(new ColumnInfo("_source", "_source"))));
         }