SQL: Add option to provide the delimiter for the CSV format (#59907) (#…

…60420)

* SQL: Add option to provide the delimiter for the CSV format (#59907)

* Add option to provide the delimiter to the CSV fmt

This adds the option to provide the desired character as the separator
for the CSV format (the default remains comma).
A set of characters are excluded though - like CR, LF, `"` - to avoid
slipping onto the CSV-dialects slope. The tab is also forbidden, the
user needs to choose the "tsv" format explicitely.

Update the doc to make it clear that the textual CSV, TSV and TXT
formats pass the cursor back to the user through the Cursor HTTP header.

(cherry picked from commit 3a8b00c)

* Java8 fixes

- replace Set#of();
- URLDecoder#decode() requires a string (vs a charset) as 2nd arg.

docs/reference/sql/endpoints/rest.asciidoc

@@ -15,7 +15,7 @@
 === Overview
 
 The SQL REST API accepts SQL in a JSON document, executes it,
-and returns the results. 
+and returns the results.
 For example:
 
 [source,console]
@@ -106,6 +106,10 @@ s|Description
 
 |===
 
+The `CSV` format accepts a formatting URL query attribute, `delimiter`, which indicates which character should be used to separate the CSV
+values. It defaults to comma (`,`) and cannot take any of the following values: double quote (`"`), carriage-return (`\r`) and new-line (`\n`).
+The tab (`\t`) can also not be used, the `tsv` format needs to be used instead.
+
 Here are some examples for the human readable formats:
 
 ==== CSV
@@ -120,7 +124,7 @@ POST /_sql?format=csv
 --------------------------------------------------
 // TEST[setup:library]
 
-Which returns:
+which returns:
 
 [source,text]
 --------------------------------------------------
@@ -133,6 +137,31 @@ James S.A. Corey,Leviathan Wakes,561,2011-06-02T00:00:00.000Z
 --------------------------------------------------
 // TESTRESPONSE[non_json]
 
+or:
+
+[source,console]
+--------------------------------------------------
+POST /_sql?format=csv&delimiter=%3b
+{
+    "query": "SELECT * FROM library ORDER BY page_count DESC",
+    "fetch_size": 5
+}
+--------------------------------------------------
+// TEST[setup:library]
+
+which returns:
+
+[source,text]
+--------------------------------------------------
+author;name;page_count;release_date
+Peter F. Hamilton;Pandora's Star;768;2004-03-02T00:00:00.000Z
+Vernor Vinge;A Fire Upon the Deep;613;1992-06-01T00:00:00.000Z
+Frank Herbert;Dune;604;1965-06-01T00:00:00.000Z
+Alastair Reynolds;Revelation Space;585;2000-03-15T00:00:00.000Z
+James S.A. Corey;Leviathan Wakes;561;2011-06-02T00:00:00.000Z
+--------------------------------------------------
+// TESTRESPONSE[non_json]
+
 ==== JSON
 
 [source,console]
@@ -210,7 +239,7 @@ Which returns:
 
 [source,text]
 --------------------------------------------------
-     author      |        name        |  page_count   |      release_date      
+     author      |        name        |  page_count   |      release_date
 -----------------+--------------------+---------------+------------------------
 Peter F. Hamilton|Pandora's Star      |768            |2004-03-02T00:00:00.000Z
 Vernor Vinge     |A Fire Upon the Deep|613            |1992-06-01T00:00:00.000Z
@@ -275,8 +304,8 @@ cursor: "sDXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAAEWWWdrRlVfSS1TbDYtcW9lc1FJNmlYdw==:BAFmB
 === Paginating through a large response
 
 Using the example from the <<sql-rest-format,previous section>>, one can
-continue to the next page by sending back the cursor field. In case of text
-format, the cursor is returned as `Cursor` http header.
+continue to the next page by sending back the cursor field. In the case of CSV, TSV and TXT
+formats, the cursor is returned in the `Cursor` HTTP header.
 
 [source,console]
 --------------------------------------------------

x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/Protocol.java

@@ -50,15 +50,21 @@ public final class Protocol {
     public static final TimeValue PAGE_TIMEOUT = TimeValue.timeValueSeconds(45);
     public static final boolean FIELD_MULTI_VALUE_LENIENCY = false;
     public static final boolean INDEX_INCLUDE_FROZEN = false;
-    
+
     /*
-     * Using the Boolean object here so that SqlTranslateRequest to set this to null (since it doesn't need a "columnar" or 
+     * Using the Boolean object here so that SqlTranslateRequest to set this to null (since it doesn't need a "columnar" or
      * binary parameter).
      * See {@code SqlTranslateRequest.toXContent}
      */
     public static final Boolean COLUMNAR = Boolean.FALSE;
     public static final Boolean BINARY_COMMUNICATION = null;
 
+    /*
+     * URL parameters
+     */
+    public static final String URL_PARAM_FORMAT = "format";
+    public static final String URL_PARAM_DELIMITER = "delimiter";
+
     /**
      * SQL-related endpoints
      */

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/RestSqlQueryAction.java

@@ -24,16 +24,22 @@
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 import static java.util.Arrays.asList;
 import static java.util.Collections.emptyList;
 import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.xpack.sql.proto.Protocol.URL_PARAM_DELIMITER;
+import static org.elasticsearch.xpack.sql.proto.Protocol.URL_PARAM_FORMAT;
 
 public class RestSqlQueryAction extends BaseRestHandler {
 
+    TextFormat textFormat;
+
     @Override
     public List<Route> routes() {
         return emptyList();
@@ -77,7 +83,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
             // enforce CBOR response for drivers and CLI (unless instructed differently through the config param)
             accept = XContentType.CBOR.name();
         } else {
-            accept = request.param("format");
+            accept = request.param(URL_PARAM_FORMAT);
         }
         if (accept == null) {
             accept = request.header("Accept");
@@ -99,7 +105,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
          * which we turn into a 400 error.
          */
         XContentType xContentType = accept == null ? XContentType.JSON : XContentType.fromMediaTypeOrFormat(accept);
-        TextFormat textFormat = xContentType == null ? TextFormat.fromMediaTypeOrFormat(accept) : null;
+        textFormat = xContentType == null ? TextFormat.fromMediaTypeOrFormat(accept) : null;
 
         if (xContentType == null && sqlRequest.columnar()) {
             throw new IllegalArgumentException("Invalid use of [columnar] argument: cannot be used in combination with "
@@ -136,6 +142,11 @@ public RestResponse buildResponse(SqlQueryResponse response) throws Exception {
         });
     }
 
+    @Override
+    protected Set<String> responseParams() {
+        return textFormat == TextFormat.CSV ? Collections.singleton(URL_PARAM_DELIMITER) : Collections.emptySet();
+    }
+
     @Override
     public String getName() {
         return "sql_query";