Add misc improvements to the framework (#760) (#761)

Adds a number of improvements and bugfixes to the JSON and utility classes:

JSON framework:

* A new `BufferingJsonpMapper` allows the creation of `JsonGenerator` that store JSON events in a buffer that can be replayed. This is useful to efficiently create synthetic JSON documents.
* An additional `JsonpMapper.deserialize` method variant accepts the current JSON event. Fixes #741
* The Jackson implementation of `JsonpMapper` now enables the `ACCEPT_SINGLE_VALUE_AS_ARRAY` deserialization feature. This allows single values in a JSON stream to be considered as a single-value collection.

API & transport framework:

* `ElasticsearchException` now holds the low level http response, so that the application can inspect the details of the error.
* Endpoints now have a `call` methods, to make calling endpoints programmatically easier. This is for advanced use, as an application will normally use the `ElasticsearchClient` that hides endpoint objects.
* A `BinaryDataResponse` can now easily be created from a byte array.

Test framework:

* The `ElasticsearchTestServer` now tries to contact an Elasticsearch server running on `http://elastic:changeme@localhost:9200` before spawning a container.
* A `MockHttpClient` has been added that allows writing integration-like tests without requiring a running server. This is an alternative to using `com.sun.net.httpserver.HttpServer` when we want to test http response decoding but don't need to test the network layer.

Co-authored-by: Sylvain Wallez <sylvain@elastic.co>

java-client-serverless/src/main/java/co/elastic/clients/elasticsearch/_types/ElasticsearchException.java

@@ -19,6 +19,10 @@
 
 package co.elastic.clients.elasticsearch._types;
 
+import co.elastic.clients.transport.http.TransportHttpClient;
+
+import javax.annotation.Nullable;
+
 /**
  * Exception thrown by API client methods when Elasticsearch could not accept or
  * process a request.
@@ -31,11 +35,18 @@ public class ElasticsearchException extends RuntimeException {
 
 	private final ErrorResponse response;
 	private final String endpointId;
+	@Nullable
+	private final TransportHttpClient.Response httpResponse;
 
-	public ElasticsearchException(String endpointId, ErrorResponse response) {
+	public ElasticsearchException(String endpointId, ErrorResponse response, @Nullable TransportHttpClient.Response httpResponse) {
 		super("[" + endpointId + "] failed: [" + response.error().type() + "] " + response.error().reason());
 		this.response = response;
 		this.endpointId = endpointId;
+		this.httpResponse = httpResponse;
+	}
+
+	public ElasticsearchException(String endpointId, ErrorResponse response) {
+		this(endpointId, response, null);
 	}
 
 	/**
@@ -66,4 +77,12 @@ public ErrorCause error() {
 	public int status() {
 		return this.response.status();
 	}
+
+	/**
+	 * The underlying http response, if available.
+	 */
+	@Nullable
+	public TransportHttpClient.Response httpResponse() {
+		return this.httpResponse;
+	}
 }

java-client/src/main/java/co/elastic/clients/elasticsearch/_types/ElasticsearchException.java

@@ -19,6 +19,10 @@
 
 package co.elastic.clients.elasticsearch._types;
 
+import co.elastic.clients.transport.http.TransportHttpClient;
+
+import javax.annotation.Nullable;
+
 /**
  * Exception thrown by API client methods when Elasticsearch could not accept or
  * process a request.
@@ -31,11 +35,18 @@ public class ElasticsearchException extends RuntimeException {
 
 	private final ErrorResponse response;
 	private final String endpointId;
+	@Nullable
+	private final TransportHttpClient.Response httpResponse;
 
-	public ElasticsearchException(String endpointId, ErrorResponse response) {
+	public ElasticsearchException(String endpointId, ErrorResponse response, @Nullable TransportHttpClient.Response httpResponse) {
 		super("[" + endpointId + "] failed: [" + response.error().type() + "] " + response.error().reason());
 		this.response = response;
 		this.endpointId = endpointId;
+		this.httpResponse = httpResponse;
+	}
+
+	public ElasticsearchException(String endpointId, ErrorResponse response) {
+		this(endpointId, response, null);
 	}
 
 	/**
@@ -66,4 +77,12 @@ public ErrorCause error() {
 	public int status() {
 		return this.response.status();
 	}
+
+	/**
+	 * The underlying http response, if available.
+	 */
+	@Nullable
+	public TransportHttpClient.Response httpResponse() {
+		return this.httpResponse;
+	}
 }

java-client/src/main/java/co/elastic/clients/elasticsearch/_types/FieldValue.java

@@ -33,6 +33,7 @@
 import co.elastic.clients.util.ObjectBuilderBase;
 import co.elastic.clients.util.TaggedUnion;
 import co.elastic.clients.util.TaggedUnionUtils;
+import jakarta.json.Json;
 import jakarta.json.stream.JsonGenerator;
 import jakarta.json.stream.JsonParser;
 
@@ -69,6 +70,10 @@ public static FieldValue of(JsonData value) {
 		return new FieldValue(Kind.Any, value);
 	}
 
+	public static FieldValue of(Object value) {
+		return of(JsonData.of(value));
+	}
+
 	public static final FieldValue NULL = new FieldValue(Kind.Null, null);
 	public static final FieldValue TRUE = new FieldValue(Kind.Boolean, Boolean.TRUE);
 	public static final FieldValue FALSE = new FieldValue(Kind.Boolean, Boolean.FALSE);

java-client/src/main/java/co/elastic/clients/json/BufferingJsonGenerator.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package co.elastic.clients.json;
+
+import jakarta.json.stream.JsonGenerator;
+import jakarta.json.stream.JsonParser;
+
+public interface BufferingJsonGenerator extends JsonGenerator {
+
+    /**
+     * Close this generator and return the buffered content.
+     */
+    JsonData getJsonData();
+
+    /**
+     * Close this generator and return the buffered content as a parser.
+     */
+    JsonParser getParser();
+
+    void copyValue(JsonParser parser);
+}

java-client/src/main/java/co/elastic/clients/json/BufferingJsonpMapper.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package co.elastic.clients.json;
+
+/**
+ * A Jsonp mapper that has the additional capability of being able to buffer events.
+ */
+public interface BufferingJsonpMapper extends JsonpMapper {
+
+    BufferingJsonGenerator createBufferingGenerator();
+}

java-client/src/main/java/co/elastic/clients/json/DelegatingJsonpMapper.java

@@ -44,6 +44,11 @@ public <T> T deserialize(JsonParser parser, Type type) {
         return mapper.deserialize(parser, type);
     }
 
+    @Override
+    public <T> T deserialize(JsonParser parser, Type type, JsonParser.Event event) {
+        return mapper.deserialize(parser, type, event);
+    }
+
     @Override
     public <T> void serialize(T value, JsonGenerator generator) {
         mapper.serialize(value, generator);

java-client/src/main/java/co/elastic/clients/json/JsonpDeserializer.java

@@ -110,7 +110,7 @@ public T deserialize(JsonParser parser, JsonpMapper mapper) {
 
             @Override
             public T deserialize(JsonParser parser, JsonpMapper mapper, JsonParser.Event event) {
-                throw new UnsupportedOperationException();
+                return mapper.deserialize(parser, type, event);
             }
         };
     }

java-client/src/main/java/co/elastic/clients/json/JsonpMapper.java

@@ -54,6 +54,18 @@ default <T> T deserialize(JsonParser parser, Class<T> clazz) {
      */
     <T> T deserialize(JsonParser parser, Type type);
 
+    /**
+     * Deserialize an object, given its class and the current event the parser is at.
+     */
+    default <T> T deserialize(JsonParser parser, Class<T> clazz, JsonParser.Event event) {
+        return deserialize(parser, (Type)clazz, event);
+    }
+
+    /**
+     * Deserialize an object, given its type and the current event the parser is at.
+     */
+    <T> T deserialize(JsonParser parser, Type type, JsonParser.Event event);
+
     /**
      * Serialize an object.
      */

java-client/src/main/java/co/elastic/clients/json/JsonpMapperBase.java

@@ -61,23 +61,44 @@ protected JsonpMapperBase addAttribute(String name, Object value) {
         return this;
     }
 
+    //----- Deserialization
+
     @Override
     public <T> T deserialize(JsonParser parser, Type type) {
+        @SuppressWarnings("unchecked")
+        T result = (T)getDeserializer(type).deserialize(parser, this);
+        return result;
+    }
+
+    @Override
+    public <T> T deserialize(JsonParser parser, Type type, JsonParser.Event event) {
+        @SuppressWarnings("unchecked")
+        T result = (T)getDeserializer(type).deserialize(parser, this, event);
+        return result;
+    }
+
+    private <T> JsonpDeserializer<T> getDeserializer(Type type) {
         JsonpDeserializer<T> deserializer = findDeserializer(type);
         if (deserializer != null) {
-            return deserializer.deserialize(parser, this);
+            return deserializer;
         }
 
         @SuppressWarnings("unchecked")
-        T result = (T)getDefaultDeserializer(type).deserialize(parser, this);
+        JsonpDeserializer<T> result = getDefaultDeserializer(type);
         return result;
     }
 
+    /**
+     * Find a built-in deserializer for a given class, if any.
+     */
     @Nullable
     public static <T> JsonpDeserializer<T> findDeserializer(Class<T> clazz) {
         return findDeserializer((Type)clazz);
     }
 
+    /**
+     * Find a built-in deserializer for a given type, if any.
+     */
     @Nullable
     @SuppressWarnings("unchecked")
     public static <T> JsonpDeserializer<T> findDeserializer(Type type) {
@@ -101,6 +122,8 @@ public static <T> JsonpDeserializer<T> findDeserializer(Type type) {
         return null;
     }
 
+    //----- Serialization
+
     @Nullable
     @SuppressWarnings("unchecked")
     public static <T> JsonpSerializer<T> findSerializer(T value) {

java-client/src/main/java/co/elastic/clients/json/JsonpUtils.java

@@ -412,10 +412,10 @@ public void close() {
         return dest;
     }
 
-    public static String toJsonString(JsonpSerializable value, JsonpMapper mapper) {
+    public static String toJsonString(Object value, JsonpMapper mapper) {
         StringWriter writer = new StringWriter();
         JsonGenerator generator = mapper.jsonProvider().createGenerator(writer);
-        value.serialize(generator, mapper);
+        mapper.serialize(value, generator);
         generator.close();
         return writer.toString();
     }

java-client/src/main/java/co/elastic/clients/json/jackson/JacksonJsonpGenerator.java

@@ -19,12 +19,16 @@
 
 package co.elastic.clients.json.jackson;
 
+import co.elastic.clients.json.BufferingJsonGenerator;
+import co.elastic.clients.json.JsonData;
 import com.fasterxml.jackson.core.JsonStreamContext;
+import com.fasterxml.jackson.databind.util.TokenBuffer;
 import jakarta.json.JsonNumber;
 import jakarta.json.JsonString;
 import jakarta.json.JsonValue;
 import jakarta.json.stream.JsonGenerationException;
 import jakarta.json.stream.JsonGenerator;
+import jakarta.json.stream.JsonParser;
 
 import java.io.IOException;
 import java.math.BigDecimal;
@@ -42,6 +46,43 @@ public JacksonJsonpGenerator(com.fasterxml.jackson.core.JsonGenerator generator)
         this.generator = generator;
     }
 
+    public static class Buffering extends JacksonJsonpGenerator implements BufferingJsonGenerator {
+
+        private final JacksonJsonpMapper mapper;
+
+        public Buffering(JacksonJsonpMapper mapper) {
+            super(new TokenBuffer(mapper.objectMapper(), false));
+            this.mapper = mapper;
+        }
+
+        @Override
+        public JsonData getJsonData() {
+            this.close();
+            return new JacksonJsonBuffer((TokenBuffer)jacksonGenerator(), mapper);
+        }
+
+        @Override
+        public JsonParser getParser() {
+            this.close();
+            TokenBuffer tokenBuffer = (TokenBuffer) jacksonGenerator();
+            return new JacksonJsonpParser(tokenBuffer.asParser(), mapper);
+        }
+
+        @Override
+        public void copyValue(JsonParser parser) {
+            if (!(parser instanceof JacksonJsonpGenerator)) {
+                throw new IllegalArgumentException("Can only be used with a JacksonJsonpGenerator");
+            }
+
+            com.fasterxml.jackson.core.JsonParser jkParser = ((JacksonJsonpParser) parser).jacksonParser();
+            try {
+                jacksonGenerator().copyCurrentStructure(jkParser);
+            } catch (IOException e) {
+                throw JacksonUtils.convertException(e);
+            }
+        }
+    }
+
     /**
      * Returns the underlying Jackson generator.
      */