Merge remote-tracking branch 'remotes/armbru/tags/pull-qobject-2018-0…

…8-24' into staging

QObject patches for 2018-08-24

# gpg: Signature made Fri 24 Aug 2018 20:28:53 BST
# gpg:                using RSA key 3870B400EB918653
# gpg: Good signature from "Markus Armbruster <armbru@redhat.com>"
# gpg:                 aka "Markus Armbruster <armbru@pond.sub.org>"
# Primary key fingerprint: 354B C8B3 D7EB 2A6B 6867  4E5F 3870 B400 EB91 8653

* remotes/armbru/tags/pull-qobject-2018-08-24: (58 commits)
  json: Update references to RFC 7159 to RFC 8259
  json: Support %% in JSON strings when interpolating
  json: Improve safety of qobject_from_jsonf_nofail() & friends
  json: Keep interpolation state in JSONParserContext
  tests/drive_del-test: Fix harmless JSON interpolation bug
  json: Clean up headers
  qobject: Drop superfluous includes of qemu-common.h
  json: Make JSONToken opaque outside json-parser.c
  json: Unbox tokens queue in JSONMessageParser
  json: Streamline json_message_process_token()
  json: Enforce token count and size limits more tightly
  qjson: Have qobject_from_json() & friends reject empty and blank
  json: Assert json_parser_parse() consumes all tokens on success
  json: Fix streamer not to ignore trailing unterminated structures
  json: Fix latent parser aborts at end of input
  qjson: Fix qobject_from_json() & friends for multiple values
  json: Improve names of lexer states related to numbers
  json: Replace %I64d, %I64u by %PRId64, %PRIu64
  json: Leave rejecting invalid interpolation to parser
  json: Pass lexical errors and limit violations to callback
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

MAINTAINERS

@@ -1715,6 +1715,7 @@ F: monitor.c
 F: docs/devel/*qmp-*
 F: scripts/qmp/
 F: tests/qmp-test.c
+F: tests/qmp-cmd-test.c
 T: git git://repo.or.cz/qemu/armbru.git qapi-next
 
 qtest

block.c

@@ -1478,11 +1478,6 @@ static QDict *parse_json_filename(const char *filename, Error **errp)
 
     options_obj = qobject_from_json(filename, errp);
     if (!options_obj) {
-        /* Work around qobject_from_json() lossage TODO fix that */
-        if (errp && !*errp) {
-            error_setg(errp, "Could not parse the JSON options");
-            return NULL;
-        }
         error_prepend(errp, "Could not parse the JSON options: ");
         return NULL;
     }

docs/interop/qmp-spec.txt

@@ -20,9 +20,9 @@ operating system.
 2. Protocol Specification
 =========================
 
-This section details the protocol format. For the purpose of this document
-"Client" is any application which is using QMP to communicate with QEMU and
-"Server" is QEMU itself.
+This section details the protocol format. For the purpose of this
+document, "Server" is either QEMU or the QEMU Guest Agent, and
+"Client" is any application communicating with it via QMP.
 
 JSON data structures, when mentioned in this document, are always in the
 following format:
@@ -34,9 +34,8 @@ by the JSON standard:
 
 http://www.ietf.org/rfc/rfc7159.txt
 
-The protocol is always encoded in UTF-8 except for synchronization
-bytes (documented below); although thanks to json-string escape
-sequences, the server will reply using only the strict ASCII subset.
+The server expects its input to be encoded in UTF-8, and sends its
+output encoded in ASCII.
 
 For convenience, json-object members mentioned in this document will
 be in a certain order. However, in real protocol usage they can be in
@@ -215,16 +214,31 @@ Some events are rate-limited to at most one per second.  If additional
 dropped, and the last one is delayed.  "Similar" normally means same
 event type.  See qmp-events.txt for details.
 
-2.6 QGA Synchronization
+2.6 Forcing the JSON parser into known-good state
+-------------------------------------------------
+
+Incomplete or invalid input can leave the server's JSON parser in a
+state where it can't parse additional commands.  To get it back into
+known-good state, the client should provoke a lexical error.
+
+The cleanest way to do that is sending an ASCII control character
+other than '\t' (horizontal tab), '\r' (carriage return), or '\n' (new
+line).
+
+Sadly, older versions of QEMU can fail to flag this as an error.  If a
+client needs to deal with them, it should send a 0xFF byte.
+
+2.7 QGA Synchronization
 -----------------------
 
-When using QGA, an additional synchronization feature is built into
-the protocol.  If the Client sends a raw 0xFF sentinel byte (not valid
-JSON), then the Server will reset its state and discard all pending
-data prior to the sentinel.  Conversely, if the Client makes use of
-the 'guest-sync-delimited' command, the Server will send a raw 0xFF
-sentinel byte prior to its response, to aid the Client in discarding
-any data prior to the sentinel.
+When a client connects to QGA over a transport lacking proper
+connection semantics such as virtio-serial, QGA may have read partial
+input from a previous client.  The client needs to force QGA's parser
+into known-good state using the previous section's technique.
+Moreover, the client may receive output a previous client didn't read.
+To help with skipping that output, QGA provides the
+'guest-sync-delimited' command.  Refer to its documentation for
+details.
 
 
 3. QMP Examples

include/qapi/qmp/json-parser.h

@@ -1,5 +1,5 @@
 /*
- * JSON Parser 
+ * JSON Parser
  *
  * Copyright IBM, Corp. 2009
  *
@@ -11,12 +11,36 @@
  *
  */
 
-#ifndef QEMU_JSON_PARSER_H
-#define QEMU_JSON_PARSER_H
+#ifndef QAPI_QMP_JSON_PARSER_H
+#define QAPI_QMP_JSON_PARSER_H
 
-#include "qemu-common.h"
+typedef struct JSONLexer {
+    int start_state, state;
+    GString *token;
+    int x, y;
+} JSONLexer;
 
-QObject *json_parser_parse(GQueue *tokens, va_list *ap);
-QObject *json_parser_parse_err(GQueue *tokens, va_list *ap, Error **errp);
+typedef struct JSONMessageParser {
+    void (*emit)(void *opaque, QObject *json, Error *err);
+    void *opaque;
+    va_list *ap;
+    JSONLexer lexer;
+    int brace_count;
+    int bracket_count;
+    GQueue tokens;
+    uint64_t token_size;
+} JSONMessageParser;
+
+void json_message_parser_init(JSONMessageParser *parser,
+                              void (*emit)(void *opaque, QObject *json,
+                                           Error *err),
+                              void *opaque, va_list *ap);
+
+void json_message_parser_feed(JSONMessageParser *parser,
+                             const char *buffer, size_t size);
+
+void json_message_parser_flush(JSONMessageParser *parser);
+
+void json_message_parser_destroy(JSONMessageParser *parser);
 
 #endif

include/qapi/qmp/qerror.h

@@ -61,9 +61,6 @@
 #define QERR_IO_ERROR \
     "An IO error has occurred"
 
-#define QERR_JSON_PARSING \
-    "Invalid JSON syntax"
-
 #define QERR_MIGRATION_ACTIVE \
     "There's a migration process in progress"
 

include/qapi/qmp/qnum.h

@@ -25,7 +25,7 @@ typedef enum {
 
 /*
  * QNum encapsulates how our dialect of JSON fills in the blanks left
- * by the JSON specification (RFC 7159) regarding numbers.
+ * by the JSON specification (RFC 8259) regarding numbers.
  *
  * Conceptually, we treat number as an abstract type with three
  * concrete subtypes: floating-point, signed integer, unsigned

include/qemu/unicode.h

@@ -2,5 +2,6 @@
 #define QEMU_UNICODE_H
 
 int mod_utf8_codepoint(const char *s, size_t n, char **end);
+ssize_t mod_utf8_encode(char buf[], size_t bufsz, int codepoint);
 
 #endif

monitor.c

@@ -58,7 +58,6 @@
 #include "qapi/qmp/qnum.h"
 #include "qapi/qmp/qstring.h"
 #include "qapi/qmp/qjson.h"
-#include "qapi/qmp/json-streamer.h"
 #include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/qlist.h"
 #include "qom/object_interfaces.h"
@@ -4256,20 +4255,14 @@ static void monitor_qmp_bh_dispatcher(void *data)
 
 #define  QMP_REQ_QUEUE_LEN_MAX  (8)
 
-static void handle_qmp_command(JSONMessageParser *parser, GQueue *tokens)
+static void handle_qmp_command(void *opaque, QObject *req, Error *err)
 {
-    QObject *req, *id = NULL;
+    Monitor *mon = opaque;
+    QObject *id = NULL;
     QDict *qdict;
-    MonitorQMP *mon_qmp = container_of(parser, MonitorQMP, parser);
-    Monitor *mon = container_of(mon_qmp, Monitor, qmp);
-    Error *err = NULL;
     QMPRequest *req_obj;
 
-    req = json_parser_parse_err(tokens, NULL, &err);
-    if (!req && !err) {
-        /* json_parser_parse_err() sucks: can fail without setting @err */
-        error_setg(&err, QERR_JSON_PARSING);
-    }
+    assert(!req != !err);
 
     qdict = qobject_to(QDict, req);
     if (qdict) {
@@ -4465,7 +4458,8 @@ static void monitor_qmp_event(void *opaque, int event)
         monitor_qmp_response_flush(mon);
         monitor_qmp_cleanup_queues(mon);
         json_message_parser_destroy(&mon->qmp.parser);
-        json_message_parser_init(&mon->qmp.parser, handle_qmp_command);
+        json_message_parser_init(&mon->qmp.parser, handle_qmp_command,
+                                 mon, NULL);
         mon_refcount--;
         monitor_fdsets_cleanup();
         break;
@@ -4683,7 +4677,8 @@ void monitor_init(Chardev *chr, int flags)
 
     if (monitor_is_qmp(mon)) {
         qemu_chr_fe_set_echo(&mon->chr, true);
-        json_message_parser_init(&mon->qmp.parser, handle_qmp_command);
+        json_message_parser_init(&mon->qmp.parser, handle_qmp_command,
+                                 mon, NULL);
         if (mon->use_io_thread) {
             /*
              * Make sure the old iowatch is gone.  It's possible when

qapi/introspect.json

@@ -120,7 +120,7 @@
 ##
 # @JSONType:
 #
-# The four primitive and two structured types according to RFC 7159
+# The four primitive and two structured types according to RFC 8259
 # section 1, plus 'int' (split off 'number'), plus the obvious top
 # type 'value'.
 #

qapi/qmp-dispatch.c

@@ -14,7 +14,6 @@
 #include "qemu/osdep.h"
 #include "qapi/error.h"
 #include "qapi/qmp/dispatch.h"
-#include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qjson.h"
 #include "qapi/qmp/qbool.h"

qapi/qobject-input-visitor.c

@@ -725,11 +725,6 @@ Visitor *qobject_input_visitor_new_str(const char *str,
     if (is_json) {
         obj = qobject_from_json(str, errp);
         if (!obj) {
-            /* Work around qobject_from_json() lossage TODO fix that */
-            if (errp && !*errp) {
-                error_setg(errp, "JSON parse error");
-                return NULL;
-            }
             return NULL;
         }
         args = qobject_to(QDict, obj);

qga/main.c

@@ -18,7 +18,6 @@
 #include <syslog.h>
 #include <sys/wait.h>
 #endif
-#include "qapi/qmp/json-streamer.h"
 #include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qjson.h"
@@ -597,24 +596,20 @@ static void process_command(GAState *s, QDict *req)
 }
 
 /* handle requests/control events coming in over the channel */
-static void process_event(JSONMessageParser *parser, GQueue *tokens)
+static void process_event(void *opaque, QObject *obj, Error *err)
 {
-    GAState *s = container_of(parser, GAState, parser);
-    QObject *obj;
+    GAState *s = opaque;
     QDict *req, *rsp;
-    Error *err = NULL;
     int ret;
 
-    g_assert(s && parser);
-
     g_debug("process_event: called");
-    obj = json_parser_parse_err(tokens, NULL, &err);
+    assert(!obj != !err);
     if (err) {
         goto err;
     }
     req = qobject_to(QDict, obj);
     if (!req) {
-        error_setg(&err, QERR_JSON_PARSING);
+        error_setg(&err, "Input must be a JSON object");
         goto err;
     }
     if (!qdict_haskey(req, "execute")) {
@@ -1320,7 +1315,7 @@ static int run_agent(GAState *s, GAConfig *config, int socket_activation)
     s->command_state = ga_command_state_new();
     ga_command_state_init(s, s->command_state);
     ga_command_state_init_all(s->command_state);
-    json_message_parser_init(&s->parser, process_event);
+    json_message_parser_init(&s->parser, process_event, s, NULL);
 
 #ifndef _WIN32
     if (!register_signal_handlers()) {