Add JSON tokeniser / lexer

Provides functions to read a single JSON token from a string or an hFILE.
User code can call this repeatedly within nested loops that correspond to
the expected structure of their JSON text to parse JSON input.

At some point we may wish to move the declarations from hts_internal.h
to the public API.  At present these functions return tokens in string
form only; before making this public, we should also interpret numeric
tokens -- see the TODO in the hts_json_token declaration.

Makefile

@@ -295,7 +295,7 @@ md5.o md5.pico: md5.c config.h $(htslib_hts_h)
 plugin.o plugin.pico: plugin.c config.h $(hts_internal_h) $(htslib_kstring_h)
 probaln.o probaln.pico: probaln.c config.h $(htslib_hts_h)
 realn.o realn.pico: realn.c config.h $(htslib_hts_h) $(htslib_sam_h)
-textutils.o textutils.pico: textutils.c config.h $(hts_internal_h)
+textutils.o textutils.pico: textutils.c config.h $(htslib_hfile_h) $(htslib_kstring_h) $(hts_internal_h)
 
 cram/cram_codecs.o cram/cram_codecs.pico: cram/cram_codecs.c config.h $(cram_h)
 cram/cram_decode.o cram/cram_decode.pico: cram/cram_decode.c config.h $(cram_h) cram/os.h $(htslib_hts_h)

hts_internal.h

@@ -32,6 +32,8 @@ DEALINGS IN THE SOFTWARE.  */
 extern "C" {
 #endif
 
+struct hFILE;
+
 /// Decode percent-encoded (URL-encoded) text
 /** On input, _dest_ should be a buffer at least the same size as _s_,
     and may be equal to _s_ to decode in place.  On output, _dest_ will be
@@ -54,6 +56,74 @@ size_t hts_base64_decoded_length(size_t len);
 int hts_decode_base64(char *dest, size_t *destlen, const char *s);
 
 
+/// Token structure returned by JSON lexing functions
+/** Token types correspond to scalar JSON values and selected punctuation
+as follows:
+  - `s` string
+  - `n` number
+  - `b` boolean literal
+  - `.` null literal
+  - `{`, `}`, `[`, `]` object and array delimiters
+  - `?` lexing error
+  - `\0` terminator at end of input
+*/
+typedef struct hts_json_token {
+    char type;    ///< Token type
+    char *str;    ///< Value as a C string (filled in for all token types)
+    // TODO Add other fields to fill in for particular data types, e.g.
+    // int inum;
+    // float fnum;
+} hts_json_token;
+
+/// Read one JSON token from a file
+/** @param str    The input C string
+    @param state  The input string state
+    @param token  On return, filled in with the token read
+    @return  The type of the token read
+
+On return, `token->str` points into the supplied input string, which
+is modified by having token-terminating characters overwritten as NULs.
+The `state` argument records the current position within `str` after each
+`hts_json_snext()` call, and should be set to 0 before the first call.
+*/
+char hts_json_snext(char *str, size_t *state, hts_json_token *token);
+
+/// Read and discard a complete JSON value from a string
+/** @param str    The input C string
+    @param state  The input string state, as per `hts_json_snext()`
+    @param type   If the first token of the value to be discarded has already
+                  been read, provide its type; otherwise `'\0'`
+    @return  One of `v` (success), `\0` (end of string), and `?` (lexing error)
+
+Skips a complete JSON value, which may be a single token or an entire object
+or array.
+*/
+char hts_json_sskip_value(char *str, size_t *state, char type);
+
+/// Read one JSON token from a file
+/** @param fp     The file stream
+    @param token  On return, filled in with the token read
+    @param kstr   Buffer used to store the token string returned
+    @return  The type of the token read
+
+The `kstr` buffer is used to store the string value of the token read,
+so `token->str` is only valid until the next time `hts_json_fnext()` is
+called with the same `kstr` argument.
+*/
+char hts_json_fnext(struct hFILE *fp, hts_json_token *token, kstring_t *kstr);
+
+/// Read and discard a complete JSON value from a file
+/** @param fp    The file stream
+    @param type  If the first token of the value to be discarded has already
+                 been read, provide its type; otherwise `'\0'`
+    @return  One of `v` (success), `\0` (EOF), and `?` (lexing error)
+
+Skips a complete JSON value, which may be a single token or an entire object
+or array.
+*/
+char hts_json_fskip_value(struct hFILE *fp, char type);
+
+
 // The <ctype.h> functions operate on ints such as are returned by fgetc(),
 // i.e., characters represented as unsigned-char-valued ints, or EOF.
 // To operate on plain chars (and to avoid warnings on some platforms),

textutils.c

@@ -24,6 +24,12 @@ DEALINGS IN THE SOFTWARE.  */
 
 #include <config.h>
 
+#include <stdio.h>
+#include <string.h>
+
+#include "htslib/hfile.h"
+#include "htslib/kstring.h"
+
 #include "hts_internal.h"
 
 static int dehex(char c)
@@ -91,3 +97,283 @@ int hts_decode_base64(char *dest, size_t *destlen, const char *s)
     *destlen = d - dest;
     return 0;
 }
+
+static char *encode_utf8(char *s, unsigned x)
+{
+    if (x >= 0x10000) {
+        *s++ = 0xF0 | (x >> 18);
+        *s++ = 0x80 | ((x >> 12) & 0x3F);
+        *s++ = 0x80 | ((x >> 6) & 0x3F);
+        *s++ = 0x80 | (x & 0x3F);
+    }
+    else if (x >= 0x800) {
+        *s++ = 0xE0 | (x >> 12);
+        *s++ = 0x80 | ((x >> 6) & 0x3F);
+        *s++ = 0x80 | (x & 0x3F);
+    }
+    else if (x >= 0x80) {
+        *s++ = 0xC0 | (x >> 6);
+        *s++ = 0x80 | (x & 0x3F);
+    }
+    else *s++ = x;
+
+    return s;
+}
+
+static char *sscan_string(char *s)
+{
+    char *d = s;
+    int d1, d2, d3, d4;
+
+    for (;;) switch (*s) {
+    case '\\':
+        switch (s[1]) {
+        case '\0': *d = '\0'; return s+1;
+        case 'b': *d++ = '\b'; s += 2; break;
+        case 'f': *d++ = '\f'; s += 2; break;
+        case 'n': *d++ = '\n'; s += 2; break;
+        case 'r': *d++ = '\r'; s += 2; break;
+        case 't': *d++ = '\t'; s += 2; break;
+        default:  *d++ = s[1]; s += 2; break;
+        case 'u':
+            if ((d1 = dehex(s[2])) >= 0 && (d2 = dehex(s[3])) >= 0 &&
+                (d3 = dehex(s[4])) >= 0 && (d4 = dehex(s[5])) >= 0) {
+                d = encode_utf8(d, d1 << 12 | d2 << 8 | d3 << 4 | d4);
+                s += 6;
+            }
+            break;
+        }
+        break;
+
+    case '"':
+        *d = '\0';
+        return s+1;
+
+    case '\0':
+        *d = '\0';
+        return s;
+
+    default:
+        *d++ = *s++;
+        break;
+    }
+}
+
+static void fscan_string(hFILE *fp, kstring_t *d)
+{
+    int c, d1, d2, d3, d4;
+
+    while ((c = hgetc(fp)) != EOF) switch (c) {
+    case '\\':
+        if ((c = hgetc(fp)) == EOF) return;
+        switch (c) {
+        case 'b': kputc('\b', d); break;
+        case 'f': kputc('\f', d); break;
+        case 'n': kputc('\n', d); break;
+        case 'r': kputc('\r', d); break;
+        case 't': kputc('\t', d); break;
+        default:  kputc(c,    d); break;
+        case 'u':
+            if ((c = hgetc(fp)) != EOF && (d1 = dehex(c)) >= 0 &&
+                (c = hgetc(fp)) != EOF && (d2 = dehex(c)) >= 0 &&
+                (c = hgetc(fp)) != EOF && (d3 = dehex(c)) >= 0 &&
+                (c = hgetc(fp)) != EOF && (d4 = dehex(c)) >= 0) {
+                char buf[8];
+                char *lim = encode_utf8(buf, d1 << 12 | d2 << 8 | d3 << 4 | d4);
+                kputsn(buf, lim - buf, d);
+            }
+            break;
+        }
+        break;
+
+    case '"':
+        return;
+
+    default:
+        kputc(c, d);
+        break;
+    }
+}
+
+static char token_type(hts_json_token *token)
+{
+    const char *s = token->str;
+
+    switch (*s) {
+    case 'f':
+        return (strcmp(s, "false") == 0)? 'b' : '?';
+    case 'n':
+        return (strcmp(s, "null") == 0)? '.' : '?';
+    case 't':
+        return (strcmp(s, "true") == 0)? 'b' : '?';
+    case '-':
+    case '0': case '1': case '2': case '3': case '4':
+    case '5': case '6': case '7': case '8': case '9':
+        return 'n';
+    default:
+        return '?';
+    }
+}
+
+char hts_json_snext(char *str, size_t *state, hts_json_token *token)
+{
+    char *s = &str[*state >> 2];
+    int hidden = *state & 3;
+
+    if (hidden) {
+        *state &= ~3;
+        return token->type = "?}]?"[hidden];
+    }
+
+#define STATE(s,h)  (((s) - str) << 2 | (h))
+
+    for (;;) switch (*s) {
+    case ' ':
+    case '\t':
+    case '\r':
+    case '\n':
+    case ',':
+    case ':':
+        s++;
+        continue;
+
+    case '\0':
+        return token->type = '\0';
+
+    case '{':
+    case '[':
+    case '}':
+    case ']':
+        *state = STATE(s+1, 0);
+        return token->type = *s;
+
+    case '"':
+        token->str = s+1;
+        *state = STATE(sscan_string(s+1), 0);
+        return token->type = 's';
+
+    default:
+        token->str = s;
+        s += strcspn(s, " \t\r\n,]}");
+        hidden = (*s == '}')? 1 : (*s == ']')? 2 : 0;
+        if (*s != '\0') *s++ = '\0';
+        *state = STATE(s, hidden);
+        return token->type = token_type(token);
+    }
+
+#undef STATE
+}
+
+char hts_json_fnext(struct hFILE *fp, hts_json_token *token, kstring_t *kstr)
+{
+    char peek;
+    int c;
+
+    for (;;) switch (c = hgetc(fp)) {
+    case ' ':
+    case '\t':
+    case '\r':
+    case '\n':
+    case ',':
+    case ':':
+        continue;
+
+    case EOF:
+        return token->type = '\0';
+
+    case '{':
+    case '[':
+    case '}':
+    case ']':
+        return token->type = c;
+
+    case '"':
+        kstr->l = 0;
+        fscan_string(fp, kstr);
+        if (kstr->l == 0) kputsn("", 0, kstr);
+        token->str = kstr->s;
+        return token->type = 's';
+
+    default:
+        kstr->l = 0;
+        kputc(c, kstr);
+        while (hpeek(fp, &peek, 1) == 1 && !strchr(" \t\r\n,]}", peek)) {
+            if ((c = hgetc(fp)) == EOF) break;
+            kputc(c, kstr);
+        }
+        token->str = kstr->s;
+        return token->type = token_type(token);
+    }
+}
+
+
+typedef char hts_json_nextfn(void *arg1, void *arg2, hts_json_token *token);
+
+static char skip_value(char type, hts_json_nextfn *next, void *arg1, void *arg2)
+{
+    hts_json_token token;
+    int level;
+
+    switch (type? type : next(arg1, arg2, &token)) {
+    case '\0':
+        return '\0';
+
+    case '?':
+    case '}':
+    case ']':
+        return '?';
+
+    case '{':
+    case '[':
+        level = 1;
+        break;
+
+    default:
+        return 'v';
+    }
+
+    while (level > 0)
+        switch (next(arg1, arg2, &token)) {
+        case '\0':
+            return '\0';
+
+        case '?':
+            return '?';
+
+        case '{':
+        case '[':
+            level++;
+            break;
+
+        case '}':
+        case ']':
+            --level;
+            break;
+
+        default:
+            break;
+        }
+
+    return 'v';
+}
+
+static char snext(void *arg1, void *arg2, hts_json_token *token)
+{
+    return hts_json_snext(arg1, arg2, token);
+}
+char hts_json_sskip_value(char *str, size_t *state, char type)
+{
+    return skip_value(type, snext, str, state);
+}
+
+static char fnext(void *arg1, void *arg2, hts_json_token *token)
+{
+    return hts_json_fnext(arg1, token, arg2);
+}
+char hts_json_fskip_value(struct hFILE *fp, char type)
+{
+    kstring_t str = { 0, 0, NULL };
+    char ret = skip_value(type, fnext, fp, &str);
+    free(str.s);
+    return ret;
+}