Permalink
Browse files

o rework yajl api

  - remove yajl_status_parse_incomplete, replace with three
    flag settings
    - yajl_allow_multiple_values
    - yajl_forbid_trailing_garbage
    - yajl_forbid_partial_values

    In the new model, callers must consistently call yajl_parse_complete
    and check it's return. Two new parse errors have been introduced:
    "premature EOF" and "trailing garbage".

    yajl_test.c demonstrates the simplifying effect on calling code.

    adds 3 flags to yajl_test
         -g forbids trailing garbage
         -p forbids partial values
         -m allows multiple values to be parsed.

    and complementary tests.
  • Loading branch information...
gno committed Oct 21, 2010
1 parent d0ba1d5 commit 0795e14e64afada0919facdaf59191707e55549a
@@ -49,9 +49,7 @@ extern "C" {
yajl_status_ok,
/** a client callback returned zero, stopping the parse */
yajl_status_client_canceled,
/** The parse cannot yet complete because more json input text
* is required, call yajl_parse with the next buffer of input text.
* (pertinent only when stream parsing) */
/** not returned. here for cmpatability */
yajl_status_insufficient_data,
/** An error occured during the parse. Call yajl_get_error for
* more information about the encountered error */
@@ -132,6 +130,27 @@ extern "C" {
const yajl_alloc_funcs * allocFuncs,
void * ctx);


/**
* Forbid trailing garbage from following a JSON document.
* Whitespace is not considered garbage.
*/
YAJL_API void yajl_forbid_trailing_garbage(yajl_handle h);

/**
* Allow multiple values to be parsed by a single handle.
* The entire text must be valid JSON, and values can be seperated
* by any kind of whitespace.
*/
YAJL_API void yajl_allow_multiple_values(yajl_handle h);

/**
* Setting this flag causes the handle to enter an error
* state if yajl_parse_complete is called in the middle of
* a value.
*/
YAJL_API void yajl_forbid_partial_values(yajl_handle h);

/** free a parser handle */
YAJL_API void yajl_free(yajl_handle handle);

@@ -16,7 +16,7 @@
* 3. Neither the name of Lloyd Hilaiel nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* b
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
@@ -97,13 +97,33 @@ yajl_alloc(const yajl_callbacks * callbacks,
hand->lexer = yajl_lex_alloc(&(hand->alloc), allowComments, validateUTF8);
hand->bytesConsumed = 0;
hand->decodeBuf = yajl_buf_alloc(&(hand->alloc));
hand->flags = allow_trailing_garbage | allow_partial_values;
yajl_bs_init(hand->stateStack, &(hand->alloc));

yajl_bs_push(hand->stateStack, yajl_state_start);

return hand;
}


void
yajl_forbid_trailing_garbage(yajl_handle h)
{
h->flags &= ~allow_trailing_garbage;
}

void
yajl_allow_multiple_values(yajl_handle h)
{
h->flags |= allow_multiple_values;
}

void
yajl_forbid_partial_values(yajl_handle h)
{
h->flags &= ~allow_partial_values;
}

void
yajl_free(yajl_handle handle)
{
@@ -122,6 +142,7 @@ yajl_parse(yajl_handle hand, const unsigned char * jsonText,
return status;
}


yajl_status
yajl_parse_complete(yajl_handle hand)
{
@@ -131,7 +152,7 @@ yajl_parse_complete(yajl_handle hand)
* A very simple approach to this is to inject whitespace to terminate
* any number in the lex buffer.
*/
return yajl_parse(hand, (const unsigned char *)" ", 1);
return yajl_do_finish(hand);
}

unsigned char *
@@ -97,11 +97,11 @@ yajl_gen_alloc2(const yajl_print_t callback,
memcpy((void *) &(g->alloc), (void *) afs, sizeof(yajl_alloc_funcs));

if (config) {
char *indent = config->indentString;
const char *indent = config->indentString;
g->pretty = config->beautify;
g->indentString = config->indentString;
if (indent) {
for (indent; *indent; indent++) {
for (; *indent; indent++) {
if (*indent != '\n'
&& *indent != '\v'
&& *indent != '\f'
@@ -158,7 +158,7 @@ yajl_gen_free(yajl_gen g)

#define ENSURE_NOT_KEY \
if (g->state[g->depth] == yajl_gen_map_key || \
g->state[g->depth] == yajl_gen_map_start) { \
g->state[g->depth] == yajl_gen_map_start) { \
return yajl_gen_keys_must_be_strings; \
} \

@@ -163,6 +163,34 @@ yajl_render_error_string(yajl_handle hand, const unsigned char * jsonText,
}


yajl_status
yajl_do_finish(yajl_handle hand) {

yajl_status stat;
stat = yajl_do_parse(hand,(const unsigned char *)" ",1);

if (stat != yajl_status_ok) return stat;
// fprintf(stderr,"%d\n",hand->flags);

switch(yajl_bs_current(hand->stateStack))
{
case yajl_state_parse_error:
case yajl_state_lexical_error:
return yajl_status_error;
case yajl_state_got_value:
return yajl_status_ok;
default:
if (!(hand->flags & allow_partial_values))
{
yajl_bs_set(hand->stateStack, yajl_state_parse_error);
hand->parseError = "premature EOF";
return yajl_status_error;
}
return yajl_status_ok;
}

}

yajl_status
yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
unsigned int jsonTextLen)
@@ -171,35 +199,51 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
const unsigned char * buf;
unsigned int bufLen;
unsigned int * offset = &(hand->bytesConsumed);

*offset = 0;


around_again:
switch (yajl_bs_current(hand->stateStack)) {
case yajl_state_parse_complete:
return yajl_status_ok;
if (hand->flags & allow_multiple_values) {
yajl_bs_set(hand->stateStack, yajl_state_got_value);
goto around_again;
}
if (!(hand->flags & allow_trailing_garbage)) {
if (*offset != jsonTextLen) {
tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen,
offset, &buf, &bufLen);
if (tok != yajl_tok_eof) {
yajl_bs_set(hand->stateStack, yajl_state_parse_error);
hand->parseError = "trailing garbage";
}
goto around_again;
}
}
return yajl_status_ok;
case yajl_state_lexical_error:
case yajl_state_parse_error:
return yajl_status_error;
case yajl_state_start:
case yajl_state_got_value:
case yajl_state_map_need_val:
case yajl_state_array_need_val:
case yajl_state_array_start: {
case yajl_state_array_start: {
/* for arrays and maps, we advance the state for this
* depth, then push the state of the next depth.
* If an error occurs during the parsing of the nesting
* enitity, the state at this level will not matter.
* a state that needs pushing will be anything other
* than state_start */

yajl_state stateToPush = yajl_state_start;

tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen,
offset, &buf, &bufLen);

switch (tok) {
case yajl_tok_eof:
return yajl_status_insufficient_data;
return yajl_status_ok;
case yajl_tok_error:
yajl_bs_set(hand->stateStack, yajl_state_lexical_error);
goto around_again;
@@ -322,7 +366,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
/* got a value. transition depends on the state we're in. */
{
yajl_state s = yajl_bs_current(hand->stateStack);
if (s == yajl_state_start) {
if (s == yajl_state_start || s == yajl_state_got_value) {
yajl_bs_set(hand->stateStack, yajl_state_parse_complete);
} else if (s == yajl_state_map_need_val) {
yajl_bs_set(hand->stateStack, yajl_state_map_got_val);
@@ -345,7 +389,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
offset, &buf, &bufLen);
switch (tok) {
case yajl_tok_eof:
return yajl_status_insufficient_data;
return yajl_status_ok;
case yajl_tok_error:
yajl_bs_set(hand->stateStack, yajl_state_lexical_error);
goto around_again;
@@ -389,7 +433,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
yajl_bs_set(hand->stateStack, yajl_state_map_need_val);
goto around_again;
case yajl_tok_eof:
return yajl_status_insufficient_data;
return yajl_status_ok;
case yajl_tok_error:
yajl_bs_set(hand->stateStack, yajl_state_lexical_error);
goto around_again;
@@ -414,7 +458,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
yajl_bs_set(hand->stateStack, yajl_state_map_need_key);
goto around_again;
case yajl_tok_eof:
return yajl_status_insufficient_data;
return yajl_status_ok;
case yajl_tok_error:
yajl_bs_set(hand->stateStack, yajl_state_lexical_error);
goto around_again;
@@ -442,7 +486,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
yajl_bs_set(hand->stateStack, yajl_state_array_need_val);
goto around_again;
case yajl_tok_eof:
return yajl_status_insufficient_data;
return yajl_status_ok;
case yajl_tok_error:
yajl_bs_set(hand->stateStack, yajl_state_lexical_error);
goto around_again;
@@ -50,7 +50,8 @@ typedef enum {
yajl_state_map_need_key,
yajl_state_array_start,
yajl_state_array_got_val,
yajl_state_array_need_val
yajl_state_array_need_val,
yajl_state_got_value,
} yajl_state;

struct yajl_handle_t {
@@ -68,12 +69,21 @@ struct yajl_handle_t {
yajl_bytestack stateStack;
/* memory allocation routines */
yajl_alloc_funcs alloc;
/* bitfield */
unsigned int flags;
};

#define allow_trailing_garbage 0x01
#define allow_multiple_values 0x02
#define allow_partial_values 0x04

yajl_status
yajl_do_parse(yajl_handle handle, const unsigned char * jsonText,
unsigned int jsonTextLen);

yajl_status
yajl_do_finish(yajl_handle handle);

unsigned char *
yajl_render_error_string(yajl_handle hand, const unsigned char * jsonText,
unsigned int jsonTextLen, int verbose);
@@ -0,0 +1 @@
falsex
@@ -0,0 +1,3 @@
bool: false
parse error: trailing garbage
memory leaks: 0
@@ -0,0 +1 @@
nullx
@@ -0,0 +1,3 @@
null
parse error: trailing garbage
memory leaks: 0
@@ -0,0 +1 @@
truex
@@ -0,0 +1,3 @@
bool: true
parse error: trailing garbage
memory leaks: 0
@@ -0,0 +1 @@
{ "123":
@@ -0,0 +1,4 @@
map open '{'
key: '123'
parse error: premature EOF
memory leaks: 0
@@ -0,0 +1,3 @@
integer: 1221

This comment has been minimized.

@lloyd

lloyd Apr 20, 2011

missing a test file for this,

assuming content is
"1221 21"

added: 888d390

integer: 21
memory leaks: 0
@@ -0,0 +1,3 @@

{}
{}
@@ -0,0 +1,5 @@
map open '{'
map close '}'
map open '{'
map close '}'
memory leaks: 0
@@ -0,0 +1,7 @@
{}
[]
[]
"sdfasd"
123
{ "123" : 123 }
3.1e124
@@ -0,0 +1,14 @@
map open '{'
map close '}'
array open '['
array close ']'
array open '['
array close ']'
string: 'sdfasd'
integer: 123
map open '{'
key: '123'
integer: 123
map close '}'
double: 3.1e+124
memory leaks: 0
@@ -0,0 +1 @@
"abc
@@ -0,0 +1 @@
memory leaks: 0
@@ -0,0 +1,3 @@

{}
{}
@@ -0,0 +1,3 @@
map open '{'
map close '}'
memory leaks: 0
Oops, something went wrong.

0 comments on commit 0795e14

Please sign in to comment.