diff --git a/ChangeLog-YAJL.txt b/ChangeLog-YAJL.txt new file mode 100644 index 0000000..4df07dc --- /dev/null +++ b/ChangeLog-YAJL.txt @@ -0,0 +1,129 @@ +1.0.12 + * Conrad Irwin - Parse null bytes correctly + * Mirek Rusin - fix LLVM warnings + * gno - Don't generate numbers for keys. closes #13 + * lth - various win32 fixes, including build documentation improvements + * John Stamp - Don't export private symbols. + * John Stamp - Install yajl_version.h, not the template. + * John Stamp - Don't use -fPIC for static lib. Cmake will automatically add it for the shared. + * lth 0 fix paths embedded in dylib upon installation on osx. closes #11 + +1.0.11 + * lth remove -Wno-missing-field-initializers for greater gcc compat (3.4.6) + +1.0.10 + * Brian Maher - yajl is now buildable without a c++ compiler present + * Brian Maher - fix header installation on OSX with cmake 2.8.0 installed + * lth & vitali - allow builder to specify alternate lib directory + for installation (i.e. lib64) + * Vitali Lovich - yajl version number now programatically accessible + * lth - prevent cmake from embedding rpaths in binaries. Static linking + makes this unneccesary. + +1.0.9 + * lth - fix inverted logic causing yajl_gen_double() to always fail on + win32 (thanks to Fredrik Kihlander for the report) + +1.0.8 + * Randall E. Barker - move dllexport defnitions so dlls with proper + exports can again be generated on windows + * lth - add yajl_get_bytes_consumed() which allows the client to + determine the offset as an error, as well as determine how + many bytes of an input buffer were consumed. + * lth - fixes to keep "error offset" up to date (like when the + client callback returns 0) + * Brian Maher - allow client to specify a printing function in + generation + +1.0.7 + * lth fix win32 build (isinf and isnan) + +1.0.6 + * lth fix several compiler warnings + * lth fix generation of invalid json from yajl_gen_double + (NaN is not JSON) + * jstamp support for combining short options in tools + * jstamp exit properly on errors from tools + * octo test success no longer depends on integer size + * max fix configure --prefix + +1.0.5 + * lth several performance improvements related to function + inlinin' + +1.0.4 + * lth fix broken utf8 validation for three & four byte represenations. + thanks to http://github.com/brianmario and + http://github.com/technoweenie + +1.0.3 + * lth fix syntax error in cplusplus extern "C" statements for wider + compiler support + +1.0.2 + * lth update doxygen documentation with new sample code, passing NULL + for allocation functions added in 1.0.0 + +1.0.1 + * lth resolve crash in json_reformatter due to incorrectly ordered + parameters. + +1.0.0 + * lth add 'make install' rules, thaks to Andrei Soroker for the + contribution. + * lth client may override allocation routines at generator or parser + allocation time + * tjw add yajl_parse_complete routine to allow client to explicitly + specify end-of-input, solving the "lonely number" case, where + json text consists only of an element with no explicit syntactic + end. + * tjw many new test cases + * tjw cleanup of code for symmetry and ease of reading + * lth integration of patches from Robert Varga which cleanup + compilation warnings on 64 bit linux + +0.4.0 + * lth buffer overflow bug in yajl_gen_double s/%lf/%g/ - thanks to + Eric Bergstrome + * lth yajl_number callback to allow passthrough of arbitrary precision + numbers to client. Thanks to Hatem Nassrat. + * lth yajl_integer now deals in long, instead of long long. This + combined with yajl_number improves compiler compatibility while + maintaining precision. + * lth better ./configure && make experience (still requires cmake and + ruby) + * lth fix handling of special characters hex 0F and 1F in yajl_encode + (thanks to Robert Geiger) + * lth allow leading zeros in exponents (thanks to Hatem Nassrat) + +0.3.0 + * lth doxygen documentation (html & man) generated as part of the + build + * lth many documentation updates. + * lth fix to work with older versions of cmake (don't use LOOSE_LOOP + constructs) + * lth work around different behavior of freebsd 4 scanf. initialize + parameter to scanf to zero. + * lth all tests run 32x with ranging buffer sizes to stress stream + parsing + * lth yajl_test accepts -b option to allow read buffer size to be + set + * lth option to validate UTF8 added to parser (argument in + yajl_parser_cfg) + * lth fix buffer overrun when chunk ends inside \u escaped text + * lth support client cancelation + +0.2.2 + * lth on windows build debug with C7 symbols and no pdb files. + +0.2.1 + * fix yajl_reformat and yajl_verify to work on arbitrarily sized + inputs. + * fix win32 build break, clean up all errors and warnings. + * fix optimized build flags. + +0.2.0 + * optionally support comments in input text + +0.1.0 + * Initial release diff --git a/numbers.c b/numbers.c index ce4c894..342c3fd 100644 --- a/numbers.c +++ b/numbers.c @@ -774,14 +774,15 @@ bf_random(Var arglist, Byte next, void *vdata, Objid progr) Var r; int e; int rnd; - const int range_l = - ((INTNUM_MAX > RAND_MAX ? RAND_MAX : (RAND_MAX - num)) + 1) % num; free_var(arglist); if (num <= 0) return make_error_pack(E_INVARG); + const int range_l = + ((INTNUM_MAX > RAND_MAX ? RAND_MAX : (RAND_MAX - num)) + 1) % num; + r.type = TYPE_INT; #if ((RAND_MAX <= 0) || 0!=(RAND_MAX & (RAND_MAX+1))) diff --git a/test/test_json.rb b/test/test_json.rb index 0977106..e1cd700 100644 --- a/test/test_json.rb +++ b/test/test_json.rb @@ -210,6 +210,88 @@ def test_that_the_argument_to_parse_json_must_be_a_string end end + def test_that_generate_json_turns_moo_binary_string_encoded_values_into_escaped_strings + run_test_as('wizard') do + assert_equal "{\"foo\":\"bar\\\"baz\"}", generate_json({"foo" => "bar\\\"baz"}) + assert_equal "{\"foo\":\"bar\\\\baz\"}", generate_json({"foo" => "bar\\\\baz"}) + assert_equal "{\"foo\":\"bar/baz\"}", generate_json({"foo" => "bar\\\/baz"}) + assert_equal "{\"foo\":\"bar/baz\"}", generate_json({"foo" => "bar\/baz"}) + assert_equal "{\"foo\":\"bar/baz\"}", generate_json({"foo" => "bar/baz"}) + + assert_equal "{\"foo\":\"bar\\bbaz\"}", generate_json({"foo" => "bar~08baz"}) + assert_equal "{\"foo\":\"bar\\fbaz\"}", generate_json({"foo" => "bar~0Cbaz"}) + assert_equal "{\"foo\":\"bar\\nbaz\"}", generate_json({"foo" => "bar~0Abaz"}) + assert_equal "{\"foo\":\"bar\\rbaz\"}", generate_json({"foo" => "bar~0Dbaz"}) + assert_equal "{\"foo\":\"bar\\tbaz\"}", generate_json({"foo" => "bar~09baz"}) + + # everything else is just passed, as is + assert_equal "{\"foo\":\"bar~00baz\"}", generate_json({"foo" => "bar~00baz"}) + assert_equal "{\"foo\":\"bar~zzbaz\"}", generate_json({"foo" => "bar~zzbaz"}) + assert_equal "{\"foo\":\"bar~f\"}", generate_json({"foo" => "bar~f"}) + assert_equal "{\"foo\":\"bar~\"}", generate_json({"foo" => "bar~"}) + end + end + + def test_that_parse_json_turns_escaped_strings_into_moo_binary_string_encoded_values + run_test_as('wizard') do + assert_equal({"foo" => "bar\"baz"}, parse_json('{\"foo\":\"bar\\\\\\"baz\"}')) + assert_equal({"foo" => "bar\\baz"}, parse_json('{\"foo\":\"bar\\\\\\\\baz\"}')) + assert_equal({"foo" => "bar/baz"}, parse_json('{\"foo\":\"bar\\\\/baz\"}')) + assert_equal({"foo" => "bar/baz"}, parse_json('{\"foo\":\"bar\\/baz\"}')) + assert_equal({"foo" => "bar/baz"}, parse_json('{\"foo\":\"bar/baz\"}')) + + assert_equal({"foo" => "bar~08baz"}, parse_json('{\"foo\":\"bar\\\\bbaz\"}')) + assert_equal({"foo" => "bar~0Cbaz"}, parse_json('{\"foo\":\"bar\\\\fbaz\"}')) + assert_equal({"foo" => "bar~0Abaz"}, parse_json('{\"foo\":\"bar\\\\nbaz\"}')) + assert_equal({"foo" => "bar~0Dbaz"}, parse_json('{\"foo\":\"bar\\\\rbaz\"}')) + assert_equal({"foo" => "bar~09baz"}, parse_json('{\"foo\":\"bar\\\\tbaz\"}')) + + # ignore other escapes... this includes \u (Unicode) for the time being + assert_equal(E_INVARG, parse_json('{\"foo\":\"bar\\\\ubaz\"}')) + assert_equal(E_INVARG, parse_json('{\"foo\":\"bar\\\\abaz\"}')) + assert_equal(E_INVARG, parse_json('{\"foo\":\"bar\\\\zbaz\"}')) + assert_equal(E_INVARG, parse_json('{\"foo\":\"bar\\\\\"}')) + end + end + + def test_that_round_tripping_works + run_test_as('wizard') do + assert(simplify(command(%Q|; x = ["foo" -> "bar\\\"baz"]; return x == parse_json(generate_json(x)); |))) + assert(simplify(command(%Q|; x = ["foo" -> "bar\\\\baz"]; return x == parse_json(generate_json(x)); |))) + assert(simplify(command(%Q|; x = ["foo" -> "bar\\\/baz"]; return x == parse_json(generate_json(x)); |))) + assert(simplify(command(%Q|; x = ["foo" -> "bar\/baz"]; return x == parse_json(generate_json(x)); |))) + assert(simplify(command(%Q|; x = ["foo" -> "bar/baz"]; return x == parse_json(generate_json(x)); |))) + + assert(simplify(command(%Q|; x = ["foo" -> "bar~08baz"]; return x == parse_json(generate_json(x)); |))) + assert(simplify(command(%Q|; x = ["foo" -> "bar~0Cbaz"]; return x == parse_json(generate_json(x)); |))) + assert(simplify(command(%Q|; x = ["foo" -> "bar~0Abaz"]; return x == parse_json(generate_json(x)); |))) + assert(simplify(command(%Q|; x = ["foo" -> "bar~0Dbaz"]; return x == parse_json(generate_json(x)); |))) + assert(simplify(command(%Q|; x = ["foo" -> "bar~09baz"]; return x == parse_json(generate_json(x)); |))) + + assert(simplify(command(%q|; x = "{\\"foo\\": \\"bar\\\\\\"baz\\"}"; return x == generate_json(parse_json(x)); |))) + assert(simplify(command(%q|; x = "{\\"foo\\": \\"bar\\\\\\\baz\\"}"; return x == generate_json(parse_json(x)); |))) + assert(simplify(command(%q|; x = "{\\"foo\\": \\"bar\\\\\\/baz\\"}"; return x == generate_json(parse_json(x)); |))) + assert(simplify(command(%q|; x = "{\\"foo\\": \\"bar\\\\/baz\\"}"; return x == generate_json(parse_json(x)); |))) + assert(simplify(command(%q|; x = "{\\"foo\\": \\"bar/baz\\"}"; return x == generate_json(parse_json(x)); |))) + + assert(simplify(command(%q|; x = "{\\"foo\\": \\"bar\\\\\\bbaz\\"}"; return x == generate_json(parse_json(x)); |))) + assert(simplify(command(%q|; x = "{\\"foo\\": \\"bar\\\\\\fbaz\\"}"; return x == generate_json(parse_json(x)); |))) + assert(simplify(command(%q|; x = "{\\"foo\\": \\"bar\\\\\\nbaz\\"}"; return x == generate_json(parse_json(x)); |))) + assert(simplify(command(%q|; x = "{\\"foo\\": \\"bar\\\\\\rbaz\\"}"; return x == generate_json(parse_json(x)); |))) + assert(simplify(command(%q|; x = "{\\"foo\\": \\"bar\\\\\\tbaz\\"}"; return x == generate_json(parse_json(x)); |))) + end + end + + def test_that_round_tripping_works_on_fuzzy_inputs + run_test_as('wizard') do + with_mutating_binary_string("~A7~CED~8E~D2L~16a~F6~F2~01UZ2~BC~B0)~EC~02~86v~CD~9B~05~E66~F3.vx<~F0") do |g| + 100.times do + assert(simplify(command(%Q|; x = "#{g.next}"; return x == parse_json(generate_json(x)); |))) + end + end + end + end + def generate_json(value, mode = nil) if mode.nil? simplify command %Q|; return generate_json(#{value_ref(value)});| diff --git a/test/test_math.rb b/test/test_math.rb new file mode 100644 index 0000000..354edca --- /dev/null +++ b/test/test_math.rb @@ -0,0 +1,26 @@ +require 'test_helper' + +class TestMath < Test::Unit::TestCase + + def test_that_random_0_is_invalid + run_test_as('programmer') do + assert_equal E_INVARG, simplify(command(%Q|; return random(0); |)) + end + end + + def test_that_random_1_returns_1 + run_test_as('programmer') do + assert_equal 1, simplify(command(%Q|; return random(1); |)) + end + end + + def test_that_random_returns_a_number_between_1_and_2147483647 + run_test_as('programmer') do + 1000.times do + r = simplify command %Q|; return random(); | + assert r > 0 && r <= 2147483647 + end + end + end + +end diff --git a/yajl_encode.c b/yajl_encode.c index 830b530..6f9e43e 100644 --- a/yajl_encode.c +++ b/yajl_encode.c @@ -37,13 +37,6 @@ #include #include -static void CharToHex(unsigned char c, char * hexBuf) -{ - const char * hexchar = "0123456789ABCDEF"; - hexBuf[0] = hexchar[c >> 4]; - hexBuf[1] = hexchar[c & 0x0F]; -} - void yajl_string_encode(yajl_buf buf, const unsigned char * str, unsigned int len) @@ -58,30 +51,34 @@ yajl_string_encode2(const yajl_print_t print, unsigned int len) { unsigned int beg = 0; - unsigned int end = 0; - char hexBuf[7]; - hexBuf[0] = '\\'; hexBuf[1] = 'u'; hexBuf[2] = '0'; hexBuf[3] = '0'; - hexBuf[6] = 0; + unsigned int end = 0; while (end < len) { - const char * escaped = NULL; - switch (str[end]) { - case '\r': escaped = "\\r"; break; - case '\n': escaped = "\\n"; break; - case '\\': escaped = "\\\\"; break; - /* case '/': escaped = "\\/"; break; */ - case '"': escaped = "\\\""; break; - case '\f': escaped = "\\f"; break; - case '\b': escaped = "\\b"; break; - case '\t': escaped = "\\t"; break; - default: - if ((unsigned char) str[end] < 32) { - CharToHex(str[end], hexBuf + 4); - escaped = hexBuf; - } - break; - } - if (escaped != NULL) { + if (end < len - 2 && str[end] == '~' && str[end + 1] == '0') { + const char * escaped = NULL; + switch (str[end + 2]) { + case '8': escaped = "\\b"; break; + case 'c': case 'C': escaped = "\\f"; break; + case 'a': case 'A': escaped = "\\n"; break; + case 'd': case 'D': escaped = "\\r"; break; + case '9': escaped = "\\t"; break; + default: + break; + } + if (escaped != NULL) { + print(ctx, (const char *) (str + beg), end - beg); + print(ctx, escaped, (unsigned int)strlen(escaped)); + beg = (end += 3); + } else { + ++end; + } + } else if (str[end] == '"') { + const char * escaped = "\\\""; + print(ctx, (const char *) (str + beg), end - beg); + print(ctx, escaped, (unsigned int)strlen(escaped)); + beg = ++end; + } else if (str[end] == '\\') { + const char * escaped = "\\\\"; print(ctx, (const char *) (str + beg), end - beg); print(ctx, escaped, (unsigned int)strlen(escaped)); beg = ++end; @@ -92,96 +89,25 @@ yajl_string_encode2(const yajl_print_t print, print(ctx, (const char *) (str + beg), end - beg); } -static void hexToDigit(unsigned int * val, const unsigned char * hex) -{ - unsigned int i; - for (i=0;i<4;i++) { - unsigned char c = hex[i]; - if (c >= 'A') c = (c & ~0x20) - 7; - c -= '0'; - assert(!(c & 0xF0)); - *val = (*val << 4) | c; - } -} - -static void Utf32toUtf8(unsigned int codepoint, char * utf8Buf) -{ - if (codepoint < 0x80) { - utf8Buf[0] = (char) codepoint; - utf8Buf[1] = 0; - } else if (codepoint < 0x0800) { - utf8Buf[0] = (char) ((codepoint >> 6) | 0xC0); - utf8Buf[1] = (char) ((codepoint & 0x3F) | 0x80); - utf8Buf[2] = 0; - } else if (codepoint < 0x10000) { - utf8Buf[0] = (char) ((codepoint >> 12) | 0xE0); - utf8Buf[1] = (char) (((codepoint >> 6) & 0x3F) | 0x80); - utf8Buf[2] = (char) ((codepoint & 0x3F) | 0x80); - utf8Buf[3] = 0; - } else if (codepoint < 0x200000) { - utf8Buf[0] =(char)((codepoint >> 18) | 0xF0); - utf8Buf[1] =(char)(((codepoint >> 12) & 0x3F) | 0x80); - utf8Buf[2] =(char)(((codepoint >> 6) & 0x3F) | 0x80); - utf8Buf[3] =(char)((codepoint & 0x3F) | 0x80); - utf8Buf[4] = 0; - } else { - utf8Buf[0] = '?'; - utf8Buf[1] = 0; - } -} - void yajl_string_decode(yajl_buf buf, const unsigned char * str, unsigned int len) { unsigned int beg = 0; - unsigned int end = 0; + unsigned int end = 0; while (end < len) { if (str[end] == '\\') { - char utf8Buf[5]; const char * unescaped = "?"; yajl_buf_append(buf, str + beg, end - beg); switch (str[++end]) { - case 'r': unescaped = "\r"; break; - case 'n': unescaped = "\n"; break; + case '"': unescaped = "\""; break; case '\\': unescaped = "\\"; break; case '/': unescaped = "/"; break; - case '"': unescaped = "\""; break; - case 'f': unescaped = "\f"; break; - case 'b': unescaped = "\b"; break; - case 't': unescaped = "\t"; break; - case 'u': { - unsigned int codepoint = 0; - hexToDigit(&codepoint, str + ++end); - end+=3; - /* check if this is a surrogate */ - if ((codepoint & 0xFC00) == 0xD800) { - end++; - if (str[end] == '\\' && str[end + 1] == 'u') { - unsigned int surrogate = 0; - hexToDigit(&surrogate, str + end + 2); - codepoint = - (((codepoint & 0x3F) << 10) | - ((((codepoint >> 6) & 0xF) + 1) << 16) | - (surrogate & 0x3FF)); - end += 5; - } else { - unescaped = "?"; - break; - } - } - - Utf32toUtf8(codepoint, utf8Buf); - unescaped = utf8Buf; - - if (codepoint == 0) { - yajl_buf_append(buf, unescaped, 1); - beg = ++end; - continue; - } - - break; - } + case 'b': unescaped = "~08"; break; + case 'f': unescaped = "~0C"; break; + case 'n': unescaped = "~0A"; break; + case 'r': unescaped = "~0D"; break; + case 't': unescaped = "~09"; break; default: assert("this should never happen" == NULL); }