Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

utf8 support #11

Merged
merged 11 commits into from

2 participants

@substack
Collaborator

This pull request firstly refactors the test.js into a proper test suite that you can run with npm test and it will output TAP and give you an exit code of 0 on success. I then added a failing utf8 test so I could make it pass. The test suite will be useful for getting this module running with browser tests on testling-ci, which I will hack together shortly.

If you don't want any of the test commits you can just cherry-pick fba4e12 which has the actual fix but the tests are nice.

Fixes issue #7.

@creationix creationix merged commit d7160d9 into from
@creationix
Owner

Thanks James!

@creationix
Owner

Published to npm as jsonparse@0.0.4

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
This page is out of date. Refresh to see the latest.
View
7 jsonparse.js
@@ -110,8 +110,11 @@ proto.write = function (buffer) {
}
}else if (this.tState === STRING1){ // After open quote
n = buffer[i];
- // TODO: Handle native utf8 characters, this code assumes ASCII input
- if (n === 0x22) { this.tState = START; this.onToken(STRING, this.string); this.string = undefined; }
+ if (n >= 128) {
+ for (var j = i; buffer[j] >= 128 && j < buffer.length; j++);
+ this.string += buffer.slice(i, j).toString();
+ i = j - 1;
+ } else if (n === 0x22) { this.tState = START; this.onToken(STRING, this.string); this.string = undefined; }
else if (n === 0x5c) { this.tState = STRING2; }
else if (n >= 0x20) { this.string += String.fromCharCode(n); }
else { this.charError(buffer, i); }
View
7 package.json
@@ -8,6 +8,13 @@
"type": "git",
"url": "http://github.com/creationix/jsonparse.git"
},
+ "devDependencies": {
+ "tape": "~0.1.1",
+ "tap": "~0.3.3"
+ },
+ "scripts": {
+ "test": "tap test/*.js"
+ },
"bugs": "http://github.com/creationix/jsonparse/issues",
"engines": ["node >= 0.2.0"],
"license": "MIT",
View
69 test.js
@@ -1,69 +0,0 @@
-/*global Buffer*/
-var Parser = require('./jsonparse');
-var fs = require('fs');
-/*
-var t = new Tokenizer();
-t.onToken = function (token, value) {
- console.log("%s %s", token, value);
-}
-t.write(new Buffer("123false1.3true{}[]"));
-t.write(new Buffer("truefalsenullnulltruefalse"));
-t.write(new Buffer("tru"));
-t.write(new Buffer("efa"));
-t.write(new Buffer("lse"));
-t.write(new Buffer(JSON.stringify({"Hello":"World"})));
-*/
-// Make sure it's parsing correctly before any benchmarking is done
-var json = fs.readFileSync("samplejson/basic.json");
-var t = new Parser();
-var v;
-t.onValue = function (value) { v = value; };
-t.write(json);
-if (JSON.stringify(v) !== JSON.stringify(JSON.parse(json))) {
- throw new Error("Invalid parse result:\n" + JSON.stringify(v));
-}
-var p = new Parser();
-p.onValue = function (value) {
- var keys = this.stack.map(function (item) {
- return '.' + item.key;
- }, this);
- keys.push('.' + this.key);
- keys.shift();
- var o = JSON.stringify(value) || "";
- if (o.length > 80) {
- o = o.substr(0, 77) + "...";
- }
- console.log("value%s = %s", keys.join(''), o);
-};
-//p.write(JSON.stringify([1,2,3]));
-//p.write(JSON.stringify([]));
-//p.write(JSON.stringify([[]]));
-p.write('"""Hello""This\\"is""\\r\\n\\f\\t\\\\\\/\\""');
-p.write('"\\u039b\\u03ac\\u03bc\\u03b2\\u03b4\\u03b1"');
-p.write('"\\\\"');
-p.write('"\\/"');
-p.write('"\\""');
-p.write('[0,1,-1]');
-p.write('[1.0,1.1,-1.1,-1.0][-1][-0.1]');
-p.write('[6.02e23]');
-// p.write('{"name": "Tim", "age": 28}');
-// p.write('{}');
-//p.write('[{"name":null}]');
-//p.write('[{}]');
-//p.write('[{},[],{}]');
-//p.write('{"a":[],"b":{}}');
-//require('fs').readFile('samplejson/basic.json', function (err, buffer) {
-// if (err) { throw err; }
-// p.write(buffer);
-//});
-/*
-p.write(new Buffer("123false1.3true{}[]"));
-p.write(new Buffer('{ "name": "Tim", "age": 28 }'));
-p.write(new Buffer('{\n\t"name": "Tim",\n\t"age": 28\n}'));
-p.write(new Buffer('{\r\n\t"name": "Tim",\r\n\t"age": 28\r\n}'));
-p.write(new Buffer("truefalsenullnulltruefalse"));
-p.write(new Buffer("tru"));
-p.write(new Buffer("efa"));
-p.write(new Buffer("lse"));
-p.write(new Buffer(JSON.stringify({"Hello":"World"})));
-*/
View
54 test/primitives.js
@@ -0,0 +1,54 @@
+var test = require('tape');
+var Parser = require('../');
+
+var expected = [
+ [ [], '' ],
+ [ [], 'Hello' ],
+ [ [], 'This"is' ],
+ [ [], '\r\n\f\t\\/"' ],
+ [ [], 'Λάμβδα' ],
+ [ [], '\\' ],
+ [ [], '/' ],
+ [ [], '"' ],
+ [ [ 0 ], 0 ],
+ [ [ 1 ], 1 ],
+ [ [ 2 ], -1 ],
+ [ [], [ 0, 1, -1 ] ],
+ [ [ 0 ], 1 ],
+ [ [ 1 ], 1.1 ],
+ [ [ 2 ], -1.1 ],
+ [ [ 3 ], -1 ],
+ [ [], [ 1, 1.1, -1.1, -1 ] ],
+ [ [ 0 ], -1 ],
+ [ [], [ -1 ] ],
+ [ [ 0 ], -0.1 ],
+ [ [], [ -0.1 ] ],
+ [ [ 0 ], 6.019999999999999e+23 ],
+ [ [], [ 6.019999999999999e+23 ] ]
+];
+
+test('primitives', function (t) {
+ t.plan(23);
+
+ var p = new Parser();
+ p.onValue = function (value) {
+ var keys = this.stack
+ .slice(1)
+ .map(function (item) { return item.key })
+ .concat(this.key !== undefined ? this.key : [])
+ ;
+ t.deepEqual(
+ [ keys, value ],
+ expected.shift()
+ );
+ };
+
+ p.write('"""Hello""This\\"is""\\r\\n\\f\\t\\\\\\/\\""');
+ p.write('"\\u039b\\u03ac\\u03bc\\u03b2\\u03b4\\u03b1"');
+ p.write('"\\\\"');
+ p.write('"\\/"');
+ p.write('"\\""');
+ p.write('[0,1,-1]');
+ p.write('[1.0,1.1,-1.1,-1.0][-1][-0.1]');
+ p.write('[6.02e23]');
+});
View
38 test/utf8.js
@@ -0,0 +1,38 @@
+var test = require('tape');
+var Parser = require('../');
+
+test('3 bytes of utf8', function (t) {
+ t.plan(1);
+
+ var p = new Parser();
+ p.onValue = function (value) {
+ t.equal(value, '├──');
+ };
+
+ p.write('"├──"');
+});
+
+test('utf8 snowman', function (t) {
+ t.plan(1);
+
+ var p = new Parser();
+ p.onValue = function (value) {
+ t.equal(value, '');
+ };
+
+ p.write('"☃"');
+});
+
+test('utf8 with regular ascii', function (t) {
+ t.plan(4);
+
+ var p = new Parser();
+ var expected = [ "snow: ☃!", "xyz", "¡que!" ];
+ expected.push(expected.slice());
+
+ p.onValue = function (value) {
+ t.deepEqual(value, expected.shift());
+ };
+
+ p.write('["snow: ☃!","xyz","¡que!"]');
+});
Something went wrong with that request. Please try again.