Permalink
Browse files

Handle left-over buffer on end of input.

+ If remaining buffer is a newline character that we were waiting to pair with another newline character, then simply emit the 'eol' token.
+ Any other remaining buffer should be treated as an error.
  • Loading branch information...
1 parent 3ffd20c commit b7a5c0879f40cb2df1a89e9adfbcb850c8b7d723 @mpareja committed Feb 14, 2013
Showing with 54 additions and 24 deletions.
  1. +18 −9 scanner.js
  2. +36 −15 test.js
View
@@ -3,7 +3,7 @@ module.exports = function () {
var buffer = '';
var position = 0;
- return through(ondata/*, onend */);
+ return through(ondata, onend);
function ondata(data) {
data = buffer + data;
@@ -68,6 +68,7 @@ module.exports = function () {
function eol(optionalChar) {
consume(); // \r or \n
if (c === null) {
+ // wait for more input before deciding if this is the whole line-feed
return null;
}
if (c === optionalChar) {
@@ -95,16 +96,24 @@ module.exports = function () {
return c !== null;
}
- function tokenize(type, value) {
- var token = { type: type };
- if (value !== undefined) {
- token.value = value;
- }
- return token;
- }
}
function onend() {
- // TODO: just raise unexpected EOF if buffer remains
+ if (!buffer) {
+ this.queue(null); // end
+ } else if (buffer === '\r' || buffer === '\n') {
+ this.queue(tokenize('eol'));
+ this.queue(null); // end
+ } else {
+ this.emit('error', new Error('Unexpected end of stream: "' + buffer + '"'));
+ }
}
};
+
+function tokenize(type, value) {
+ var token = { type: type };
+ if (value !== undefined) {
+ token.value = value;
+ }
+ return token;
+}
View
51 test.js
@@ -15,17 +15,42 @@ test('"first","second",third', [['first', 'second', 'third']]);
test('first,"seco\nnd",third', [['first', 'seco\nnd', 'third']]);
test('first,"seco,nd",third', [['first', 'seco,nd', 'third']]);
-run();
+var tests = generateTests().concat(testErrors);
+async.series(tests, function (err) {
+ if (err) {
+ console.log('FAIL!');
+ console.log(err.message);
+ process.exit(1);
+ } else {
+ console.log('Pass.');
+ process.exit(0);
+ }
+});
function test(input, expected) {
queue.push({input: input, expected: expected});
}
-function run() {
- var fns = queue.map(function (test) {
+function testErrors(cb) {
+ var s = decsv();
+ s.on('data', function () {
+ cb(new Error('Should not have received incomplete data'));
+ });
+ s.on('error', function (e) {
+ cb(null);
+ });
+ s.on('end', function () {
+ cb(new Error('End called before error.'));
+ });
+ s.write('"first');
+ s.end();
+}
+
+function generateTests() {
+ return queue.map(function (test) {
return function (cb) {
var d = decsv();
- var found = [], errors = [], scanned = [], parsed = [];
+ var found = [], scanned = [], parsed = [];
d.on('data', function (values) {
found.push(values);
});
@@ -36,27 +61,23 @@ function run() {
} catch (e) {
console.log('scanned:'); console.log(scanned);
console.log('parsed:'); console.log(parsed);
+ console.log('input:' + test.input);
+ console.log('expected:' + test.expected);
cb(e);
}
});
d.on('error', function (err) {
- errors.push(err);
+ console.log('scanned:'); console.log(scanned);
+ console.log('parsed:'); console.log(parsed);
+ console.log('input:' + test.input);
+ console.log('expected:' + test.expected);
+ cb(err);
});
d._scanner.on('data', function (data) { scanned.push(data); });
d._parser.on('data', function (data) { parsed.push(data); });
d.write(test.input);
d.end();
};
});
- async.series(fns, function (err) {
- if (err) {
- console.log('FAIL!');
- console.log(err.message);
- process.exit(1);
- } else {
- console.log('Pass.');
- process.exit(0);
- }
- });
}

0 comments on commit b7a5c08

Please sign in to comment.