Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Streaming #39

Merged
merged 13 commits into from

2 participants

This page is out of date. Refresh to see the latest.
Showing with 352 additions and 75 deletions.
  1. +56 −1 CHANGES.md
  2. +1 −0  Makefile
  3. +2 −0  TODO.md
  4. +71 −66 lib/jsontool.js
  5. +1 −1  package.json
  6. +1 −5 test/Makefile
  7. +5 −0 test/json3/cmd
  8. +3 −2 test/json3/expected.stdout
  9. +1 −0  test/stream-100-continue/cmd
  10. +1 −0  test/stream-100-continue/expected.exitCode
  11. +13 −0 test/stream-100-continue/expected.stdout
  12. +11 −0 test/stream-100-continue/input
  13. +12 −0 test/stream-flat-array-input/README.md
  14. +6 −0 test/stream-flat-array-input/arrays-complex.input
  15. +1 −0  test/stream-flat-array-input/arrays-on-same-line-fail.input
  16. +1 −0  test/stream-flat-array-input/arrays-on-same-line.input
  17. +2 −0  test/stream-flat-array-input/arrays.input
  18. +48 −0 test/stream-flat-array-input/cmd
  19. +1 −0  test/stream-flat-array-input/dir/a.json
  20. +1 −0  test/stream-flat-array-input/dir/b.json
  21. +1 −0  test/stream-flat-array-input/dir/c.json
  22. +100 −0 test/stream-flat-array-input/expected.stdout
  23. +1 −0  test/stream-flat-array-input/mixed-objects-and-arrays.input
  24. +6 −0 test/stream-flat-array-input/objects-complex.input
  25. +1 −0  test/stream-flat-array-input/objects-on-same-line-fail.input
  26. +1 −0  test/stream-flat-array-input/objects-on-same-line.input
  27. +2 −0  test/stream-flat-array-input/objects.input
  28. +1 −0  test/stream-flat-array-input/one.input
  29. +1 −0  test/stream-flat-array-input/two.input
View
57 CHANGES.md
@@ -1,11 +1,66 @@
# json (aka jsontool) Changelog
-## json 4.0.1 (not yet released)
+## json 5.0.1 (not yet released)
(nothing yet)
+## json 5.0.0
+
+- [**backward incompatible**, issue #35] Special case the output for **a single
+ lookup AND JSON output** (i.e. `-j` or `-o json*`) to only output the value
+ instead of the more general array or table that is necessary for multiple
+ lookups. For objects:
+
+ # Before:
+ $ echo '{"one": "un", "two": "deux", "three": "troix"}' | json -j one
+ {
+ "one": "un"
+ }
+
+ # After:
+ $ echo '{"one": "un", "two": "deux", "three": "troix"}' | jsondev -j one
+ "un"
+
+ # Unchanged:
+ $ echo '{"one": "un", "two": "deux", "three": "troix"}' | jsondev -j one two
+ {
+ "one": "un",
+ "two": "deux"
+ }
+
+ For arrays:
+
+ # Before:
+ $ echo '["a", "b", "c"]' | json -j 0
+ [
+ "a"
+ ]
+
+ # After:
+ $ echo '["a", "b", "c"]' | jsondev -j 0
+ "a"
+
+ # Unchanged:
+ $ echo '["a", "b", "c"]' | jsondev -j 0 1
+ [
+ "a",
+ "b"
+ ]
+
+ The motivation for this change was (a) the WTF of the first example above and
+ (b) issue #36 where one could no longer extract a single value using '-j' to
+ explicitly get JSON string quoting.
+
+
+
+## json 4.0.1
+
+- [issue #36] Turn off coloring for inspect output (`json -i`, `json -o
+ inspect`) if stdout is not a TTY.
+
+
## json 4.0.0
- Add `--validate` option to just validate (no processing and output)
View
1  Makefile
@@ -11,6 +11,7 @@ deps/JSON-js/json_parse.js:
# Ensure jsontool.js and package.json have the same version.
.PHONY: versioncheck
versioncheck:
+ [[ `cat package.json | json version` == `grep '^## ' CHANGES.md | head -1 | awk '{print $$3}'` ]]
[[ `cat package.json | bin/json version` == `grep '^var VERSION' lib/jsontool.js | awk -F'"' '{print $$2}'` ]]
.PHONY: docs
View
2  TODO.md
@@ -3,6 +3,8 @@
# someday/maybe
+- pjson-like coloring of JSON output: http://igorgue.com/pjson/
+
- fix this in man page (ronn fix?):
Index: /Users/trentm/tm/json/docs/json.1
View
137 lib/jsontool.js
@@ -411,44 +411,38 @@ function chunkEmitter(opts) {
var streaming = true;
var chunks = [];
var leftover = '';
- var splitter = /(})(\s*\n\s*)?({\s*")/;
- function stripHeaders (stream) {
- var headerLeftover = '';
- stream.on('data', function onBlock(block) {
- var s = headerLeftover + block;
- // Take off a leading HTTP header if any and pass it through.
- while (true) {
- if (s.slice(0,5) === "HTTP/") {
- var index = s.indexOf('\r\n\r\n');
- var sepLen = 4;
- if (index == -1) {
- index = s.indexOf('\n\n');
- sepLen = 2;
- }
- if (index != -1) {
- if (! opts.dropHeaders) {
- emit(s.slice(0, index+sepLen));
- }
- var is100Continue = (s.slice(0, 21) === "HTTP/1.1 100 Continue");
- s = s.slice(index+sepLen);
- if (is100Continue) {
- continue;
- }
- finishedHeaders = true;
- }
- } else {
- finishedHeaders = true;
+ var finishedHeaders = false;
+ function stripHeaders(s) {
+ // Take off a leading HTTP header if any and pass it through.
+ while (true) {
+ if (s.slice(0,5) === "HTTP/") {
+ var index = s.indexOf('\r\n\r\n');
+ var sepLen = 4;
+ if (index == -1) {
+ index = s.indexOf('\n\n');
+ sepLen = 2;
+ }
+ if (index != -1) {
+ if (! opts.dropHeaders) {
+ emit(s.slice(0, index+sepLen));
}
- headerLeftover = s;
- break;
+ var is100Continue = (s.slice(0, 21) === "HTTP/1.1 100 Continue");
+ s = s.slice(index+sepLen);
+ if (is100Continue) {
+ continue;
+ }
+ finishedHeaders = true;
}
- if (finishedHeaders) {
- stream.removeListener('on', onBlock);
- return headerLeftover;
+ } else {
+ finishedHeaders = true;
}
- });
+ break;
+ }
+ //console.warn("XXX stripHeaders done, finishedHeaders=%s", finishedHeaders)
+ return s;
}
- function emitChunks (block, emitter) {
+ function emitChunks(block, emitter) {
+ var splitter = /(})(\s*\n\s*)?({\s*")/;
if (block[0] !== '{') { // Only support streaming consecutive *objects*.
streaming = false;
chunks.push(block);
@@ -467,7 +461,6 @@ function chunkEmitter(opts) {
* 'a":"b"}' ]
*/
var bits = block.split(splitter);
- //console.warn("XXX bits: ", bits)
if (bits.length === 1) {
return block;
} else {
@@ -479,60 +472,72 @@ function chunkEmitter(opts) {
return bits[n] + bits[n+1];
}
}
- function addListeners(stream) {
+ function addDataListener(stream) {
stream.on('data', function (chunk) {
- if (!streaming) {
- chunks.push(chunk);
- return;
- }
var s = leftover + chunk;
- leftover = emitChunks(s, emitter);
- });
- stream.on('end', function () {
- if (!streaming) {
- emitter.emit('chunk', chunks.join(''));
- } else if (leftover) {
- leftover = emitChunks(leftover, emitter);
- emitter.emit('chunk', leftover);
+ if (!finishedHeaders) {
+ s = stripHeaders(s);
+ }
+ if (!finishedHeaders) {
+ leftover = s;
+ } else {
+ if (!streaming) {
+ chunks.push(chunk);
+ return;
+ }
+ leftover = emitChunks(s, emitter);
}
});
}
+
if (opts.inputFiles.length > 0) {
- // Stream each file in order. Strips headers from the first file.
+ // Stream each file in order.
var i = 0;
- function addEndListener (file) {
+ function addErrorListener(file) {
+ file.on('error', function (err) {
+ emitter.emit(
+ 'error',
+ format('could not read "%s": %s', opts.inputFiles[i], e)
+ );
+ });
+ }
+ function addEndListener(file) {
file.on('end', function () {
if (i < opts.inputFiles.length) {
- var next = opts.inputFiles[++i];
+ var next = opts.inputFiles[i++];
var nextFile = fs.createReadStream(next, {encoding: 'utf8'});
- addListeners(nextFile);
+ addErrorListener(nextFile);
addEndListener(nextFile);
+ addDataListener(nextFile);
} else {
+ if (!streaming) {
+ emitter.emit('chunk', chunks.join(''));
+ } else if (leftover) {
+ leftover = emitChunks(leftover, emitter);
+ emitter.emit('chunk', leftover);
+ }
emitter.emit('end');
}
});
}
- try {
- var first = fs.createReadStream(opts.inputFiles[i], {encoding: 'utf8'});
- leftover = stripHeaders(first) || '';
- addEndListener(first);
- addListeners(first);
- } catch (e) {
- emitter.emit(
- 'error',
- format('could not read "%s": %s', opts.inputFiles[i], e)
- );
- }
+ var first = fs.createReadStream(opts.inputFiles[i++], {encoding: 'utf8'});
+ addErrorListener(first);
+ addEndListener(first);
+ addDataListener(first);
} else {
// Streaming from stdin.
var stdin = process.openStdin();
stdin.setEncoding('utf8');
- leftover = stripHeaders(stdin) || '';
- console.warn("XXX 525 leftover: " + leftover);
+ addDataListener(stdin);
stdin.on('end', function () {
+ if (!streaming) {
+ emitter.emit('chunk', chunks.join(''));
+ } else if (leftover) {
+ leftover = emitChunks(leftover, emitter);
+ emitter.emit('chunk', leftover);
+ }
emitter.emit('end');
});
- addListeners(stdin);
}
return emitter;
}
View
2  package.json
@@ -1,7 +1,7 @@
{
"name": "jsontool",
"description": "a 'json' command for massaging JSON on the command line",
- "version": "4.0.1",
+ "version": "5.0.1",
"repository": {
"type": "git",
"url": "git://github.com/trentm/json.git"
View
6 test/Makefile
@@ -7,13 +7,9 @@ test:
# Test will all node versions (presumes install locations I use on my machine).
.PHONY: testall
-testall: test08 test06 test04 test09 testmaster
+testall: test08 test06 test04 test09
-.PHONY: testmaster
-testmaster:
- @echo "# Test node master (with node `$(HOME)/opt/node-master/bin/node --version`)"
- PATH="$(HOME)/opt/node-master/bin:$(PATH)" $(NODEUNIT) test.js
.PHONY: test08
test08:
@echo "# Test node 0.8.x (with node `$(HOME)/opt/node-0.8/bin/node --version`)"
View
5 test/json3/cmd
@@ -77,7 +77,12 @@ echo '[{"name":"trent", "age":38}, {"name":"ewan", "age":4}]' | $JSON -o json-0
# [{"name":"trent", "age":38}, {"name":"ewan", "age":4}]
echo '[{"name":"trent", "age":38}, {"name":"ewan", "age":4}]' | $JSON -o json-0 0 2
# [{"name":"trent", "age":38}]
+echo -n 'index 2:'
echo '[{"name":"trent", "age":38}, {"name":"ewan", "age":4}]' | $JSON -o json-0 2
+echo
+# (empty)
+echo -n 'index 2,3:'
+echo '[{"name":"trent", "age":38}, {"name":"ewan", "age":4}]' | $JSON -o json-0 2 3
# []
echo '{"0": "zero", "1": "one"}' | $JSON 0 1
# zero
View
5 test/json3/expected.stdout
@@ -66,10 +66,11 @@ ewan
"name": "trent",
"age": 38
}
-[{"name":"trent","age":38}]
+{"name":"trent","age":38}
[{"name":"trent","age":38},{"name":"ewan","age":4}]
[{"name":"trent","age":38}]
-[]
+index 2:
+index 2,3:[]
zero
one
{"0":"zero","1":"one"}
View
1  test/stream-100-continue/cmd
@@ -0,0 +1 @@
+cat input | ../../lib/jsontool.js -ga
View
1  test/stream-100-continue/expected.exitCode
@@ -0,0 +1 @@
+0
View
13 test/stream-100-continue/expected.stdout
@@ -0,0 +1,13 @@
+HTTP/1.1 100 Continue
+
+HTTP/1.1 201 Created
+Server: nginx/0.8.53
+Date: Thu, 14 Apr 2011 09:50:35 GMT
+Content-Type: application/json
+Connection: keep-alive
+Status: 201 Created
+Content-Length: 14
+
+{
+ "foo": "bar"
+}
View
11 test/stream-100-continue/input
@@ -0,0 +1,11 @@
+HTTP/1.1 100 Continue
+
+HTTP/1.1 201 Created
+Server: nginx/0.8.53
+Date: Thu, 14 Apr 2011 09:50:35 GMT
+Content-Type: application/json
+Connection: keep-alive
+Status: 201 Created
+Content-Length: 14
+
+{"foo": "bar"}
View
12 test/stream-flat-array-input/README.md
@@ -0,0 +1,12 @@
+Test handling of a flat array of objects as input:
+
+ $ echo '{"one": 1}{"two": 1}' | json
+ [
+ {
+ "one": 1
+ },
+ {
+ "two": 1
+ }
+ ]
+
View
6 test/stream-flat-array-input/arrays-complex.input
@@ -0,0 +1,6 @@
+ ["one", 1]
+["two", 2]
+["three", 3]
+ ["four", 4]
+
+["five", 5]
View
1  test/stream-flat-array-input/arrays-on-same-line-fail.input
@@ -0,0 +1 @@
+["a", "b"] ["c", "d"]
View
1  test/stream-flat-array-input/arrays-on-same-line.input
@@ -0,0 +1 @@
+["a", "b"]["c", "d"]
View
2  test/stream-flat-array-input/arrays.input
@@ -0,0 +1,2 @@
+["a", "b"]
+["c", "d"]
View
48 test/stream-flat-array-input/cmd
@@ -0,0 +1,48 @@
+JSON="node ../../lib/jsontool.js -q --group --array"
+
+echo "# simple"
+cat objects.input | $JSON
+echo ""
+echo "# simple (-f)"
+$JSON -f objects.input
+
+echo ""
+echo "# separate"
+cat one.input two.input | $JSON
+echo ""
+echo "# separate (-f)"
+$JSON -f one.input -f two.input
+
+echo ""
+echo "# on same line"
+cat objects-on-same-line.input | $JSON
+
+echo ""
+echo "# more complex"
+cat objects-complex.input | $JSON
+
+echo ""
+echo "# simple arrays"
+cat arrays.input | $JSON
+
+echo ""
+echo "# on same line"
+cat arrays-on-same-line.input | $JSON
+
+echo ""
+echo "# more complex"
+cat arrays-complex.input | $JSON
+
+echo ""
+echo "# cat json files"
+cat dir/*.json | $JSON
+
+echo ""
+echo "# objects on same line (space FAIL)"
+cat objects-on-same-line-fail.input | $JSON
+echo ""
+echo "# arrays on same line (space FAIL)"
+cat arrays-on-same-line-fail.input | $JSON
+echo ""
+echo "# mixed objects and arrays FAIL"
+cat mixed-objects-and-arrays.input | $JSON
View
1  test/stream-flat-array-input/dir/a.json
@@ -0,0 +1 @@
+{"a": "A"}
View
1  test/stream-flat-array-input/dir/b.json
@@ -0,0 +1 @@
+{"b": "B"}
View
1  test/stream-flat-array-input/dir/c.json
@@ -0,0 +1 @@
+{"c": "C"}
View
100 test/stream-flat-array-input/expected.stdout
@@ -0,0 +1,100 @@
+# simple
+{
+ "one": 1
+}
+{
+ "two": 2
+}
+
+# simple (-f)
+{
+ "one": 1
+}
+{
+ "two": 2
+}
+
+# separate
+{
+ "one": 1
+}
+{
+ "two": 2
+}
+
+# separate (-f)
+{
+ "one": 1
+}
+{
+ "two": 2
+}
+
+# on same line
+{
+ "one": 1
+}
+{
+ "two": 2
+}
+
+# more complex
+{
+ "one": 1
+}
+{
+ "two": 2
+}
+{
+ "three": 3
+}
+{
+ "four": 4
+}
+{
+ "five": 5
+}
+
+# simple arrays
+a
+b
+c
+d
+
+# on same line
+a
+b
+c
+d
+
+# more complex
+one
+1
+two
+2
+three
+3
+four
+4
+five
+5
+
+# cat json files
+{
+ "a": "A"
+}
+{
+ "b": "B"
+}
+{
+ "c": "C"
+}
+
+# objects on same line (space FAIL)
+{"one": 1} {"two": 2}
+
+# arrays on same line (space FAIL)
+["a", "b"] ["c", "d"]
+
+# mixed objects and arrays FAIL
+{"one": 1} ["a", "b"] {"two": 2}
View
1  test/stream-flat-array-input/mixed-objects-and-arrays.input
@@ -0,0 +1 @@
+{"one": 1} ["a", "b"] {"two": 2}
View
6 test/stream-flat-array-input/objects-complex.input
@@ -0,0 +1,6 @@
+ {"one": 1}
+{"two": 2}
+{"three": 3}
+ {"four": 4}
+
+{"five": 5}
View
1  test/stream-flat-array-input/objects-on-same-line-fail.input
@@ -0,0 +1 @@
+{"one": 1} {"two": 2}
View
1  test/stream-flat-array-input/objects-on-same-line.input
@@ -0,0 +1 @@
+{"one": 1}{"two": 2}
View
2  test/stream-flat-array-input/objects.input
@@ -0,0 +1,2 @@
+{"one": 1}
+{"two": 2}
View
1  test/stream-flat-array-input/one.input
@@ -0,0 +1 @@
+{"one": 1}
View
1  test/stream-flat-array-input/two.input
@@ -0,0 +1 @@
+{"two": 2}
Something went wrong with that request. Please try again.