Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

initial

  • Loading branch information...
commit f59e9fd846a8eb96f59b77fab58c19ff033ff12c 0 parents
Dominic Tarr dominictarr authored
3  .gitignore
... ... @@ -0,0 +1,3 @@
  1 +node_modules
  2 +node_modules/*
  3 +npm_debug.log
13 examples/all_docs.js
... ... @@ -0,0 +1,13 @@
  1 +var request = require('request')
  2 + , JSONStream = require('JSONStream')
  3 + , es = require('event-stream')
  4 +
  5 +var parser = JSONStream.parse(['rows', /./]) //emit parts that match this path (any element of the rows array)
  6 + , req = request({url: 'http://isaacs.couchone.com/registry/_all_docs'})
  7 + , logger = es.mapSync(function (data) { //create a stream that logs to stderr,
  8 + console.error(data)
  9 + return data
  10 + })
  11 +
  12 +req.pipe(parser)
  13 +parser.pipe(logger)
80 index.js
... ... @@ -0,0 +1,80 @@
  1 +
  2 +var Parser = require('jsonparse')
  3 + , Stream = require('stream').Stream
  4 +
  5 +/*
  6 +
  7 + the value of this.stack that creationix's jsonparse has is weird.
  8 +
  9 + it makes this code ugly, but his problem is way harder that mine,
  10 + so i'll forgive him.
  11 +
  12 +
  13 +
  14 +*/
  15 +
  16 +exports.parse = function (path) {
  17 +
  18 + var stream = new Stream()
  19 + var parser = new Parser()
  20 + var count = 0
  21 + if(!path.length)
  22 + path = null
  23 + parser.onValue = function () {
  24 + if(!this.root && this.stack.length == 1){
  25 + stream.root = this.value
  26 + }
  27 + if(!path || this.stack.length !== path.length)
  28 + return
  29 + var _path = []
  30 + for( var i = 0; i < (path.length - 1); i++) {
  31 + var key = path[i]
  32 + var c = this.stack[1 + (+i)]
  33 +
  34 + if(!c) {
  35 + console.log(c, this.stack.length)
  36 + return
  37 + }
  38 + var m =
  39 + ( 'string' === typeof key
  40 + ? c.key == key
  41 + : key.exec(c.key))
  42 + _path.push(c.key)
  43 +
  44 + if(!m)
  45 + return
  46 +
  47 + }
  48 + var c = this
  49 +
  50 + var key = path[path.length - 1]
  51 + var m =
  52 + ( 'string' === typeof key
  53 + ? c.key == key
  54 + : key.exec(c.key))
  55 + if(!m)
  56 + return
  57 + _path.push(c.key)
  58 +
  59 + count ++
  60 + stream.emit('data', this.value[this.key])
  61 + }
  62 +
  63 +
  64 + parser.onError = function (err) {
  65 + stream.emit('error', err)
  66 + }
  67 + stream.readable = true
  68 + stream.writable = true
  69 + stream.write = function (chunk) {
  70 + parser.write(chunk)
  71 + }
  72 + stream.end = function (data) {
  73 + if(data)
  74 + stream.write(data)
  75 + if(!count)
  76 + stream.emit('data', stream.root)
  77 + stream.emit('end')
  78 + }
  79 + return stream
  80 +}
11 package.json
... ... @@ -0,0 +1,11 @@
  1 +{ "name": "JSONStream"
  2 +, "version": "0.0.0"
  3 +, "description": "rawStream.pipe(JSONStream.parse()).pipe(streamOfObjects)"
  4 +, "homepage": "http://github.com/dominictarr/JSONStream"
  5 +, "repository":
  6 + { "type": "git"
  7 + , "url": "https://github.com/dominictarr/JSONStream.git" }
  8 +, "dependencies": {}
  9 +, "devDependencies": {}
  10 +, "author": "Dominic Tarr <dominic.tarr@gmail.com> (http://bit.ly/dominictarr)"
  11 +, "scripts": { "test": "meta-test test/*.js" } }
84 readme.markdown
Source Rendered
... ... @@ -0,0 +1,84 @@
  1 +# JSONStream
  2 +
  3 +streaming JSON.parse and stringify
  4 +
  5 +## example
  6 +
  7 +in node v0.4.x
  8 +```js
  9 +
  10 +var request = require('request')
  11 + , JSONStream = require('JSONStream')
  12 + , es = require('event-stream')
  13 +
  14 +var parser = JSONStream.parse(['rows', /./]) //emit parts that match this path (any element of the rows array)
  15 + , req = request({url: 'http://isaacs.couchone.com/registry/_all_docs'})
  16 +```
  17 +in node 0.4.x
  18 +
  19 +``` js
  20 +
  21 +req.pipe(parser)
  22 +parser.pipe(es.log(''))
  23 +
  24 +```
  25 +
  26 +in node v0.5.x
  27 +
  28 +```js
  29 +req.pipe(parser).pipe(es.log(''))
  30 +
  31 +```
  32 +
  33 +## JSONStream.parse(path)
  34 +
  35 +usally, a json API will return a list of objects.
  36 +
  37 +`path` should be an array of property names and/or `RedExp`s.
  38 +any object that matches the path will be emitted as 'data' (and `pipe()`d down stream)
  39 +
  40 +if `path` is empty or null, or if no matches are made:
  41 +JSONStream.parse will only emit 'data' once, emitting the root object.
  42 +
  43 +for example, couchdb returns views like this:
  44 +
  45 +``` bash
  46 +curl -sS localhost:5984/tests/_all_docs
  47 +```
  48 +returns this:
  49 +
  50 +``` js
  51 +{"total_rows":129,"offset":0,"rows":[
  52 + { "id":"change1_0.6995461115147918"
  53 + , "key":"change1_0.6995461115147918"
  54 + , "value":{"rev":"1-e240bae28c7bb3667f02760f6398d508"}
  55 + , "doc":{
  56 + "_id": "change1_0.6995461115147918"
  57 + , "_rev": "1-e240bae28c7bb3667f02760f6398d508","hello":1}
  58 + },
  59 +{"id":"change2_0.6995461115147918","key":"change2_0.6995461115147918","value":{"rev":"1-13677d36b98c0c075145bb8975105153"},"doc":{"_id":"change2_0.6995461115147918","_rev":"1-13677d36b98c0c075145bb8975105153","hello":2}},
  60 +...
  61 +]}
  62 +
  63 +```
  64 +
  65 +we are probably interested in the `rows.*.docs`
  66 +
  67 +create a `Stream` that parses the documents from the feed like this:
  68 +
  69 +``` js
  70 +JSONStream.parse(['rows', /./, 'doc']) //rows, ANYTHING, doc
  71 +```
  72 +
  73 +awesome!
  74 +
  75 +## todo
  76 +
  77 + * JSONStream.stringify()
  78 +
  79 +## Acknowlegements
  80 +
  81 + this module depends on https://github.com/creationix/jsonparse
  82 + by Tim Caswell
  83 + and also thanks to Florent Jaby for teaching me about parsing with:
  84 + https://github.com/Floby/node-json-streams
4,030 test/fixtures/all_npm.json
4,030 additions, 0 deletions not shown
35 test/test.js
... ... @@ -0,0 +1,35 @@
  1 +
  2 +
  3 +var fs = require ('fs')
  4 + , join = require('path').join
  5 + , file = join(__dirname, 'fixtures','all_npm.json')
  6 + , JSONStream = require('../')
  7 + , it = require('it-is')
  8 +
  9 +var expected = JSON.parse(fs.readFileSync(file))
  10 + , parser = JSONStream.parse(['rows', /\d+/ /*, 'value'*/])
  11 + , called = 0
  12 + , ended = false
  13 + , parsed = []
  14 +
  15 +fs.createReadStream(file).pipe(parser)
  16 +
  17 +parser.on('data', function (data) {
  18 + called ++
  19 + it.has({
  20 + id: it.isString(),
  21 + value: {rev: it.isString()},
  22 + key:it.isString()
  23 + })
  24 + parsed.push(data)
  25 +})
  26 +
  27 +parser.on('end', function () {
  28 + ended = true
  29 +})
  30 +
  31 +process.on('exit', function () {
  32 + it(called).equal(expected.rows.length)
  33 + it(parsed).deepEqual(expected.rows)
  34 + console.error('PASSED')
  35 +})
29 test/test2.js
... ... @@ -0,0 +1,29 @@
  1 +
  2 +
  3 +var fs = require ('fs')
  4 + , join = require('path').join
  5 + , file = join(__dirname, '..','package.json')
  6 + , JSONStream = require('../')
  7 + , it = require('it-is')
  8 +
  9 +var expected = JSON.parse(fs.readFileSync(file))
  10 + , parser = JSONStream.parse([])
  11 + , called = 0
  12 + , ended = false
  13 + , parsed = []
  14 +
  15 +fs.createReadStream(file).pipe(parser)
  16 +
  17 +parser.on('data', function (data) {
  18 + called ++
  19 + it(data).deepEqual(expected)
  20 +})
  21 +
  22 +parser.on('end', function () {
  23 + ended = true
  24 +})
  25 +
  26 +process.on('exit', function () {
  27 + it(called).equal(1)
  28 + console.error('PASSED')
  29 +})

0 comments on commit f59e9fd

Please sign in to comment.
Something went wrong with that request. Please try again.