Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

add pipable, and improve readme

  • Loading branch information...
commit b0333ed89207eae7dd1516bf0f36b6c840ecb8c7 1 parent 42b5c5c
Dominic Tarr authored September 17, 2011
25  examples/pretty.js
... ...
@@ -0,0 +1,25 @@
  1
+
  2
+var inspect = require('util').inspect
  3
+
  4
+if(!module.parent) {
  5
+  var es = require('..')              //load event-stream
  6
+  es.pipe(                            //pipe joins streams together
  7
+    process.openStdin(),              //open stdin
  8
+    es.split(),                       //split stream to break on newlines
  9
+    es.map(function (data, callback) {//turn this async function into a stream
  10
+      var j 
  11
+      try {
  12
+        j = JSON.parse(data)          //try to parse input into json
  13
+      } catch (err) {
  14
+        return callback(null, data)   //if it fails just pass it anyway
  15
+      }
  16
+      callback(null, inspect(j))      //render it nicely
  17
+    }),
  18
+    process.stdout                    // pipe it to stdout !
  19
+    )
  20
+  }
  21
+  
  22
+// run this
  23
+// 
  24
+// curl -sS registry.npmjs.org/event-stream | node pretty.js 
  25
+//
98  index.js
@@ -5,12 +5,12 @@
5 5
 
6 6
 
7 7
 var Stream = require('stream').Stream
8  
-
  8
+  , es = exports
9 9
 // writable stream, collects all events into an array 
10 10
 // and calls back when 'end' occurs
11 11
 // mainly I'm using this to test the other functions
12 12
 
13  
-exports.writeArray = function (done) {
  13
+es.writeArray = function (done) {
14 14
   if ('function' !== typeof done)
15 15
     throw new Error('function writeArray (done): done must be function')
16 16
 
@@ -30,7 +30,7 @@ exports.writeArray = function (done) {
30 30
 //return a Stream that reads the properties of an object
31 31
 //respecting pause() and resume()
32 32
 
33  
-exports.readArray = function (array) {
  33
+es.readArray = function (array) {
34 34
   var stream = new Stream()
35 35
     , i = 0
36 36
     , paused = false
@@ -61,7 +61,7 @@ exports.readArray = function (array) {
61 61
 //emitting each response as data
62 62
 //unless it's an empty callback
63 63
 
64  
-exports.map = function (mapper) {
  64
+es.map = function (mapper) {
65 65
   var stream = new Stream()
66 66
     , inputs = 0
67 67
     , outputs = 0
@@ -126,7 +126,7 @@ exports.map = function (mapper) {
126 126
 // combine multiple streams together so that they act as a single stream
127 127
 //
128 128
 
129  
-exports.pipe = function () {
  129
+es.pipe = function () {
130 130
 
131 131
   var streams = [].slice.call(arguments)
132 132
     , first = streams[0]
@@ -176,7 +176,7 @@ exports.pipe = function () {
176 176
   return thepipe
177 177
 }
178 178
 
179  
-exports.split = function (matcher) {
  179
+es.split = function (matcher) {
180 180
   var stream = new Stream()
181 181
     , soFar = ''  
182 182
   
@@ -195,7 +195,6 @@ exports.split = function (matcher) {
195 195
         var n = soFar;
196 196
         soFar = '' 
197 197
         this.emit('data', n)
198  
-        console.log('data',n)
199 198
       }
200 199
     i++
201 200
     }
@@ -208,3 +207,88 @@ exports.split = function (matcher) {
208 207
 
209 208
   return stream
210 209
 }
  210
+
  211
+//
  212
+// helper to make your module into a unix pipe
  213
+// simply add 
  214
+// 
  215
+// if(!module.parent)
  216
+//  require('event-stream').pipable(asyncFunctionOrStreams)
  217
+// 
  218
+// asyncFunctionOrStreams may be one or more Streams or if it is a function, 
  219
+// it will be automatically wrapped in es.map
  220
+//
  221
+// then pipe stuff into from the command line!
  222
+// 
  223
+// curl registry.npmjs.org/event-stream | node hello-pipeable.js | grep whatever
  224
+//
  225
+// etc!
  226
+//
  227
+// also, start pipeable running as a server!
  228
+//
  229
+// > node hello-pipeable.js --port 44444
  230
+// 
  231
+
  232
+var setup = function (args) {
  233
+  return args.map(function (f) {
  234
+    console.log(f)
  235
+    var x = f()
  236
+      if('function' === typeof x)
  237
+        return es.map(x)
  238
+      return x
  239
+    })
  240
+}
  241
+
  242
+es.pipeable = function () {
  243
+  var opts = require('optimist').argv
  244
+  var args = [].slice.call(arguments)
  245
+  
  246
+  if(opts.h || opts.help) {
  247
+    var name = process.argv[1]
  248
+    console.error([
  249
+      'Usage:',
  250
+      '',
  251
+      'node ' + name + ' [options]',
  252
+      '  --port PORT        turn this stream into a server',
  253
+      '  --host HOST        host of server (localhost is default)',
  254
+      '  --protocol         protocol http|net will require(protocol).createServer(...',
  255
+      '  --help             display this message',
  256
+      '',
  257
+      ' if --port is not set, will stream input from stdin',
  258
+      '',
  259
+      'also, pipe from or to files:',
  260
+      '',
  261
+      ' node '+name+ ' < file    #pipe from file into this stream',
  262
+      ' node '+name+ ' < infile > outfile    #pipe from file into this stream',     
  263
+      '',
  264
+    ].join('\n'))
  265
+  
  266
+  } else if (!opts.port) {
  267
+    var streams = setup(args)
  268
+    streams.unshift(es.split())
  269
+    streams.unshift(process.openStdin())
  270
+    streams.push(process.stdout)
  271
+  
  272
+    return es.pipe.apply(null, streams)
  273
+
  274
+  } else {
  275
+  
  276
+    opts.host = opts.host || 'localhost'
  277
+    opts.protocol = opts.protocol || 'http'
  278
+    
  279
+    var protocol = require(opts.protocol)
  280
+        
  281
+    var server = protocol.createServer(function (instream, outstream) {  
  282
+      var streams = setup(args)
  283
+      streams.unshift(es.split())
  284
+      streams.unshift(instream)
  285
+      streams.push(outstream || instream)
  286
+      console.error(streams)
  287
+      es.pipe.apply(null, streams)
  288
+    })
  289
+    
  290
+    server.listen(opts.port, opts.host)
  291
+
  292
+    console.error(process.argv[1] +' is listening for "' + opts.protocol + '" on ' + opts.host + ':' + opts.port)  
  293
+  }
  294
+}
0  out
No changes.
103  readme.markdown
Source Rendered
... ...
@@ -1,75 +1,86 @@
1 1
 # EventStreams
2 2
 
3  
-EventEmitters in node are a brilliant idea that unfortunatly are under utilized by the node community.
4  
-Yes, that is right. _under utilized_. there are many more things that EventEmitters could be used for, especially, 
5  
-the `Stream`s, a subclass of EventEmitters.
  3
+Streams are node's best and sadly misunderstood idea,  
  4
+this is a toolkit to make creating and working with streams <em>easy</em>.
6 5
 
7  
-A stream of events is a bit like an array, but an array layed out in time, rather than in memory.
  6
+`Stream` is a subclass of `EventEmitter`, it adds one very useful function:
8 7
 
9  
-You can apply a map function to an array and create a new array, you could apply a similar 
10  
-map function to a stream of events to create a new stream. `map` functions, but also `fitler`, `reduce`
11  
-and other functional programming idioms!
  8
+``` js
  9
+  readibleStream.pipe(writableStream)
12 10
 
13  
-event streams are great because they have a naturally scalable API. 
14  
-if the events in a stream can be stringified ane parsed then it will be relatively simple to split heavy 
15  
-parts of a stream into seperate processes, and to incorperate middlewares into the stream that might 
16  
-buffer or rate-limit or parallelize critical aspects of your event stream.
  11
+  //if a stream is both readable and writable it can be pipe on again
  12
+  
  13
+  readibleStream.pipe(readableWritableStream)
  14
+  readableWritableStream.pipe(writableStream)
17 15
 
18  
-Supporting this sort of programming is the purpose of this library.
  16
+  // just like on the command line!
  17
+  // readibleStream | readableWritableStream | writableStream
  18
+  //
  19
+```
19 20
 
20  
-[test are in event-stream_tests](https://github.com/dominictarr/event-stream_tests)
  21
+the `event-stream` functions are just like the array functions,  
  22
+because Streams are like Arrays, but laid out in time, rather in memory.
21 23
 
22  
-[node Stream documentation](http://nodejs.org/api/streams.html)
  24
+###for example:
23 25
 
24  
-##Examples
  26
+``` js
25 27
 
26  
-###map
  28
+//pretty.js
27 29
 
28  
-Turns an asyncronous function it into an readable/writable EventStream
29  
-it can be used to perform a transformation upon a stream before writing it to something.
  30
+if(!module.parent) {
  31
+  var es = require('..')              //load event-stream
  32
+  es.pipe(                            //pipe joins streams together
  33
+    process.openStdin(),              //open stdin
  34
+    es.split(),                       //split stream to break on newlines
  35
+    es.map(function (data, callback) {//turn this async function into a stream
  36
+      callback(null
  37
+        , inspect(JSON.parse(data)))  //render it nicely
  38
+    }),
  39
+    process.stdout                    // pipe it to stdout !
  40
+    )
  41
+  }
  42
+  
  43
+//curl -sS registry.npmjs.org/event-stream | node pretty.js
30 44
 
31  
-If error, `callback(error)` like normal. If you `callback()` (no args) the stream will not emit 
32  
-anything from that map.
  45
+```
  46
+ 
  47
+[test are in event-stream_tests](https://github.com/dominictarr/event-stream_tests)
33 48
 
34  
-`map` does not guarantee mapped output order will be the same an written input.
  49
+[node Stream documentation](http://nodejs.org/api/streams.html)
35 50
 
36  
-`map` will hold off on emitting `end` until all of it's map callbacks are complete.
  51
+##map
37 52
 
38  
-Each map MUST call the callback. it may callback with data, with an error or with no arguments, 
  53
+create a readable and writable stream from an asyncronous function.  
39 54
 
40 55
 ``` js
  56
+var es = require('event-stream')
41 57
 
42  
-  //callback mapped data
43  
-  
44  
-  callback(null, data) //may use multiple args, but first is always error
45  
-    
46  
-  //drop this peice of data
47  
-  
48  
-  callback()
49  
-  
50  
-  //if there was on error
51  
-  
52  
-  callback (error) //the event stream will emit 'error' instead of data for this step.
  58
+es.map(function (data, callback) {
  59
+  //transform data
  60
+  // ...
  61
+  callback(null, data)
  62
+})
53 63
 
54 64
 ```
55 65
 
56  
-if a callback is not called map will think that it is still being worked on.
57  
-
58  
-If the callback is called more than once, every call but the first will be ignored.
  66
+Each map MUST call the callback. it may callback with data, with an error or with no arguments, 
59 67
 
60  
-''' js
  68
+  * `callback()` drop this data.  
  69
+    this makes the map work like `filter`,  
  70
+    note:`callback(null,null)` is not the same, and will emit `null`
61 71
 
62  
-var es = require('event-stream')
  72
+  * `callback(null, newData)` turn data into newData
  73
+    
  74
+  * `callback(error)` emit an error for this item.
63 75
 
64  
-  es.map(function (data, callback) {
65  
-    //do something to data
66  
-    callback(null, data)   
67  
-  })
68  
-'''
  76
+>Note: if a callback is not called map will think that it is still being worked on,   
  77
+>every call must be answered or the stream will not know when to end.  
  78
+>
  79
+>also, if the callback is called more than once, every call but the first will be ignored.
69 80
 
70  
-###read
  81
+###readArray
71 82
 
72  
-Makes an readable `EventStream` from an `Array`.
  83
+makes a readable stream from an array.  
73 84
 
74 85
 Just emit each item as a data event, respecting `pause` and `resume`.
75 86
 

0 notes on commit b0333ed

Please sign in to comment.
Something went wrong with that request. Please try again.