Permalink
Browse files

make csv gut server return a stream right away

  • Loading branch information...
1 parent 024d60e commit bf283279cde3dfda37e1a45132ce99add9d75f38 @maxogden committed Mar 1, 2012
Showing with 8,125 additions and 19 deletions.
  1. +20 −18 csv-to-json/lib.js
  2. +1 −1 csv-to-json/server.js
  3. +1 −0 shapefile-to-json/node_modules/.bin/shp2json
  4. +75 −0 shapefile-to-json/node_modules/shp2json/README.markdown
  5. +23 −0 shapefile-to-json/node_modules/shp2json/bin/cli.js
  6. BIN shapefile-to-json/node_modules/shp2json/data/shape.zip
  7. +3 −0 shapefile-to-json/node_modules/shp2json/example/shp2json.js
  8. +75 −0 shapefile-to-json/node_modules/shp2json/index.js
  9. +101 −0 shapefile-to-json/node_modules/shp2json/node_modules/findit/README.markdown
  10. +3 −0 shapefile-to-json/node_modules/shp2json/node_modules/findit/examples/callback.js
  11. +9 −0 shapefile-to-json/node_modules/shp2json/node_modules/findit/examples/emitter.js
  12. +2 −0 shapefile-to-json/node_modules/shp2json/node_modules/findit/examples/sync.js
  13. +142 −0 shapefile-to-json/node_modules/shp2json/node_modules/findit/index.js
  14. +34 −0 shapefile-to-json/node_modules/shp2json/node_modules/findit/package.json
  15. +31 −0 shapefile-to-json/node_modules/shp2json/node_modules/findit/test/cb.js
  16. +79 −0 shapefile-to-json/node_modules/shp2json/node_modules/findit/test/foo.js
  17. 0 shapefile-to-json/node_modules/shp2json/node_modules/findit/test/foo/a/b/c/w
  18. 0 shapefile-to-json/node_modules/shp2json/node_modules/findit/test/foo/a/b/z
  19. 0 shapefile-to-json/node_modules/shp2json/node_modules/findit/test/foo/a/y
  20. 0 shapefile-to-json/node_modules/shp2json/node_modules/findit/test/foo/x
  21. +29 −0 shapefile-to-json/node_modules/shp2json/node_modules/findit/test/module.js
  22. +73 −0 shapefile-to-json/node_modules/shp2json/node_modules/findit/test/symlinks.js
  23. +1 −0 shapefile-to-json/node_modules/shp2json/node_modules/findit/test/symlinks/dir1/dangling-symlink
  24. 0 shapefile-to-json/node_modules/shp2json/node_modules/findit/test/symlinks/dir1/file1
  25. 0 shapefile-to-json/node_modules/shp2json/node_modules/findit/test/symlinks/dir2/file2
  26. 0 shapefile-to-json/node_modules/shp2json/node_modules/findit/test/symlinks/file
  27. +101 −0 shapefile-to-json/node_modules/shp2json/node_modules/morestreams/main.js
  28. +17 −0 shapefile-to-json/node_modules/shp2json/node_modules/morestreams/package.json
  29. +24 −0 shapefile-to-json/node_modules/shp2json/node_modules/morestreams/test-upgrade.js
  30. +43 −0 shapefile-to-json/node_modules/shp2json/node_modules/morestreams/tests.js
  31. +1 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/.npmignore
  32. +442 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/README.markdown
  33. +18 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/examples/join.js
  34. +12 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/examples/parseq.coffee
  35. +19 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/examples/parseq.js
  36. +16 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/examples/stat_all.coffee
  37. +17 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/examples/stat_all.js
  38. +520 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/index.js
  39. +1 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/node_modules/chainsaw/.npmignore
  40. +140 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/node_modules/chainsaw/README.markdown
  41. +25 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/node_modules/chainsaw/examples/add_do.js
  42. +67 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/node_modules/chainsaw/examples/prompt.js
  43. +108 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/node_modules/chainsaw/index.js
  44. +1 −0 ...son/node_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/.npmignore
  45. +24 −0 ...o-json/node_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/LICENSE
  46. +247 −0 ...ode_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/README.markdown
  47. +16 −0 ...de_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/examples/json.js
  48. +15 −0 ..._modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/examples/leaves.js
  49. +8 −0 ...odules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/examples/negative.js
  50. +38 −0 ...dules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/examples/stringify.js
  51. +322 −0 ...-json/node_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/index.js
  52. +18 −0 ...n/node_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/package.json
  53. +114 −0 ...de_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/test/circular.js
  54. +35 −0 ...n/node_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/test/date.js
  55. +219 −0 .../node_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/test/equal.js
  56. +17 −0 ...de_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/test/instance.js
  57. +42 −0 ...e_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/test/interface.js
  58. +47 −0 ...n/node_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/test/json.js
  59. +21 −0 ...node_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/test/leaves.js
  60. +203 −0 ..._modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/test/mutability.js
  61. +20 −0 ...de_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/test/negative.js
  62. +15 −0 ...on/node_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/test/obj.js
  63. +41 −0 ...n/node_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/test/stop.js
  64. +36 −0 ...e_modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/test/stringify.js
  65. +54 −0 ..._modules/shp2json/node_modules/seq/node_modules/chainsaw/node_modules/traverse/test/super_deep.js
  66. +23 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/node_modules/chainsaw/package.json
  67. +418 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/node_modules/chainsaw/test/chainsaw.js
  68. +191 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/node_modules/hashish/README.markdown
  69. +9 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/node_modules/hashish/examples/chain.js
  70. +7 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/node_modules/hashish/examples/map.js
  71. +253 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/node_modules/hashish/index.js
  72. +1 −0 ...json/node_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/.npmignore
  73. +24 −0 ...to-json/node_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/LICENSE
  74. +256 −0 ...node_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/README.markdown
  75. +16 −0 ...ode_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/examples/json.js
  76. +15 −0 ...e_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/examples/leaves.js
  77. +8 −0 ...modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/examples/negative.js
  78. +10 −0 ...de_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/examples/scrub.js
  79. +38 −0 ...odules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/examples/stringify.js
  80. +279 −0 ...o-json/node_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/index.js
  81. +18 −0 ...on/node_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/package.json
  82. +115 −0 ...ode_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/circular.js
  83. +35 −0 ...on/node_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/date.js
  84. +220 −0 ...n/node_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/equal.js
  85. +13 −0 ...son/node_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/has.js
  86. +17 −0 ...ode_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/instance.js
  87. +42 −0 ...de_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/interface.js
  88. +47 −0 ...on/node_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/json.js
  89. +29 −0 ...on/node_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/keys.js
  90. +21 −0 .../node_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/leaves.js
  91. +92 −0 ...dules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/lib/deep_equal.js
  92. +252 −0 ...e_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/mutability.js
  93. +20 −0 ...ode_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/negative.js
  94. +15 −0 ...son/node_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/obj.js
  95. +35 −0 ...ode_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/siblings.js
  96. +41 −0 ...on/node_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/stop.js
  97. +36 −0 ...de_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/stringify.js
  98. +34 −0 ...node_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/subexpr.js
  99. +55 −0 ...e_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/test/super_deep.js
  100. +22 −0 ...e_modules/shp2json/node_modules/seq/node_modules/hashish/node_modules/traverse/testling/leaves.js
  101. +33 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/node_modules/hashish/package.json
  102. +250 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/node_modules/hashish/test/hash.js
  103. +69 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/node_modules/hashish/test/property.js
  104. +33 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/package.json
  105. +35 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/test/readdir.js
  106. +946 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/test/seq.js
  107. +149 −0 shapefile-to-json/node_modules/shp2json/node_modules/seq/test/seq_.js
  108. +49 −0 shapefile-to-json/node_modules/shp2json/package.json
  109. +24 −0 shapefile-to-json/node_modules/shp2json/test/tojson.js
  110. +4 −0 shapefile-to-json/server.js
View
@@ -4,40 +4,42 @@ var vm = require('vm'),
csv = require('csv'),
request = require('request').defaults({json: true}),
http = require('http'),
- crypto = require('crypto')
+ crypto = require('crypto'),
+ stream = require('stream')
;
module.exports = http.createServer(function (req, res) {
- var headers, dataset, rows = [];
+ var headers, dataset;
+
+ var jsonStream = new stream.Stream();
+ jsonStream.writeable = true;
+ jsonStream.readable = true;
+
+ jsonStream.pipe(res)
+
+ var separator = ''
csv()
.fromStream(req)
.on('data',function(data, index) {
if (!headers) {
+ jsonStream.emit("data", "{\"docs\":[")
headers = data;
return;
}
var row = {}
_(_.zip(headers, data)).each(function(tuple) {
row[_.first(tuple)] = _.last(tuple)
})
- rows.push(row);
+ jsonStream.emit('data', separator + JSON.stringify(row))
+ separator = ','
})
.on('end', function(count) {
- res.statusCode = 202;
- res.end();
- headers = _.map(headers, function(head) { return {name: head} })
- request({uri: req.headers['x-callback'], method: "POST", body: {headers: headers, rows: rows}}, function(e,r,b) {
- if (e) console.log('upload error on ' + dataset + ': ' + e);
- });
+ jsonStream.emit('data', "]}")
+ jsonStream.emit('end')
})
- .on('error',function(error){
- console.log("csv error!", error.message);
+ .on('error',function(error) {
+ jsonStream.emit('error', error)
+ console.log('csv error', error.message);
});
-})
-
-// sandbox = {
-// animal: 'cat',
-// count: 2
-// };
-// vm.runInNewContext('count += 1; name = "kitty"', sandbox, 'myfile.vm');
+})
View
@@ -1 +1 @@
-require('./lib').listen(80)
+require('./lib').listen(process.argv[2] || 80)

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.
Binary file not shown.

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.

Some generated files are not rendered by default. Learn more.

Oops, something went wrong.
Oops, something went wrong.

0 comments on commit bf28327

Please sign in to comment.