Permalink
Browse files

Asynchronous transformations

  • Loading branch information...
1 parent 9703d5d commit 3e35a18185b1f8c9310022e1f06a175c2b616c27 @wdavidw committed Oct 18, 2012
Showing with 375 additions and 106 deletions.
  1. +96 −15 doc/transformer.md
  2. +1 −1 lib/Stringifier.js
  3. +118 −35 lib/Transformer.js
  4. +12 −8 lib/csv.js
  5. +1 −1 lib/state.js
  6. +14 −10 src/csv.coffee
  7. +1 −1 src/state.coffee
  8. +1 −1 src/stringifier.coffee
  9. +94 −28 src/transformer.coffee
  10. +36 −5 test/transform.coffee
  11. +1 −1 test/write.coffee
View
@@ -2,7 +2,7 @@
language: en
layout: page
title: "Transforming data"
-date: 2012-10-09T16:24:28.047Z
+date: 2012-10-18T13:01:04.697Z
comments: false
sharing: false
footer: false
@@ -11,44 +11,125 @@ github: https://github.com/wdavidw/node-csv-parser
---
-The contract is quite simple, you receive an array of fields for
-each record and return the transformed record. The return value
-may be an array, an associative array, a string or null. If null,
-the record will simply be skipped.
+Transformation may occur synchronously or asynchronously dependending
+on the provided transform callback and its declared arguments length.
+
+Callback are called for each line and its arguments are :
+* data CSV record
+* index Incremented counter
+* callback Callback function to be called in asynchronous mode
Unless you specify the `columns` read option, `data` are provided
as arrays, otherwise they are objects with keys matching columns
names.
-When the returned value is an array, the fields are merged in
-order. When the returned value is an object, it will search for
+In synchronous mode, the contract is quite simple, you receive an array
+of fields for each record and return the transformed record.
+
+In asynchronous mode, it is your responsibility to call the callback
+provided as the third argument. It must be called with two arguments,
+the first one is an error if any, the second is the transformed record.
+
+Transformed records may be an array, an associative array, a
+string or null. If null, the record will simply be skipped. When the
+returned value is an array, the fields are merged in order.
+When the returned value is an object, it will search for
the `columns` property in the write or in the read options and
smartly order the values. If no `columns` options are found,
it will merge the values in their order of appearance. When the
returned value is a string, it is directly sent to the destination
source and it is your responsibility to delimit, quote, escape
or define line breaks.
-Example of transform returning a string:
+Transform callback run synchronously:
```javascript
csv()
.from('82,Preisner,Zbigniew\n94,Gainsbourg,Serge')
.to(console.log)
-.transform(function(data,index){
- return (index>0 ? ',' : '') + data[0] + ":" + data[2] + ' ' + data[1];
+.transform(function(data, index){
+ return data.reverse()
+});
+// Executing `node samples/transform.js`, print:
+// 94,Gainsbourg,Serge\n82,Preisner,Zbigniew
+
+```
+
+Transform callback run asynchronously:
+
+```javascript
+
+csv()
+.from('82,Preisner,Zbigniew\n94,Gainsbourg,Serge')
+.to(console.log)
+.transform(function(data, index, callback){
+ process.nextTick(function(){
+ callback(null, data.reverse());
+ });
});
+// Executing `node samples/transform.js`, print:
+// 94,Gainsbourg,Serge\n82,Preisner,Zbigniew
+
+```
+
+Transform callback returning a string:
+
+```javascript
+csv()
+.from('82,Preisner,Zbigniew\n94,Gainsbourg,Serge')
+.to(console.log)
+.transform(function(data, index){
+ return (index>0 ? ',' : '') + data[0] + ":" + data[2] + ' ' + data[1];
+});
// Executing `node samples/transform.js`, print:
// 82:Zbigniew Preisner,94:Serge Gainsbourg
```
+Transformer.prototype.transform = (line) ->
+ csv = @csv
+ columns = csv.options.from.columns
+ if columns
-<a name="transformer"></a>
-`transformer(csv).transform(line)`
------------------
+```javascript
+# Extract column names from the first line
+if csv.state.count is 0 and columns is true
+ csv.options.from.columns = columns = line
+ return
+# Line stored as an object in which keys are column names
+lineAsObject = {}
+for column, i in columns
+ lineAsObject[column] = line[i] or null
+line = lineAsObject
+nish = ( (line) ->
+if csv.state.count is 1 and csv.options.to.header is true
+ csv.stringifier.write csv.options.to.columns or columns
+csv.stringifier.write line
+# csv.state.count++
+@emit 'end', csv.state.count if csv.state.transforming is 0 and @closed is true
+bind @
+v.state.count++
+ @callback
+sync = @callback.length isnt 3
+csv.state.transforming++
+done = (err, line) ->
+ return csv.error err if err
+ isObject = typeof line is 'object' and not Array.isArray line
+ if csv.options.to.newColumns and not csv.options.to.columns and isObject
+ Object.keys(line)
+ .filter( (column) -> columns.indexOf(column) is -1 )
+ .forEach( (column) -> columns.push(column) )
+ csv.state.transforming--
+ finish line
+if sync
+ try done null, @callback line, csv.state.count - 1
+ catch err then return done err
+else
+ try @callback line, csv.state.count - 1, (err, line) ->
+ done err, line
+se
+finish line
+```
-Internal function. Call a callback to transform a line. Called from the `parse` function on each
-line. It is responsible for transforming the data and finally calling `write`.
View
Oops, something went wrong.
View
Oops, something went wrong.
Oops, something went wrong.

0 comments on commit 3e35a18

Please sign in to comment.