added filter serialize

index.js

@@ -350,49 +350,54 @@ Adapter.prototype.parse = function ( parse ) {
  *  Serialize content of files into one dataset
  *
  *  - `options`
- *    - `merge` {function} Function that takes to data arrays and merge tem into
- *    - `path_full` {string} Filename of file with full snapshot
+ *    - `name_full` {string} Filename of file with full snapshot
  *    - `overlap` {integer} Overlap in minutes between full and first incremental
  *  one
  */
-Adapter.prototype.serialize = function(options) {
+Adapter.prototype.filter_serialize = function(options) {
 
     if (!options){
 
-        debug( 'serialize skip');
+        debug( 'filter serialize skip');
         return this
 
     }
 
     let name_full = options.name_full;
-    let serialize = options.merge;
     let overlap =  options.overlap;
 
     let self=this;
 
     overlap = overlap ? overlap : 0;
 
-    this._promise = this._promise.then( () => {
+    this._promise = this._promise.then( files => {
 
-        debug( 'serialize');
+        debug( 'serialize filter');
 
-        let full = [];
-        let incremental = [];
-        self.data.forEach( d => {
+        let full;
+        let filtered = [];
+        files.forEach( d => {
 
-            if ( d.file == name_full ) {
+
+            if ( d.name == name_full ) {
 
                 full = d;
-                return
             }
 
-            incremental.push( d )
+            filtered.push( d )
 
         } );
 
-        debug( 'serialize incremental', incremental.length );
 
-        incremental = incremental.sort( ( a, b ) => {
+        if (!full){
+
+            let err = new Error('Missing file with snapshot. Need to set name_full options correct')
+            console.error(err)
+            throw err
+
+        }
+
+        filtered = filtered.sort( ( a, b ) => {
 
             a = a.last_modified;
             b = b.last_modified;
@@ -414,10 +419,17 @@ Adapter.prototype.serialize = function(options) {
         } );
 
         let date_full=moment(full.last_modified).subtract(overlap, 'minutes')
-        incremental = incremental.reduce((tot,val)=>{
+
+
+        debug( filtered );
+        debug( date_full );
+
+        filtered = filtered.reduce((tot,val)=>{
 
             let date_inc = moment(val.last_modified);
 
+            debug(date_inc);
+
             if (date_full<=date_inc){
 
                 tot.push(val)
@@ -428,9 +440,90 @@ Adapter.prototype.serialize = function(options) {
 
         }, []);
 
+
+        debug(filtered)
+
+
+        debug( 'serialize filter', filtered.length );
+
+        self.files_filtered=filtered;
+
+        return self.files_filtered;
+
+    } )
+
+    return this
+
+}
+
+
+/**
+ *  Serialize content of files into one dataset
+ *
+ *  - `options`
+ *    - `merge` {function} Function that takes to data arrays and merge tem into
+ *    - `name_full` {string} Filename of file with full snapshot
+ *  one
+ */
+Adapter.prototype.serialize = function(options) {
+
+    if (!options){
+
+        debug( 'serialize skip');
+        return this
+
+    }
+
+    let name_full = options.name_full;
+    let serialize = options.merge;
+
+    let self=this;
+
+
+    this._promise = this._promise.then( () => {
+
+        debug( 'serialize');
+
+        let full = [];
+        let incremental = [];
+        self.data.forEach( d => {
+
+            if ( d.file == name_full ) {
+
+                full = d;
+                return
+            }
+
+            incremental.push( d )
+
+        } );
+
+        debug( 'serialize incremental', incremental.length );
+
+        incremental = incremental.sort( ( a, b ) => {
+
+            a = a.last_modified;
+            b = b.last_modified;
+
+            if ( a < b ) {
+
+                return 1
+
+            } else if ( a > b ) {
+
+                return -1
+
+            } else {
+
+                return 0
+
+            }
+
+        } );
+
         incremental.forEach( inc => {
 
-            full.json = serialize( full.json, inc.json )
+            full.json = serialize( full.json, inc.json );
 
             if (! full.files_incremental){
 
@@ -450,7 +543,7 @@ Adapter.prototype.serialize = function(options) {
 
         return full;
 
-    } )
+    } );
 
     return this
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "text-file-import",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "Extends jsftp used fetch multiple file content from a remote directory",
   "main": "index.js",
   "scripts": {

test/index.js

@@ -265,6 +265,7 @@ lab.experiment( 'text file import', function () {
         let io = IO( options )
             .list( './test/full' )
             .list( './test/incremental' )
+            .filter_serialize({name_full:'full.json'})
             .then( results => {
 
                 debug(results);