transform progress in ui

max-mapper · Feb 22, 2012 · 44ba81d · 44ba81d · rufuspollock · Feb 22, 2012
1 parent aa64b26
commit 44ba81d
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 15 deletions.
diff --git a/attachments/script/costco.js b/attachments/script/costco.js
@@ -74,12 +74,13 @@ var costco = function() {
       var _id = data.uuids[0]
       transformDoc["_id"] = _id
       app.io.emit('save', transformDoc)
-      util.notify("Transforming documents. This could take a while... (you can close and come back later)", {persist: true, loader: true})
-      app.io.on(_id, function (err, doc) {
+      util.notify("Transforming documents...", {persist: true, loader: true})
+      app.io.on(_id, function (err, data) {
         if (err && callback) callback(err)
+        if (data.progress) return util.notify("Transforming documents... " + data.progress + "%", {persist: true, loader: true})
         util.notify("Documents updated successfully!")
         recline.initializeTable(app.offset)
-        if (callback) callback(false, doc)
+        if (callback) callback(false, data)
       })
     })
 

diff --git a/service/transformer.js b/service/transformer.js
@@ -14,24 +14,40 @@ module.exports = function (t) {
 
   follow({db: db, include_docs: true, filter: "datacouch/by_value", query_params: {k: "type", v: "pendingTransformation"}}, function(error, change) {
     if (error) return console.error(error)
-    txn({"uri": db + '/' + change.id}, transform, function(error, newData) {
-      if (error) return console.error('transformation error on ' + change.id, err)
+    txn({"uri": db + '/' + change.id, "timeout": 120000}, transform, function(err, newData) {
+      if (err) return console.error('transformation error on ' + change.id, err)
       t.sockets.emit(newData._id, false, newData)
       return console.log('transformation success on ' + newData.dataset)
     })
   })
+
+  function getDBInfo(db, callback) {
+    request({url: t.couchurl + db}, callback)
+  }
 
   function transform(doc, txncb) {
-    var down = request({url: t.couchurl + "/" + doc.dataset + '/_all_docs?include_docs=true'}),
-      up = request({url: t.couchurl + '/' + doc.dataset + '/_bulk_docs', method: "POST"}),
-      tr = transfuse(['rows', /./, 'doc'], doc.transform, JSONStream.stringify("{\"docs\":[\n", "\n,\n", "\n]}\n"));
-    down.pipe(tr)
-    tr.pipe(up)
-    up.on('error', txncb)
-    up.on('end', function() {
-      doc.type = "transformation"
-      doc.finishedAt = new Date()
-      txncb()
+    getDBInfo(doc.dataset, function(err, resp, dbInfo) {
+      if (err) return txncb(err)
+      var down = request({url: t.couchurl + doc.dataset + '/_all_docs?include_docs=true'}),
+        up = request({url: t.couchurl + doc.dataset + '/_bulk_docs', method: "POST"}),
+        tr = transfuse(['rows', /./, 'doc'], doc.transform, JSONStream.stringify("{\"docs\":[\n", "\n,\n", "\n]}\n"));
+      down.pipe(tr)
+      tr.pipe(up)
+      var count = 0
+      tr.on('data', function(chunk) {
+        count++
+        var progress = (count/dbInfo.doc_count)*100
+        if ((count % 500) === 0) {
+          console.log(progress)
+          t.sockets.emit(doc._id, false, {progress: progress})
+        }
+      })
+      up.on('error', txncb)
+      up.on('end', function() {
+        doc.type = "transformation"
+        doc.finishedAt = new Date()
+        txncb()
+      })
     })
   }