fix: don't retry to save log when db error (#1381)

Only keep the last 50kb log on db.
cnpm · Sep 8, 2018 · 3c20267 · 3c20267
1 parent 26d7147
commit 3c20267
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 18 deletions.
diff --git a/controllers/sync_module_worker.js b/controllers/sync_module_worker.js
@@ -83,6 +83,7 @@ SyncModuleWorker.prototype.finish = function () {
   this.removeAllListeners();
 };
 
+var MAX_LEN = 10 * 1024;
 // log(format, arg1, arg2, ...)
 SyncModuleWorker.prototype.log = function () {
   var str = '[' + utility.YYYYMMDDHHmmss() + '] ' + util.format.apply(util, arguments);
@@ -93,50 +94,62 @@ SyncModuleWorker.prototype.log = function () {
       this._log += '\n';
     }
     this._log += str;
-    this._saveLog();
+    if (this._log.length >= MAX_LEN) {
+      this._saveLog();
+    }
   }
 };
 
-SyncModuleWorker.prototype._saveLog = function () {
+// isEnd will be set to true when sync process success or error
+SyncModuleWorker.prototype._saveLog = function (isEnd) {
   var that = this;
-  if (that._loging) {
+  if (that._loging && !isEnd) {
     return;
   }
   that._loging = true;
   var logstr = that._log;
   that._log = '';
+
+  if (!logstr) {
+    that._loging = false;
+    return;
+  }
+
   co(function* () {
     yield logService.append(that._logId, logstr);
   }).then(function () {
     that._loging = false;
-    if (that._log) {
+    if (isEnd && that._log) {
       that._saveLog();
     }
   }).catch(function (err) {
-    logger.error(err);
     that._loging = false;
-    if (that._log) {
-      that._saveLog();
+    logger.error(err);
+    // ignore the unsave log
+    if (isEnd) {
+      logger.error('[SyncModuleWorker] skip to save %s logstr: %s', that._logId, logstr);
     }
   });
 };
 
 SyncModuleWorker.prototype.start = function () {
-  this.log('user: %s, sync %s worker start, %d concurrency, nodeps: %s, publish: %s, syncUpstreamFirst: %s',
-    this.username, this.names[0], this.concurrency, this.noDep, this._publish, this.syncUpstreamFirst);
   var that = this;
+  that.log('user: %s, sync %s worker start, %d concurrency, nodeps: %s, publish: %s, syncUpstreamFirst: %s',
+    that.username, that.names[0], that.concurrency, that.noDep, that._publish, that.syncUpstreamFirst);
   co(function* () {
     // sync upstream
     if (that.syncUpstreamFirst) {
       try {
         yield that.syncUpstream(that.startName);
+        that._saveLog();
       } catch (err) {
         logger.error(err);
       }
     }
 
     if (that.type === 'user') {
       yield that.syncUser();
+      that._saveLog(true);
       return;
     }
 
@@ -145,8 +158,10 @@ SyncModuleWorker.prototype.start = function () {
       arr.push(that.next(i));
     }
     yield arr;
+    that._saveLog(true);
   }).catch(function (err) {
     logger.error(err);
+    that._saveLog(true);
   });
 };
 

diff --git a/services/module_log.js b/services/module_log.js
@@ -12,8 +12,8 @@ exports.create = function* (data) {
   return yield row.save();
 };
 
-var ONE_MB = 1024 * 1024;
-
+// 50kb
+var MAX_LEN = 50 * 1024;
 exports.append = function* (id, log) {
   if (!log) {
     return null;
@@ -29,8 +29,9 @@ exports.append = function* (id, log) {
   } else {
     row.log = log;
   }
-  if (row.log.length >= ONE_MB) {
-    row.log = '...\n' + row.log.substring(ONE_MB / 2);
+  if (row.log.length > MAX_LEN) {
+    // only keep the last 50kb log string
+    row.log = '...\n' + row.log.substring(row.log.length - MAX_LEN);
   }
   return yield row.save(['log']);
 };

diff --git a/test/services/module_log.test.js b/test/services/module_log.test.js
@@ -2,8 +2,8 @@
 
 var ModuleLog = require('../../services/module_log');
 
-describe('test/services/module_log.test.js', function () {
-  describe('create(), append()', function () {
+describe('test/services/module_log.test.js', () => {
+  describe('create(), append()', () => {
     it('should create a log row', function* () {
       var log = yield ModuleLog.create({name: 'utility', username: 'fengmk2'});
       log.id.should.be.a.Number;
@@ -25,14 +25,24 @@ describe('test/services/module_log.test.js', function () {
       log.log.should.equal('a new line\nsecond line');
     });
 
-    it('should slice log when size bigger than 1MB', function* () {
+    it('should slice log when size bigger than 50kb', function* () {
       var log = yield ModuleLog.create({name: 'module_log-append', username: 'fengmk2'});
       var logid = log.id;
 
-      var biglog = Buffer.alloc(1024 * 1024).fill(71).toString();
+      var biglog = Buffer.alloc(50 * 1024 + 1).fill(71).toString();
       log = yield ModuleLog.append(logid, biglog);
       log.log.substring(0, 4).should.equal('...\n');
-      log.log.length.should.equal(1024 * 1024 / 2 + 4);
+      log.log.length.should.equal(50 * 1024 + 4);
+    });
+
+    it('should slice log when size equal 50kb', function* () {
+      var log = yield ModuleLog.create({name: 'module_log-append', username: 'fengmk2'});
+      var logid = log.id;
+
+      var biglog = Buffer.alloc(50 * 1024).fill(71).toString();
+      log = yield ModuleLog.append(logid, biglog);
+      log.log.substring(0, 4).should.equal('GGGG');
+      log.log.length.should.equal(50 * 1024);
     });
   });
 });