From fc799304d8c6710e71364bdf1d1ed0961b9e8695 Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Sun, 9 Sep 2018 00:20:50 +0800 Subject: [PATCH] fix: should return `[done] Sync {name}` string when task finished (#1382) the cnpm client sync need the `[done] Sync {name}` string to detect when sync task finished --- controllers/sync.js | 8 ++++++- controllers/sync_module_worker.js | 38 +++++++++++++++---------------- services/module_log.js | 4 ++-- services/npm.js | 14 +++++++++++- services/package.js | 4 ++++ test/controllers/sync.test.js | 10 ++++---- test/services/module_log.test.js | 4 ++-- view/web/sync.html | 29 +++++++++++------------ 8 files changed, 64 insertions(+), 47 deletions(-) diff --git a/controllers/sync.js b/controllers/sync.js index 47f7cee96..4695d2680 100644 --- a/controllers/sync.js +++ b/controllers/sync.js @@ -63,8 +63,14 @@ exports.getSyncLog = function* (next) { } var log = row.log.trim(); + var syncDone = row.log.indexOf('[done] Sync') >= 0; if (offset > 0) { log = log.split('\n').slice(offset).join('\n'); + if (!log && syncDone) { + // append the last 1k string + // the cnpm client sync need the `[done] Sync {name}` string to detect when sync task finished + log = '... ignore long logs ...\n' + row.log.substring(row.log.length - 1024); + } } - this.body = {ok: true, log: log}; + this.body = { ok: true, syncDone: syncDone, log: log }; }; diff --git a/controllers/sync_module_worker.js b/controllers/sync_module_worker.js index 677c0f1ae..961bdbb83 100644 --- a/controllers/sync_module_worker.js +++ b/controllers/sync_module_worker.js @@ -37,6 +37,7 @@ function SyncModuleWorker(options) { this._logId = options.logId; this._log = ''; this._loging = false; + this._isEnd = false; if (!Array.isArray(options.name)) { options.name = [options.name]; } @@ -78,6 +79,8 @@ SyncModuleWorker.prototype.finish = function () { this.type, this.successes.length, this.fails.length, this.successes.join(', '), this.fails.join(', ')); + this._saveLog(); + this._isEnd = true; this.emit('end'); // make sure all event listeners release this.removeAllListeners(); @@ -100,10 +103,9 @@ SyncModuleWorker.prototype.log = function () { } }; -// isEnd will be set to true when sync process success or error -SyncModuleWorker.prototype._saveLog = function (isEnd) { +SyncModuleWorker.prototype._saveLog = function () { var that = this; - if (that._loging && !isEnd) { + if (that._loging) { return; } that._loging = true; @@ -119,14 +121,14 @@ SyncModuleWorker.prototype._saveLog = function (isEnd) { yield logService.append(that._logId, logstr); }).then(function () { that._loging = false; - if (isEnd && that._log) { + if (that._log) { that._saveLog(); } }).catch(function (err) { that._loging = false; logger.error(err); // ignore the unsave log - if (isEnd) { + if (this._isEnd) { logger.error('[SyncModuleWorker] skip to save %s logstr: %s', that._logId, logstr); } }); @@ -149,7 +151,8 @@ SyncModuleWorker.prototype.start = function () { if (that.type === 'user') { yield that.syncUser(); - that._saveLog(true); + that._saveLog(); + that._isEnd = true; return; } @@ -158,10 +161,10 @@ SyncModuleWorker.prototype.start = function () { arr.push(that.next(i)); } yield arr; - that._saveLog(true); + that._saveLog(); }).catch(function (err) { logger.error(err); - that._saveLog(true); + that._saveLog(); }); }; @@ -253,17 +256,12 @@ SyncModuleWorker.prototype.syncUpstream = function* (name) { } var data = rs.data; - var syncDone = false; - if (data.log && data.log.indexOf('[done] Sync') >= 0) { - syncDone = true; - data.log = data.log.replace('[done] Sync', '[Upstream done] Sync'); - } - if (data.log) { + data.log = data.log.replace('[done] Sync', '[Upstream done] Sync'); this.log(data.log); } - if (syncDone) { + if (data.syncDone) { break; } @@ -297,17 +295,17 @@ SyncModuleWorker.prototype.syncUser = function* () { }; SyncModuleWorker.prototype.next = function* (concurrencyId) { + var name = this.names.shift(); + if (!name) { + return setImmediate(this.finish.bind(this)); + } + if (config.syncModel === 'none') { this.log('[c#%d] [%s] syncModel is none, ignore', concurrencyId, name); return this.finish(); } - var name = this.names.shift(); - if (!name) { - return setImmediate(this.finish.bind(this)); - } - // try to sync from official replicate when source npm registry is not cnpmjs.org const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : config.officialNpmReplicate; diff --git a/services/module_log.js b/services/module_log.js index 2feff86c4..d770ed4ff 100644 --- a/services/module_log.js +++ b/services/module_log.js @@ -30,8 +30,8 @@ exports.append = function* (id, log) { row.log = log; } if (row.log.length > MAX_LEN) { - // only keep the last 50kb log string - row.log = '...\n' + row.log.substring(row.log.length - MAX_LEN); + // only keep the fisrt 1kb and the last 50kb log string + row.log = row.log.substring(0, 1024) + '\n... ignore long logs ...\n' + row.log.substring(row.log.length - MAX_LEN); } return yield row.save(['log']); }; diff --git a/services/npm.js b/services/npm.js index afa0e104e..40aeded0f 100644 --- a/services/npm.js +++ b/services/npm.js @@ -153,11 +153,23 @@ exports.getAllToday = function* (timeout) { }; exports.getShort = function* (timeout) { + const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : 'https://r.cnpmjs.org'; var r = yield request('/-/short', { timeout: timeout || 300000, // registry.npmjs.org/-/short is 404 now therefore have a fallback - registry: config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : 'http://r.cnpmjs.org', + registry: registry, }); + if (r.status !== 200) { + const data = r.data; + if (data && data.code && data.message) { + // { code: 'MethodNotAllowedError', message: 'GET is not allowed' } + const url = registry + '/-/short'; + const err = new Error(data.message + ', url: ' + url); + err.name = data.code; + err.url = url; + throw err; + } + } return r.data; }; diff --git a/services/package.js b/services/package.js index f8039ebea..4d196ef6d 100644 --- a/services/package.js +++ b/services/package.js @@ -324,6 +324,10 @@ exports.saveModule = function* (mod) { }; exports.listModuleAbbreviatedsByName = function* (name) { + if (!config.enableAbbreviatedMetadata) { + return []; + } + var rows = yield models.ModuleAbbreviated.findAll({ where: { name: name, diff --git a/test/controllers/sync.test.js b/test/controllers/sync.test.js index f6d8dbd0f..10b33384f 100644 --- a/test/controllers/sync.test.js +++ b/test/controllers/sync.test.js @@ -94,7 +94,7 @@ describe('test/controllers/sync.test.js', () => { .get('/pedding/sync/log/' + logIdRegistry) .expect(200, function (err, res) { should.not.exist(err); - res.body.should.have.keys('ok', 'log'); + res.body.should.have.keys('ok', 'log', 'syncDone'); done(); }); @@ -102,7 +102,7 @@ describe('test/controllers/sync.test.js', () => { .get('/sync/pedding/log/' + logIdWeb + '?offset=1') .expect(200, function (err, res) { should.not.exist(err); - res.body.should.have.keys('ok', 'log'); + res.body.should.have.keys('ok', 'log', 'syncDone'); done(); }); }); @@ -131,8 +131,8 @@ describe('test/controllers/sync.test.js', () => { .get('/pedding/sync/log/' + logIdRegistry2) .expect(200, function (err, res) { should.not.exist(err); - res.body.should.have.keys('ok', 'log'); - res.body.log.should.containEql(', syncUpstreamFirst: true'); + res.body.should.have.keys('ok', 'log', 'syncDone'); + // res.body.log.should.containEql(', syncUpstreamFirst: true'); done(); }); @@ -140,7 +140,7 @@ describe('test/controllers/sync.test.js', () => { .get('/sync/pedding/log/' + logIdRegistry2 + '?offset=1') .expect(200, function (err, res) { should.not.exist(err); - res.body.should.have.keys('ok', 'log'); + res.body.should.have.keys('ok', 'log', 'syncDone'); done(); }); }, 3000); diff --git a/test/services/module_log.test.js b/test/services/module_log.test.js index f0cfac297..579b5ac70 100644 --- a/test/services/module_log.test.js +++ b/test/services/module_log.test.js @@ -31,8 +31,8 @@ describe('test/services/module_log.test.js', () => { var biglog = Buffer.alloc(50 * 1024 + 1).fill(71).toString(); log = yield ModuleLog.append(logid, biglog); - log.log.substring(0, 4).should.equal('...\n'); - log.log.length.should.equal(50 * 1024 + 4); + log.log.substring(1023, 1024 + 27).should.equal('G\n... ignore long logs ...\nG'); + log.log.length.should.equal(50 * 1024 + 26 + 1024); }); it('should slice log when size equal 50kb', function* () { diff --git a/view/web/sync.html b/view/web/sync.html index 6c7fa5831..436e72c26 100644 --- a/view/web/sync.html +++ b/view/web/sync.html @@ -51,41 +51,38 @@

Log

$notify.html('
Sync request error.
'); } - var offset = 0; - var logs = ''; var syncDone = false; var hasFail = false; function getSyncLog(id) { $.ajax({ - url: resourceURL + '/log/' + id + '?offset=' + offset, + url: resourceURL + '/log/' + id, type: 'GET', dataType: 'json', success: function (data) { - if (!data.ok || !data.log) { + if (!data.ok) { return; } - offset += data.log.split('\n').length; - logs = logs + '\n' + data.log; - - if (data.log.indexOf('[done] Sync') >= 0) { - syncDone = true; - } - if (data.log.indexOf('Fail: [') >= 0) { - var failInfo = data.log.match(/Fail: \[ (.*?) \]/); + syncDone = data.syncDone; + var log = data.log || ''; + if (log.indexOf('Fail: [') >= 0) { + var failInfo = log.match(/Fail: \[ (.*?) \]/); if (failInfo && failInfo[1]) { hasFail = true; } } if (syncDone) { - logs += '\nSync ' + name + ' complete!'; + log += '\nSync ' + name + ' complete!'; if (hasFail) { - logs += ' But some packages sync failed, you can refresh to sync again.'; + log += ' But some packages sync failed, you can refresh to sync again.'; location.hash = ''; + $notify.html('
Sync failed.
'); + } else { + $notify.html('
Sync success.
'); } clearInterval(timer); } - $log.html(logs); + $log.html(log); } - }) + }); }