Skip to content

Commit

Permalink
fix: should return [done] Sync {name} string when task finished (#1382
Browse files Browse the repository at this point in the history 
)

the cnpm client sync need the `[done] Sync {name}` string to detect when sync task finished
  • Loading branch information
@fengmk2
fengmk2 committed Sep 8, 2018
1 parent 68ef698 commit fc79930
Show file tree
Hide file tree
Showing 8 changed files with 64 additions and 47 deletions.
8 changes: 7 additions & 1 deletion controllers/sync.js
View file
Expand Up @@ -63,8 +63,14 @@ exports.getSyncLog = function* (next) {
}

var log = row.log.trim();
var syncDone = row.log.indexOf('[done] Sync') >= 0;
if (offset > 0) {
log = log.split('\n').slice(offset).join('\n');
if (!log && syncDone) {
// append the last 1k string
// the cnpm client sync need the `[done] Sync {name}` string to detect when sync task finished
log = '... ignore long logs ...\n' + row.log.substring(row.log.length - 1024);
}
}
this.body = {ok: true, log: log};
this.body = { ok: true, syncDone: syncDone, log: log };
};
38 changes: 18 additions & 20 deletions controllers/sync_module_worker.js
View file
Expand Up @@ -37,6 +37,7 @@ function SyncModuleWorker(options) {
this._logId = options.logId;
this._log = '';
this._loging = false;
this._isEnd = false;
if (!Array.isArray(options.name)) {
options.name = [options.name];
}
Expand Down Expand Up @@ -78,6 +79,8 @@ SyncModuleWorker.prototype.finish = function () {
this.type,
this.successes.length, this.fails.length,
this.successes.join(', '), this.fails.join(', '));
this._saveLog();
this._isEnd = true;
this.emit('end');
// make sure all event listeners release
this.removeAllListeners();
Expand All @@ -100,10 +103,9 @@ SyncModuleWorker.prototype.log = function () {
}
};

// isEnd will be set to true when sync process success or error
SyncModuleWorker.prototype._saveLog = function (isEnd) {
SyncModuleWorker.prototype._saveLog = function () {
var that = this;
if (that._loging && !isEnd) {
if (that._loging) {
return;
}
that._loging = true;
Expand All @@ -119,14 +121,14 @@ SyncModuleWorker.prototype._saveLog = function (isEnd) {
yield logService.append(that._logId, logstr);
}).then(function () {
that._loging = false;
if (isEnd && that._log) {
if (that._log) {
that._saveLog();
}
}).catch(function (err) {
that._loging = false;
logger.error(err);
// ignore the unsave log
if (isEnd) {
if (this._isEnd) {
logger.error('[SyncModuleWorker] skip to save %s logstr: %s', that._logId, logstr);
}
});
Expand All @@ -149,7 +151,8 @@ SyncModuleWorker.prototype.start = function () {

if (that.type === 'user') {
yield that.syncUser();
that._saveLog(true);
that._saveLog();
that._isEnd = true;
return;
}

Expand All @@ -158,10 +161,10 @@ SyncModuleWorker.prototype.start = function () {
arr.push(that.next(i));
}
yield arr;
that._saveLog(true);
that._saveLog();
}).catch(function (err) {
logger.error(err);
that._saveLog(true);
that._saveLog();
});
};

Expand Down Expand Up @@ -253,17 +256,12 @@ SyncModuleWorker.prototype.syncUpstream = function* (name) {
}

var data = rs.data;
var syncDone = false;
if (data.log && data.log.indexOf('[done] Sync') >= 0) {
syncDone = true;
data.log = data.log.replace('[done] Sync', '[Upstream done] Sync');
}

if (data.log) {
data.log = data.log.replace('[done] Sync', '[Upstream done] Sync');
this.log(data.log);
}

if (syncDone) {
if (data.syncDone) {
break;
}

Expand Down Expand Up @@ -297,17 +295,17 @@ SyncModuleWorker.prototype.syncUser = function* () {
};

SyncModuleWorker.prototype.next = function* (concurrencyId) {
var name = this.names.shift();
if (!name) {
return setImmediate(this.finish.bind(this));
}

if (config.syncModel === 'none') {
this.log('[c#%d] [%s] syncModel is none, ignore',
concurrencyId, name);
return this.finish();
}

var name = this.names.shift();
if (!name) {
return setImmediate(this.finish.bind(this));
}

// try to sync from official replicate when source npm registry is not cnpmjs.org
const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : config.officialNpmReplicate;

Expand Down
4 changes: 2 additions & 2 deletions services/module_log.js
View file
Expand Up @@ -30,8 +30,8 @@ exports.append = function* (id, log) {
row.log = log;
}
if (row.log.length > MAX_LEN) {
// only keep the last 50kb log string
row.log = '...\n' + row.log.substring(row.log.length - MAX_LEN);
// only keep the fisrt 1kb and the last 50kb log string
row.log = row.log.substring(0, 1024) + '\n... ignore long logs ...\n' + row.log.substring(row.log.length - MAX_LEN);
}
return yield row.save(['log']);
};
Expand Down
14 changes: 13 additions & 1 deletion services/npm.js
View file
Expand Up @@ -153,11 +153,23 @@ exports.getAllToday = function* (timeout) {
};

exports.getShort = function* (timeout) {
const registry = config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : 'https://r.cnpmjs.org';
var r = yield request('/-/short', {
timeout: timeout || 300000,
// registry.npmjs.org/-/short is 404 now therefore have a fallback
registry: config.sourceNpmRegistryIsCNpm ? config.sourceNpmRegistry : 'http://r.cnpmjs.org',
registry: registry,
});
if (r.status !== 200) {
const data = r.data;
if (data && data.code && data.message) {
// { code: 'MethodNotAllowedError', message: 'GET is not allowed' }
const url = registry + '/-/short';
const err = new Error(data.message + ', url: ' + url);
err.name = data.code;
err.url = url;
throw err;
}
}
return r.data;
};

Expand Down
4 changes: 4 additions & 0 deletions services/package.js
View file
Expand Up @@ -324,6 +324,10 @@ exports.saveModule = function* (mod) {
};

exports.listModuleAbbreviatedsByName = function* (name) {
if (!config.enableAbbreviatedMetadata) {
return [];
}

var rows = yield models.ModuleAbbreviated.findAll({
where: {
name: name,
Expand Down
10 changes: 5 additions & 5 deletions test/controllers/sync.test.js
View file
Expand Up @@ -94,15 +94,15 @@ describe('test/controllers/sync.test.js', () => {
.get('/pedding/sync/log/' + logIdRegistry)
.expect(200, function (err, res) {
should.not.exist(err);
res.body.should.have.keys('ok', 'log');
res.body.should.have.keys('ok', 'log', 'syncDone');
done();
});

request(webApp)
.get('/sync/pedding/log/' + logIdWeb + '?offset=1')
.expect(200, function (err, res) {
should.not.exist(err);
res.body.should.have.keys('ok', 'log');
res.body.should.have.keys('ok', 'log', 'syncDone');
done();
});
});
Expand Down Expand Up @@ -131,16 +131,16 @@ describe('test/controllers/sync.test.js', () => {
.get('/pedding/sync/log/' + logIdRegistry2)
.expect(200, function (err, res) {
should.not.exist(err);
res.body.should.have.keys('ok', 'log');
res.body.log.should.containEql(', syncUpstreamFirst: true');
res.body.should.have.keys('ok', 'log', 'syncDone');
// res.body.log.should.containEql(', syncUpstreamFirst: true');
done();
});

request(webApp)
.get('/sync/pedding/log/' + logIdRegistry2 + '?offset=1')
.expect(200, function (err, res) {
should.not.exist(err);
res.body.should.have.keys('ok', 'log');
res.body.should.have.keys('ok', 'log', 'syncDone');
done();
});
}, 3000);
Expand Down
4 changes: 2 additions & 2 deletions test/services/module_log.test.js
View file
Expand Up @@ -31,8 +31,8 @@ describe('test/services/module_log.test.js', () => {

var biglog = Buffer.alloc(50 * 1024 + 1).fill(71).toString();
log = yield ModuleLog.append(logid, biglog);
log.log.substring(0, 4).should.equal('...\n');
log.log.length.should.equal(50 * 1024 + 4);
log.log.substring(1023, 1024 + 27).should.equal('G\n... ignore long logs ...\nG');
log.log.length.should.equal(50 * 1024 + 26 + 1024);
});

it('should slice log when size equal 50kb', function* () {
Expand Down
29 changes: 13 additions & 16 deletions view/web/sync.html
View file
Expand Up @@ -51,41 +51,38 @@ <h2>Log</h2>
$notify.html('<div class="ant-alert ant-alert-error">Sync request error.</div>');
}

var offset = 0;
var logs = '';
var syncDone = false;
var hasFail = false;
function getSyncLog(id) {
$.ajax({
url: resourceURL + '/log/' + id + '?offset=' + offset,
url: resourceURL + '/log/' + id,
type: 'GET',
dataType: 'json',
success: function (data) {
if (!data.ok || !data.log) {
if (!data.ok) {
return;
}
offset += data.log.split('\n').length;
logs = logs + '\n' + data.log;

if (data.log.indexOf('[done] Sync') >= 0) {
syncDone = true;
}
if (data.log.indexOf('Fail: [') >= 0) {
var failInfo = data.log.match(/Fail: \[ (.*?) \]/);
syncDone = data.syncDone;
var log = data.log || '';
if (log.indexOf('Fail: [') >= 0) {
var failInfo = log.match(/Fail: \[ (.*?) \]/);
if (failInfo && failInfo[1]) {
hasFail = true;
}
}
if (syncDone) {
logs += '\nSync ' + name + ' complete!';
log += '\nSync ' + name + ' complete!';
if (hasFail) {
logs += ' But some packages sync failed, you can refresh to sync again.';
log += ' But some packages sync failed, you can refresh to sync again.';
location.hash = '';
$notify.html('<div class="ant-alert ant-alert-error">Sync failed.</div>');
} else {
$notify.html('<div class="ant-alert ant-alert-success">Sync success.</div>');
}
clearInterval(timer);
}
$log.html(logs);
$log.html(log);
}
})
});
}
</script>

0 comments on commit fc79930

Please sign in to comment.