Skip to content

Commit e563e56

Browse files
lisonglisong
committed
support google diff-match-patch
1 parent 3452d9b commit e563e56

File tree

16 files changed

+3117
-306
lines changed

16 files changed

+3117
-306
lines changed

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ test: test-integration
88

99
test-integration:
1010
@echo "\nRunning integration tests..."
11-
@NODE_ENV=test CONFIG_FILE=${ROOT}/config/config.test.js mocha test/api/init &
11+
@NODE_ENV=test CONFIG_FILE=${ROOT}/config/config.test.js mocha test/api/init
1212
@NODE_ENV=test PORT=3000 HOST=127.0.0.1 CONFIG_FILE=${ROOT}/config/config.test.js node bin/www &
1313
@NODE_ENV=test CONFIG_FILE=${ROOT}/config/config.test.js mocha \
1414
test/api/users test/api/auth test/api/account test/api/accessKeys test/api/apps test/api/index --recursive --timeout 15000
@@ -18,7 +18,7 @@ coverage:
1818
rm -rf coverage
1919
# @NODE_ENV=test CONFIG_FILE=${ROOT}/config/config.test.js ./node_modules/istanbul/lib/cli.js cover --report lcovonly --dir coverage/core ./node_modules/.bin/_mocha \
2020
# test/unit -- -R spec --recursive --timeout 15000
21-
@NODE_ENV=test CONFIG_FILE=${ROOT}/config/config.test.js mocha test/api/init &
21+
@NODE_ENV=test CONFIG_FILE=${ROOT}/config/config.test.js mocha test/api/init
2222
@NODE_ENV=test PORT=3000 HOST=127.0.0.1 CONFIG_FILE=${ROOT}/config/config.test.js node bin/www &
2323
@NODE_ENV=test CONFIG_FILE=${ROOT}/config/config.test.js ./node_modules/istanbul/lib/cli.js cover --report lcovonly --dir coverage/api ./node_modules/.bin/_mocha \
2424
test/api/users test/api/auth test/api/account test/api/accessKeys test/api/apps test/api/index -- -R spec --recursive --timeout 15000

‎bin/db‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ var _ = require('lodash');
99
var mysql = require('mysql2');
1010
var Promise = require("bluebird");
1111
var common = require("../core/utils/common");
12+
var constConfig = require('../core/const');
1213
var yargs = require('yargs')
1314
.usage('Usage: $0 <command> [options]')
1415
.command('init', '初始化数据库', {
@@ -97,15 +98,16 @@ if (command === 'init') {
9798
var version_no = '0.0.1';
9899
var rs = yield connection.queryAsync('select `version` from `versions` where `type`=1 limit 1');
99100
version_no = _.get(rs,'0.version', '0.0.1');
100-
if (version_no == '0.4.0') {
101+
if (version_no == constConfig.CURRENT_DB_VERSION) {
101102
console.log('Everything up-to-date.');
102103
process.exit(0);
103104
}
104105
var allSqlFile = [
105106
{version:'0.2.14', 'path':path.resolve(__dirname, '../sql/codepush-v0.2.14-patch.sql')},
106107
{version:'0.2.15', 'path':path.resolve(__dirname, '../sql/codepush-v0.2.15-patch.sql')},
107108
{version:'0.3.0', 'path':path.resolve(__dirname, '../sql/codepush-v0.3.0-patch.sql')},
108-
{version:'0.4.0', 'path':path.resolve(__dirname, '../sql/codepush-v0.4.0-patch.sql')}
109+
{version:'0.4.0', 'path':path.resolve(__dirname, '../sql/codepush-v0.4.0-patch.sql')},
110+
{version:'0.5.0', 'path':path.resolve(__dirname, '../sql/codepush-v0.5.0-patch.sql')}
109111
];
110112
for (var i = 0; i < allSqlFile.length; i++) {
111113
if(!_.gt(allSqlFile[i]['version'], version_no)) {

‎bin/www‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ var http = require('http');
99
var validator = require('validator')
1010
var _ = require('lodash')
1111
var config = require('../core/config');
12+
var constConfig = require('../core/const');
1213
log4js.configure(_.get(config, 'log4js', {
1314
appenders: {console: { type: 'console'}},
1415
categories : { default: { appenders: ['console'], level: 'info' }}
@@ -48,8 +49,8 @@ var server = http.createServer(app);
4849
var models = require('../models');
4950
models.Versions.findOne({where:{type:1}})
5051
.then(function(v){
51-
if (!v || v.get('version') != '0.4.0') {
52-
throw new Error(`Please upgrade your database. usage bin/db upgrade or code-push-server-db upgrade`);
52+
if (!v || v.get('version') != constConfig.CURRENT_DB_VERSION) {
53+
throw new Error('Please upgrade your database. usage `npm run upgrade` or `code-push-server-db upgrade`');
5354
}
5455
server.listen(port, host);
5556
server.on('error', onError);

‎core/const.js‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,4 +39,11 @@ define("DEPLOYMENT_FAILED", 2);
3939

4040
define("DIFF_MANIFEST_FILE_NAME", 'hotcodepush.json');
4141

42+
//文本文件是否使用google diff-match-patch 计算差异
43+
define("IS_USE_DIFF_TEXT_NO", 0);
44+
define("IS_USE_DIFF_TEXT_YES", 1);
45+
46+
47+
define("CURRENT_DB_VERSION", '0.5.0');
48+
4249

‎core/services/datacenter-manager.js‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ const CONTENTS_NAME = 'contents';
1111
var AppError = require('../app-error');
1212
var log4js = require('log4js');
1313
var log = log4js.getLogger("cps:DataCenterManager");
14+
var path = require('path');
1415

1516
var proto = module.exports = function (){
1617
function DataCenterManager() {
@@ -30,18 +31,18 @@ proto.getDataDir = function () {
3031

3132
proto.hasPackageStoreSync = function (packageHash) {
3233
var dataDir = this.getDataDir();
33-
var packageHashPath = `${dataDir}/${packageHash}`;
34-
var manifestFile = `${packageHashPath}/${MANIFEST_FILE_NAME}`;
35-
var contentPath = `${packageHashPath}/${CONTENTS_NAME}`;
34+
var packageHashPath = path.join(dataDir, packageHash);
35+
var manifestFile = path.join(packageHashPath, MANIFEST_FILE_NAME);
36+
var contentPath = path.join(packageHashPath, CONTENTS_NAME);
3637
return fs.existsSync(manifestFile) && fs.existsSync(contentPath);
3738
}
3839

3940
proto.getPackageInfo = function (packageHash) {
4041
if (this.hasPackageStoreSync(packageHash)){
4142
var dataDir = this.getDataDir();
42-
var packageHashPath = `${dataDir}/${packageHash}`;
43-
var manifestFile = `${packageHashPath}/${MANIFEST_FILE_NAME}`;
44-
var contentPath = `${packageHashPath}/${CONTENTS_NAME}`;
43+
var packageHashPath = path.join(dataDir, packageHash);
44+
var manifestFile = path.join(packageHashPath, MANIFEST_FILE_NAME);
45+
var contentPath = path.join(packageHashPath, CONTENTS_NAME);
4546
return this.buildPackageInfo(packageHash, packageHashPath, contentPath, manifestFile);
4647
} else {
4748
throw new AppError.AppError('can\'t get PackageInfo');
@@ -59,9 +60,9 @@ proto.buildPackageInfo = function (packageHash, packageHashPath, contentPath, ma
5960

6061
proto.validateStore = function (providePackageHash) {
6162
var dataDir = this.getDataDir();
62-
var packageHashPath = `${dataDir}/${providePackageHash}`;
63-
var manifestFile = `${packageHashPath}/${MANIFEST_FILE_NAME}`;
64-
var contentPath = `${packageHashPath}/${CONTENTS_NAME}`;
63+
var packageHashPath = path.join(dataDir, providePackageHash);
64+
var manifestFile = path.join(packageHashPath, MANIFEST_FILE_NAME);
65+
var contentPath = path.join(packageHashPath, CONTENTS_NAME);
6566
if (!this.hasPackageStoreSync(providePackageHash)) {
6667
log.debug(`validateStore providePackageHash not exist`);
6768
return Promise.resolve(false);
@@ -98,9 +99,9 @@ proto.storePackage = function (sourceDst, force) {
9899
var packageHash = security.packageHashSync(manifestJson);
99100
log.debug('storePackage manifestJson packageHash:', packageHash);
100101
var dataDir = self.getDataDir();
101-
var packageHashPath = `${dataDir}/${packageHash}`;
102-
var manifestFile = `${packageHashPath}/${MANIFEST_FILE_NAME}`;
103-
var contentPath = `${packageHashPath}/${CONTENTS_NAME}`;
102+
var packageHashPath = path.join(dataDir, packageHash);
103+
var manifestFile = path.join(packageHashPath, MANIFEST_FILE_NAME);
104+
var contentPath = path.join(packageHashPath, CONTENTS_NAME);
104105
return self.validateStore(packageHash)
105106
.then((isValidate) => {
106107
if (!force && isValidate) {
@@ -109,7 +110,7 @@ proto.storePackage = function (sourceDst, force) {
109110
log.debug(`storePackage cover from sourceDst:`, sourceDst);
110111
return common.createEmptyFolder(packageHashPath)
111112
.then(() => {
112-
return common.move(sourceDst, contentPath)
113+
return common.copy(sourceDst, contentPath)
113114
.then(() => {
114115
var manifestString = JSON.stringify(manifestJson);
115116
fs.writeFileSync(manifestFile, manifestString);

‎core/services/package-manager.js‎

Lines changed: 98 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -166,9 +166,9 @@ proto.downloadPackageAndExtract = function (workDirectoryPath, packageHash, blob
166166
return dataCenterManager.getPackageInfo(packageHash);
167167
} else {
168168
var downloadURL = common.getBlobDownloadUrl(blobHash);
169-
return common.createFileFromRequest(downloadURL, `${workDirectoryPath}/${blobHash}`)
169+
return common.createFileFromRequest(downloadURL, path.join(workDirectoryPath, blobHash))
170170
.then((download) => {
171-
return common.unzipFile(`${workDirectoryPath}/${blobHash}`, `${workDirectoryPath}/current`)
171+
return common.unzipFile(path.join(workDirectoryPath, blobHash), path.join(workDirectoryPath, 'current'))
172172
.then((outputPath) => {
173173
return dataCenterManager.storePackage(outputPath, true);
174174
});
@@ -193,14 +193,22 @@ proto.zipDiffPackage = function (fileName, files, baseDirectoryPath, hotCodePush
193193
});
194194
for (var i = 0; i < files.length; ++i) {
195195
var file = files[i];
196-
zipFile.addFile(`${baseDirectoryPath}/${file}`, slash(file));
196+
zipFile.addFile(path.join(baseDirectoryPath, file), slash(file));
197197
}
198198
zipFile.addFile(hotCodePushFile, constConfig.DIFF_MANIFEST_FILE_NAME);
199199
zipFile.end();
200200
});
201201
}
202202

203-
proto.generateOneDiffPackage = function (workDirectoryPath, packageId, dataCenter, diffPackageHash, diffManifestBlobHash) {
203+
proto.generateOneDiffPackage = function (
204+
workDirectoryPath,
205+
packageId,
206+
originDataCenter,
207+
oldPackageDataCenter,
208+
diffPackageHash,
209+
diffManifestBlobHash,
210+
isUseDiffText
211+
) {
204212
var self = this;
205213
return models.PackagesDiff.findOne({
206214
where:{
@@ -212,24 +220,53 @@ proto.generateOneDiffPackage = function (workDirectoryPath, packageId, dataCente
212220
if (!_.isEmpty(diffPackage)) {
213221
return;
214222
}
223+
log.debug('originDataCenter', originDataCenter);
224+
log.debug('oldPackageDataCenter', oldPackageDataCenter);
215225
var downloadURL = common.getBlobDownloadUrl(diffManifestBlobHash);
216-
return common.createFileFromRequest(downloadURL, `${workDirectoryPath}/${diffManifestBlobHash}`)
226+
return common.createFileFromRequest(downloadURL, path.join(workDirectoryPath,diffManifestBlobHash))
217227
.then(() => {
218-
var originContentPath = dataCenter.contentPath;
219-
var originManifestJson = JSON.parse(fs.readFileSync(dataCenter.manifestFilePath, "utf8"))
220-
var diffManifestJson = JSON.parse(fs.readFileSync(`${workDirectoryPath}/${diffManifestBlobHash}`, "utf8"))
228+
var dataCenterContentPath = path.join(workDirectoryPath, 'dataCenter');
229+
common.copySync(originDataCenter.contentPath, dataCenterContentPath);
230+
var oldPackageDataCenterContentPath = oldPackageDataCenter.contentPath;
231+
var originManifestJson = JSON.parse(fs.readFileSync(originDataCenter.manifestFilePath, "utf8"))
232+
var diffManifestJson = JSON.parse(fs.readFileSync(path.join(workDirectoryPath, diffManifestBlobHash), "utf8"))
221233
var json = common.diffCollectionsSync(originManifestJson, diffManifestJson);
222234
var files = _.concat(json.diff, json.collection1Only);
223-
var hotcodepush = {deletedFiles: json.collection2Only};
224-
var hotCodePushFile = `${workDirectoryPath}/${diffManifestBlobHash}_hotcodepush`;
235+
var hotcodepush = {deletedFiles: json.collection2Only, patchedFiles:[]};
236+
if (isUseDiffText == constConfig.IS_USE_DIFF_TEXT_YES) {
237+
//使用google diff-match-patch
238+
_.forEach(json.diff, function(tmpFilePath) {
239+
var dataCenterContentPathTmpFilePath = path.join(dataCenterContentPath, tmpFilePath);
240+
var oldPackageDataCenterContentPathTmpFilePath = path.join(oldPackageDataCenterContentPath, tmpFilePath);
241+
if (
242+
fs.existsSync(dataCenterContentPathTmpFilePath)
243+
&& fs.existsSync(oldPackageDataCenterContentPathTmpFilePath)
244+
&& common.detectIsTextFile(dataCenterContentPathTmpFilePath)
245+
&& common.detectIsTextFile(oldPackageDataCenterContentPathTmpFilePath)
246+
) {
247+
var textOld = fs.readFileSync(oldPackageDataCenterContentPathTmpFilePath, 'utf-8');
248+
var textNew = fs.readFileSync(dataCenterContentPathTmpFilePath, 'utf-8');
249+
if (!textOld || !textNew) {
250+
return;
251+
}
252+
var DiffMatchPatch = require('diff-match-patch');
253+
var dmp = new DiffMatchPatch();
254+
var patchs = dmp.patch_make(textOld, textNew);
255+
var patchText = dmp.patch_toText(patchs);
256+
if (patchText && patchText.length < _.parseInt(textNew.length * 0.8)) {
257+
fs.writeFileSync(dataCenterContentPathTmpFilePath, patchText);
258+
hotcodepush.patchedFiles.push(tmpFilePath);
259+
}
260+
}
261+
});
262+
}
263+
var hotCodePushFile = path.join(workDirectoryPath,`${diffManifestBlobHash}_hotcodepush`);;
225264
fs.writeFileSync(hotCodePushFile, JSON.stringify(hotcodepush));
226-
var fileName = `${workDirectoryPath}/${diffManifestBlobHash}.zip`;
227-
228-
return self.zipDiffPackage(fileName, files, originContentPath, hotCodePushFile)
265+
var fileName = path.join(workDirectoryPath,`${diffManifestBlobHash}.zip`);;
266+
return self.zipDiffPackage(fileName, files, dataCenterContentPath, hotCodePushFile)
229267
.then((data) => {
230268
return security.qetag(data.path)
231269
.then((diffHash) => {
232-
log.debug('diff');
233270
return common.uploadFileToStorage(diffHash, fileName)
234271
.then(() => {
235272
var stats = fs.statSync(fileName);
@@ -246,40 +283,36 @@ proto.generateOneDiffPackage = function (workDirectoryPath, packageId, dataCente
246283
});
247284
};
248285

249-
proto.createDiffPackagesByLastNums = function (packageId, num) {
286+
proto.createDiffPackagesByLastNums = function (appId, originalPackage, num) {
250287
var self = this;
251-
return models.Packages.findById(packageId)
252-
.then((originalPackage) => {
253-
if (_.isEmpty(originalPackage)) {
254-
throw AppError.AppError('can\'t find Package');
255-
}
256-
var Sequelize = require('sequelize');
257-
return Promise.all([
258-
models.Packages.findAll({
259-
where:{
260-
deployment_version_id: originalPackage.deployment_version_id,
261-
id: {[Sequelize.Op.lt]: packageId}},
262-
order: [['id','desc']],
263-
limit: num
264-
}),
265-
models.Packages.findAll({
266-
where:{
267-
deployment_version_id: originalPackage.deployment_version_id,
268-
id: {[Sequelize.Op.lt]: packageId}},
269-
order: [['id','asc']],
270-
limit: 2
271-
})
272-
])
273-
.spread((lastNumsPackages, basePackages) => {
274-
return _.unionBy(lastNumsPackages, basePackages, 'id');
275-
})
276-
.then((lastNumsPackages) => {
277-
return self.createDiffPackages(originalPackage, lastNumsPackages);
278-
});
288+
var Sequelize = require('sequelize');
289+
var packageId = originalPackage.id;
290+
return Promise.all([
291+
models.Packages.findAll({
292+
where:{
293+
deployment_version_id: originalPackage.deployment_version_id,
294+
id: {[Sequelize.Op.lt]: packageId}},
295+
order: [['id','desc']],
296+
limit: num
297+
}),
298+
models.Packages.findAll({
299+
where:{
300+
deployment_version_id: originalPackage.deployment_version_id,
301+
id: {[Sequelize.Op.lt]: packageId}},
302+
order: [['id','asc']],
303+
limit: 2
304+
}),
305+
models.Apps.findById(appId),
306+
])
307+
.spread((lastNumsPackages, basePackages, appInfo) => {
308+
return [_.uniqBy(_.unionBy(lastNumsPackages, basePackages, 'id'), 'package_hash'), appInfo];
309+
})
310+
.spread((lastNumsPackages, appInfo) => {
311+
return self.createDiffPackages(originalPackage, lastNumsPackages, _.get(appInfo, 'is_use_diff_text', constConfig.IS_USE_DIFF_TEXT_NO));
279312
});
280313
};
281314

282-
proto.createDiffPackages = function (originalPackage, destPackages) {
315+
proto.createDiffPackages = function (originalPackage, destPackages, isUseDiffText) {
283316
if (!_.isArray(destPackages)) {
284317
return Promise.reject(new AppError.AppError('第二个参数必须是数组'));
285318
}
@@ -291,10 +324,26 @@ proto.createDiffPackages = function (originalPackage, destPackages) {
291324
var manifest_blob_url = _.get(originalPackage, 'manifest_blob_url');
292325
var blob_url = _.get(originalPackage, 'blob_url');
293326
var workDirectoryPath = path.join(os.tmpdir(), 'codepush_' + security.randToken(32));
327+
log.debug('workDirectoryPath', workDirectoryPath);
294328
return common.createEmptyFolder(workDirectoryPath)
295329
.then(() => self.downloadPackageAndExtract(workDirectoryPath, package_hash, blob_url))
296-
.then((dataCenter) => Promise.map(destPackages,
297-
(v) => self.generateOneDiffPackage(workDirectoryPath, originalPackage.id, dataCenter, v.package_hash, v.manifest_blob_url)
330+
.then((originDataCenter) => Promise.map(destPackages,
331+
(v) => {
332+
var diffWorkDirectoryPath = path.join(workDirectoryPath, _.get(v, 'package_hash'));
333+
common.createEmptyFolderSync(diffWorkDirectoryPath);
334+
return self.downloadPackageAndExtract(diffWorkDirectoryPath, _.get(v, 'package_hash'), _.get(v, 'blob_url'))
335+
.then((oldPackageDataCenter) =>
336+
self.generateOneDiffPackage(
337+
diffWorkDirectoryPath,
338+
originalPackage.id,
339+
originDataCenter,
340+
oldPackageDataCenter,
341+
v.package_hash,
342+
v.manifest_blob_url,
343+
isUseDiffText
344+
)
345+
)
346+
}
298347
))
299348
.finally(() => common.deleteFolderSync(workDirectoryPath));
300349
}
@@ -312,7 +361,8 @@ proto.releasePackage = function (appId, deploymentId, packageInfo, filePath, rel
312361
var rollout = packageInfo.rollout; //灰度百分比
313362
var isMandatory = packageInfo.isMandatory; //是否强制更新，无法跳过
314363
var tmpDir = os.tmpdir();
315-
var directoryPath = path.join(tmpDir, 'codepush_' + security.randToken(32));
364+
var directoryPathParent = path.join(tmpDir, 'codepuh_' + security.randToken(32));
365+
var directoryPath = path.join(directoryPathParent, 'current');
316366
log.debug(`releasePackage generate an random dir path: ${directoryPath}`);
317367
return Promise.all([
318368
security.qetag(filePath),
@@ -382,7 +432,7 @@ proto.releasePackage = function (appId, deploymentId, packageInfo, filePath, rel
382432
}
383433
return self.createPackage(deploymentId, appVersion, packageHash, manifestHash, blobHash, params);
384434
})
385-
.finally(() => common.deleteFolderSync(directoryPath))
435+
.finally(() => common.deleteFolderSync(directoryPathParent))
386436
};
387437

388438
proto.modifyReleasePackage = function(packageId, params) {

0 commit comments

Comments
 (0)