Skip to content

Commit

Permalink
Merge pull request #105 from githubsaturn/feature/detached_builds
Browse files Browse the repository at this point in the history 
Feature/detached builds
  • Loading branch information
@githubsaturn
githubsaturn committed Jan 8, 2018
2 parents d0fa232 + 049c29f commit b64c317
Show file tree
Hide file tree
Showing 6 changed files with 342 additions and 57 deletions.
2 changes: 2 additions & 0 deletions app-backend/src/api/ApiStatusCodes.js
View file
Expand Up @@ -10,6 +10,8 @@ let apiStatusCode = {

STATUS_OK: 100,

STATUS_OK_DEPLOY_STARTED: 101,

STATUS_ERROR_GENERIC: 1000,

STATUS_ERROR_CAPTAIN_NOT_INITIALIZED: 1001,
Expand Down
74 changes: 56 additions & 18 deletions app-backend/src/docker/DockerApi.js
View file
Expand Up @@ -6,6 +6,17 @@ const CaptainConstants = require('../utils/CaptainConstants');
const Logger = require('../utils/Logger');
const EnvVars = require('../utils/EnvVars');

function safeParseChunk(chunk) {
try {
return JSON.parse(chunk);
}
catch (ignore) {
return {
stream: 'Cannot parse ' + chunk
}
}
}

class DockerApi {

constructor(connectionParams) {
Expand Down Expand Up @@ -174,7 +185,7 @@ class DockerApi {
});
}

buildImageFromDockerFile(imageName, newVersion, tarballFilePath) {
buildImageFromDockerFile(imageName, newVersion, tarballFilePath, buildLogs) {

const self = this;

Expand All @@ -191,32 +202,29 @@ class DockerApi {
return new Promise(function (resolve, reject) {

let errorMessage = '';
let logsBeforeError = [];
for (let i = 0; i < 20; i++) {
logsBeforeError.push('');
}

stream.setEncoding('utf8');

// THIS BLOCK HAS TO BE HERE. "end" EVENT WON'T GET CALLED OTHERWISE.
stream.on('data', function (chunk) {

Logger.dev('stream data ' + chunk);
chunk = JSON.parse(chunk);
chunk = safeParseChunk(chunk);

let chuckStream = chunk.stream;
if (chuckStream) {
// Logger.dev('stream data ' + chuckStream);
logsBeforeError.shift();
logsBeforeError.push(chuckStream);
buildLogs.log(chuckStream);
}

if (chunk.error) {
Logger.e(chunk.error);
Logger.e(JSON.stringify(chunk.errorDetail));
errorMessage += '\n [truncated] \n';
errorMessage += logsBeforeError.join('');
let errorDetails = JSON.stringify(chunk.errorDetail);
Logger.e(errorDetails);
buildLogs.log(errorDetails);
buildLogs.log(chunk.error);
errorMessage += '\n';
errorMessage += errorDetails;
errorMessage += chunk.error;
}
});
Expand Down Expand Up @@ -279,7 +287,7 @@ class DockerApi {
stream.on('data', function (chunk) {

Logger.dev('stream data ' + chunk);
chunk = JSON.parse(chunk);
chunk = safeParseChunk(chunk);

let chuckStream = chunk.stream;
if (chuckStream) {
Expand Down Expand Up @@ -433,7 +441,7 @@ class DockerApi {
});
}

pushImage(imageName, newVersion, authObj) {
pushImage(imageName, newVersion, authObj, buildLogs) {

const self = this;

Expand All @@ -454,23 +462,53 @@ class DockerApi {
})
.then(function (stream) {

return new Promise(function (resolve) {
return new Promise(function (resolve, reject) {

let errorMessage = '';

stream.setEncoding('utf8');

// THIS BLOCK HAS TO BE HERE. "end" EVENT WON'T GET CALLED OTHERWISE.
stream.on('data', function (chunk) {
// THIS BLOCK HAS TO BE HERE. "end" EVENT WON'T GET CALLED OTHERWISE.
// ('stream data ' + chunk);
});

Logger.dev('stream data ' + chunk);
chunk = safeParseChunk(chunk);

let chuckStream = chunk.stream;
if (chuckStream) {
// Logger.dev('stream data ' + chuckStream);
buildLogs.log(chuckStream);
}

if (chunk.error) {
Logger.e(chunk.error);
let errorDetails = JSON.stringify(chunk.errorDetail);
Logger.e(errorDetails);
buildLogs.log(errorDetails);
buildLogs.log(chunk.error);
errorMessage += '\n';
errorMessage += errorDetails;
errorMessage += chunk.error;
}
});

// stream.pipe(process.stdout, {end: true});
// IncomingMessage
// https://nodejs.org/api/stream.html#stream_event_end

stream.on('end', function () {
if (errorMessage) {
reject(errorMessage);
return;
}
resolve();
});
});

stream.on('error', function (chunk) {
errorMessage += chunk;
});

});
});
}

Expand Down
116 changes: 87 additions & 29 deletions app-backend/src/routes/AppDataRouter.js
View file
Expand Up @@ -8,6 +8,39 @@ const fs = require('fs-extra');
const TEMP_UPLOAD = 'temp_upload/';
const upload = multer({dest: TEMP_UPLOAD});


router.get('/:appName/', function (req, res, next) {

let appName = req.params.appName;
let serviceManager = res.locals.user.serviceManager;

return Promise.resolve()
.then(function () {

return serviceManager.getBuildStatus(appName);

})
.then(function (data) {

let baseApi = new BaseApi(ApiStatusCodes.STATUS_OK, 'App build status retrieved');
baseApi.data = data;
res.send(baseApi);

})
.catch(function (error) {

Logger.e(error);

if (error && error.captainErrorType) {
res.send(new BaseApi(error.captainErrorType, error.apiMessage));
return;
}

res.sendStatus(500);
});

});

router.post('/:appName/', function (req, res, next) {

let dataStore = res.locals.user.dataStore;
Expand Down Expand Up @@ -39,6 +72,7 @@ router.post('/:appName/', upload.single('sourceFile'), function (req, res, next)

let dataStore = res.locals.user.dataStore;
let serviceManager = res.locals.user.serviceManager;
let isDetachedBuild = !!req.query.detached;

let appName = req.params.appName;

Expand Down Expand Up @@ -86,34 +120,19 @@ router.post('/:appName/', upload.single('sourceFile'), function (req, res, next)
})
.then(function () {

return serviceManager.createImage(appName, {
pathToSrcTarballFile: tarballSourceFilePath
}, gitHash);

})
.then(function (version) {

fs.removeSync(tarballSourceFilePath);
return version;

})
.catch(function (error) {

return new Promise(function (resolve, reject) {
fs.removeSync(tarballSourceFilePath);
reject(error);
})

})
.then(function (version) {

return serviceManager.ensureServiceInitedAndUpdated(appName, version);

})
.then(function () {

res.send(new BaseApi(ApiStatusCodes.STATUS_OK, 'App Data Saved'));

if (isDetachedBuild) {
res.send(new BaseApi(ApiStatusCodes.STATUS_OK_DEPLOY_STARTED, 'Deploy is started'));
startBuildProcess()
.catch(function (error) {
Logger.e(error);
});
}
else {
return startBuildProcess()
.then(function () {
res.send(new BaseApi(ApiStatusCodes.STATUS_OK, 'Deploy is done'));
});
}
})
.catch(function (error) {

Expand All @@ -130,8 +149,47 @@ router.post('/:appName/', upload.single('sourceFile'), function (req, res, next)

res.send(new BaseApi(ApiStatusCodes.STATUS_ERROR_GENERIC, error.stack + ''));


try {
fs.removeSync(tarballSourceFilePath);
} catch (ignore) {
}
});


function startBuildProcess() {

return serviceManager
.createImage(appName, {
pathToSrcTarballFile: tarballSourceFilePath
}, gitHash)
.then(function (version) {

fs.removeSync(tarballSourceFilePath);
return version;

})
.catch(function (error) {

return new Promise(function (resolve, reject) {
fs.removeSync(tarballSourceFilePath);
reject(error);
})

})
.then(function (version) {

return serviceManager.ensureServiceInitedAndUpdated(appName, version);

})
.catch(function (error) {

return new Promise(function (resolve, reject) {
serviceManager.logBuildFailed(appName, error);
reject(error);
})

});
}
});

module.exports = router;
9 changes: 9 additions & 0 deletions app-backend/src/routes/UserRouter.js
View file
Expand Up @@ -46,6 +46,7 @@ router.use(function (req, res, next) {
return;
}

const serviceManager = res.locals.user.serviceManager;

// All requests except GET might be making changes to some stuff that are not designed for an asynchronous process
// I'm being extra cautious. But removal of this lock mechanism requires testing and consideration of edge cases.
Expand All @@ -57,6 +58,14 @@ router.use(function (req, res, next) {
return;
}

let activeBuildAppName = serviceManager.isAnyBuildRunning();
if (activeBuildAppName) {
let response = new BaseApi(ApiStatusCodes.STATUS_ERROR_GENERIC,
`An active build (${activeBuildAppName}) is in progress... please wait...`);
res.send(response);
return;
}

// we don't want the same space to go under two simultaneous changes
threadLockNamespace[namespace] = true;
onFinished(res, function () {
Expand Down

0 comments on commit b64c317

Please sign in to comment.