From e2d614bc0cbb35ed407c49d23fcf2334bd58726f Mon Sep 17 00:00:00 2001 From: Durgesh1988 Date: Mon, 28 Nov 2016 20:43:26 +0530 Subject: [PATCH 1/2] Task Scheduler Implementation --- .../orchestration/popups/newTaskCtrl.js | 4 +- .../catalyst-scheduler/catalystScheduler.js | 45 +++++++++++++++++-- server/app/model/classes/tasks/tasks.js | 7 ++- .../app/routes/v1.0/routes_organizations.js | 5 ++- server/app/routes/v1.0/routes_tasks.js | 26 +++++++---- server/app/services/auditTrailService.js | 2 +- 6 files changed, 72 insertions(+), 17 deletions(-) diff --git a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTaskCtrl.js b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTaskCtrl.js index bb471efb7..5bfdf69e6 100644 --- a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTaskCtrl.js +++ b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTaskCtrl.js @@ -268,10 +268,10 @@ $scope.repeatPattern = 'Repeat Every -' + $scope.chefJenkScriptTaskObj.repeats; $scope.cronDetails = { cronStartOn : $scope.chefJenkScriptTaskObj.cronStart, - cronStartEnd : $scope.chefJenkScriptTaskObj.cronEnd, + cronEndOn : $scope.chefJenkScriptTaskObj.cronEnd, cronRepeatEvery : $scope.chefJenkScriptTaskObj.repeatBy, cronFrequency: $scope.chefJenkScriptTaskObj.repeats, - cronTime: startTimeHour + ':' + startTimeMinute, + cronTime: typeof startTimeHour !=='undefined'? startTimeHour : new Date().getHours() + ':' + typeof startTimeMinute !=='undefined'? startTimeMinute:new Date().getMinutes(), cronDays: $scope.chefJenkScriptTaskObj.dayOfWeek, cronMonth: $scope.chefJenkScriptTaskObj.monthOfYear } diff --git a/server/app/cronjobs/catalyst-scheduler/catalystScheduler.js b/server/app/cronjobs/catalyst-scheduler/catalystScheduler.js index 24fc17c34..33fb6260c 100644 --- a/server/app/cronjobs/catalyst-scheduler/catalystScheduler.js +++ b/server/app/cronjobs/catalyst-scheduler/catalystScheduler.js @@ -43,12 +43,51 @@ catalystSync.executeScheduledTasks = function executeScheduledTasks() { return; } if (tasks && tasks.length) { - var resultList =[]; + var resultList =[],parallelTaskList=[],serialTaskList=[]; for (var i = 0; i < tasks.length; i++) { (function(task) { - resultList.push(function(callback){schedulerService.executeSchedulerForTasks(task,callback);}); + if(task.executionOrder === 'PARALLEL'){ + resultList.push(function(callback){schedulerService.executeSchedulerForTasks(task,callback);}); + parallelTaskList.push(function(callback){schedulerService.executeSchedulerForTasks(task,callback);}); + }else{ + resultList.push(function(callback){schedulerService.executeSchedulerForTasks(task,callback);}); + if(serialTaskList.length ===0) { + serialTaskList.push(function (next) { + schedulerService.executeSchedulerForTasks(task, next); + }); + }else{ + serialTaskList.push(function (execution,next) { + schedulerService.executeSchedulerForTasks(task, next); + }); + } + } if(resultList.length === tasks.length){ - async.parallel(resultList,function(err,results){ + async.parallel({ + parallelTask: function(callback){ + console.log("Parallel"); + async.parallel(parallelTaskList,function(err,data){ + if(err){ + callback(err,null); + return; + } + logger.debug("Parallel Task Scheduler Completed"); + callback(null,data); + return; + }) + }, + serialTask: function(callback){ + console.log("Serial"); + async.waterfall(serialTaskList,function(err,data){ + if(err){ + callback(err,null); + return; + } + logger.debug("Serial Task Scheduler Completed"); + callback(null,data); + return; + }) + } + },function(err,results){ if(err){ logger.error(err); return; diff --git a/server/app/model/classes/tasks/tasks.js b/server/app/model/classes/tasks/tasks.js index 3734dddeb..6e16c7810 100755 --- a/server/app/model/classes/tasks/tasks.js +++ b/server/app/model/classes/tasks/tasks.js @@ -136,6 +136,11 @@ var taskSchema = new Schema({ required: false, default:false }, + executionOrder:{ + type: String, + required: false, + trim: true + }, taskScheduler:{ cronStartOn: { type: String, @@ -147,7 +152,7 @@ var taskSchema = new Schema({ required: false, trim: true }, - cronPatten: { + cronPattern: { type: String, required: false, trim: true diff --git a/server/app/routes/v1.0/routes_organizations.js b/server/app/routes/v1.0/routes_organizations.js index 530cde351..f2d0c0aaf 100755 --- a/server/app/routes/v1.0/routes_organizations.js +++ b/server/app/routes/v1.0/routes_organizations.js @@ -975,8 +975,9 @@ module.exports.setRoutes = function(app, sessionVerification) { taskData.orgName = project[0].orgname; taskData.bgName = project[0].productgroupname; taskData.projectName = project[0].projectname; - if(req.body.taskScheduler && req.body.taskScheduler !== null) { - taskData.taskScheduler = apiUtil.createCronJobPattern(req.body.taskScheduler); + console.log(JSON.stringify(taskData)); + if(taskData.taskScheduler && taskData.taskScheduler !== null) { + taskData.taskScheduler = apiUtil.createCronJobPattern(taskData.taskScheduler); taskData.isTaskScheduled = true; } configmgmtDao.getEnvNameFromEnvId(req.params.envId, function(err, envName) { diff --git a/server/app/routes/v1.0/routes_tasks.js b/server/app/routes/v1.0/routes_tasks.js index 69b76cd2b..cb518b18e 100755 --- a/server/app/routes/v1.0/routes_tasks.js +++ b/server/app/routes/v1.0/routes_tasks.js @@ -542,11 +542,16 @@ module.exports.setRoutes = function(app, sessionVerification) { } if (updateCount) { if(taskData.isTaskScheduled === true){ - schedulerService.executeSchedulerForTasks(task,function(err,data){ - if(err){ - logger.error("Error in executing task scheduler"); + Tasks.getTaskById(req.params.taskId, function(err, scriptTask) { + if (err) { + logger.error(err); } - }) + schedulerService.executeSchedulerForTasks(scriptTask, function (err, data) { + if (err) { + logger.error("Error in executing task scheduler"); + } + }) + }); }; res.send({ updateCount: updateCount @@ -567,11 +572,16 @@ module.exports.setRoutes = function(app, sessionVerification) { } if (updateCount) { if(taskData.isTaskScheduled === true){ - schedulerService.executeSchedulerForTasks(task,function(err,data){ - if(err){ - logger.error("Error in executing task scheduler"); + Tasks.getTaskById(req.params.taskId, function(err, scriptTask) { + if (err) { + logger.error(err); } - }) + schedulerService.executeSchedulerForTasks(scriptTask, function (err, data) { + if (err) { + logger.error("Error in executing task scheduler"); + } + }) + }); }; res.send({ updateCount: updateCount diff --git a/server/app/services/auditTrailService.js b/server/app/services/auditTrailService.js index 6d0210da4..bca9cdbac 100644 --- a/server/app/services/auditTrailService.js +++ b/server/app/services/auditTrailService.js @@ -235,7 +235,7 @@ auditTrailService.getBOTsSummary = function getBOTsSummary(callback){ for(var i = 0; i < botAuditTrail.length; i++){ (function(auditTrail){ count++; - if(auditTrail.endedOn && auditTrail.endedOn !== null) { + if(auditTrail.endedOn && auditTrail.endedOn !== null && auditTrail.actionStatus !== 'failed') { var executionTime = getExecutionTime(auditTrail.endedOn, auditTrail.startedOn); totalTimeInSeconds = totalTimeInSeconds + (600 - executionTime); } From 74d8be0bf8484d09a25c3af1d202ee396e8174a0 Mon Sep 17 00:00:00 2001 From: Durgesh1988 Date: Mon, 28 Nov 2016 22:07:59 +0530 Subject: [PATCH 2/2] Task Scheduler Implementation --- .../catalyst-scheduler/catalystScheduler.js | 9 +++++-- .../app/routes/v1.0/routes_organizations.js | 15 +++-------- server/app/routes/v1.0/routes_tasks.js | 23 +++-------------- server/app/services/instanceService.js | 25 ++----------------- 4 files changed, 15 insertions(+), 57 deletions(-) diff --git a/server/app/cronjobs/catalyst-scheduler/catalystScheduler.js b/server/app/cronjobs/catalyst-scheduler/catalystScheduler.js index 33fb6260c..606410c1a 100644 --- a/server/app/cronjobs/catalyst-scheduler/catalystScheduler.js +++ b/server/app/cronjobs/catalyst-scheduler/catalystScheduler.js @@ -3,6 +3,7 @@ var instancesDao = require('_pr/model/classes/instance/instance'); var taskDao = require('_pr/model/classes/tasks/tasks.js'); var schedulerService = require('_pr/services/schedulerService'); var async = require('async'); +var cronTab = require('node-crontab'); var catalystSync = module.exports = {}; @@ -16,6 +17,9 @@ catalystSync.executeScheduledInstances = function executeScheduledInstances() { var resultList =[]; for (var i = 0; i < instances.length; i++) { (function(instance) { + if(instance.cronJobIds && instance.cronJobIds !== null){ + cronTab.cancelJobIds(instance.cronJobIds); + } resultList.push(function(callback){schedulerService.executeSchedulerForInstances(instance,callback);}); if(resultList.length === instances.length){ async.parallel(resultList,function(err,results){ @@ -46,6 +50,9 @@ catalystSync.executeScheduledTasks = function executeScheduledTasks() { var resultList =[],parallelTaskList=[],serialTaskList=[]; for (var i = 0; i < tasks.length; i++) { (function(task) { + if(task.cronJobId && task.cronJobId !== null){ + cronTab.cancelJob(task.cronJobId); + } if(task.executionOrder === 'PARALLEL'){ resultList.push(function(callback){schedulerService.executeSchedulerForTasks(task,callback);}); parallelTaskList.push(function(callback){schedulerService.executeSchedulerForTasks(task,callback);}); @@ -64,7 +71,6 @@ catalystSync.executeScheduledTasks = function executeScheduledTasks() { if(resultList.length === tasks.length){ async.parallel({ parallelTask: function(callback){ - console.log("Parallel"); async.parallel(parallelTaskList,function(err,data){ if(err){ callback(err,null); @@ -76,7 +82,6 @@ catalystSync.executeScheduledTasks = function executeScheduledTasks() { }) }, serialTask: function(callback){ - console.log("Serial"); async.waterfall(serialTaskList,function(err,data){ if(err){ callback(err,null); diff --git a/server/app/routes/v1.0/routes_organizations.js b/server/app/routes/v1.0/routes_organizations.js index f2d0c0aaf..21cc40403 100755 --- a/server/app/routes/v1.0/routes_organizations.js +++ b/server/app/routes/v1.0/routes_organizations.js @@ -49,10 +49,10 @@ var Docker = require('_pr/model/docker.js'); var orgValidator = require('_pr/validators/organizationValidator'); var validate = require('express-validation'); var taskService = require('_pr/services/taskService'); -var schedulerService = require('_pr/services/schedulerService'); var instanceLogModel = require('_pr/model/log-trail/instanceLog.js'); var compositeBlueprintModel = require('_pr/model/composite-blueprints/composite-blueprints.js'); var Cryptography = require('_pr/lib/utils/cryptography'); +var catalystSync = require('_pr/cronjobs/catalyst-scheduler/catalystScheduler.js'); module.exports.setRoutes = function(app, sessionVerification) { /* @@ -975,7 +975,6 @@ module.exports.setRoutes = function(app, sessionVerification) { taskData.orgName = project[0].orgname; taskData.bgName = project[0].productgroupname; taskData.projectName = project[0].projectname; - console.log(JSON.stringify(taskData)); if(taskData.taskScheduler && taskData.taskScheduler !== null) { taskData.taskScheduler = apiUtil.createCronJobPattern(taskData.taskScheduler); taskData.isTaskScheduled = true; @@ -1001,11 +1000,7 @@ module.exports.setRoutes = function(app, sessionVerification) { return; } if(task.isTaskScheduled === true){ - schedulerService.executeSchedulerForTasks(task,function(err,data){ - if(err){ - logger.error("Error in executing task scheduler"); - } - }) + catalystSync.executeScheduledTasks(); }; res.send(task); logger.debug("Exit post() for /organizations/%s/businessGroups/%s/projects/%s/environments/%s/tasks", req.params.orgId, req.params.bgId, req.params.projectId, req.params.environments); @@ -1020,11 +1015,7 @@ module.exports.setRoutes = function(app, sessionVerification) { return; } if(task.isTaskScheduled === true){ - schedulerService.executeSchedulerForTasks(task,function(err,data){ - if(err){ - logger.error("Error in executing task scheduler"); - } - }) + catalystSync.executeScheduledTasks(); }; res.send(task); logger.debug("Exit post() for /organizations/%s/businessGroups/%s/projects/%s/environments/%s/tasks", req.params.orgId, req.params.bgId, req.params.projectId, req.params.environments); diff --git a/server/app/routes/v1.0/routes_tasks.js b/server/app/routes/v1.0/routes_tasks.js index cb518b18e..e56bf0541 100755 --- a/server/app/routes/v1.0/routes_tasks.js +++ b/server/app/routes/v1.0/routes_tasks.js @@ -28,6 +28,7 @@ var async = require('async'); var apiUtil = require('_pr/lib/utils/apiUtil.js'); var Cryptography = require('_pr/lib/utils/cryptography'); var schedulerService = require('_pr/services/schedulerService'); +var catalystSync = require('_pr/cronjobs/catalyst-scheduler/catalystScheduler.js'); @@ -542,16 +543,7 @@ module.exports.setRoutes = function(app, sessionVerification) { } if (updateCount) { if(taskData.isTaskScheduled === true){ - Tasks.getTaskById(req.params.taskId, function(err, scriptTask) { - if (err) { - logger.error(err); - } - schedulerService.executeSchedulerForTasks(scriptTask, function (err, data) { - if (err) { - logger.error("Error in executing task scheduler"); - } - }) - }); + catalystSync.executeScheduledTasks(); }; res.send({ updateCount: updateCount @@ -572,16 +564,7 @@ module.exports.setRoutes = function(app, sessionVerification) { } if (updateCount) { if(taskData.isTaskScheduled === true){ - Tasks.getTaskById(req.params.taskId, function(err, scriptTask) { - if (err) { - logger.error(err); - } - schedulerService.executeSchedulerForTasks(scriptTask, function (err, data) { - if (err) { - logger.error("Error in executing task scheduler"); - } - }) - }); + catalystSync.executeScheduledTasks(); }; res.send({ updateCount: updateCount diff --git a/server/app/services/instanceService.js b/server/app/services/instanceService.js index fe7a59a4d..ed4555535 100644 --- a/server/app/services/instanceService.js +++ b/server/app/services/instanceService.js @@ -46,7 +46,7 @@ var AppData = require('_pr/model/app-deploy/app-data'); var instancesDao = require('_pr/model/classes/instance/instance'); var providerService = require('_pr/services/providerService.js'); var schedulerService = require('_pr/services/schedulerService.js'); -var crontab = require('node-crontab'); +var catalystSync = require('_pr/cronjobs/catalyst-scheduler/catalystScheduler.js'); var instanceService = module.exports = {}; @@ -1473,34 +1473,13 @@ function updateScheduler(instanceScheduler, callback) { function(schedulerDetails,next){ schedulerDetails.interval = instanceScheduler.interval; instancesDao.updateScheduler(instanceScheduler.instanceIds, schedulerDetails,next); - }, - function(updateSchedulerDetails,next){ - instancesDao.getInstances(instanceScheduler.instanceIds,next); } ],function(err,results){ if(err){ return callback(err, null); }else{ callback(null, {"message": "Scheduler Updated."}); - var resultList =[]; - for (var i = 0; i < results.length; i++) { - (function(instance) { - if(instance.cronJobIds && instance.cronJobIds.length > 0){ - crontab.cancelJobIds(instance.cronJobIds); - } - resultList.push(function(callback){schedulerService.executeSchedulerForInstances(instance,callback);}); - if(resultList.length === results.length){ - async.parallel(resultList,function(err,results){ - if(err){ - logger.error(err); - return; - } - logger.debug("Instance Scheduler Results>>"+results); - return; - }) - } - })(results[i]); - } + catalystSync.executeScheduledInstances(); return; } });