From 6b9bcfc9b546ef52a10bbad08fda95949e33da40 Mon Sep 17 00:00:00 2001 From: Karthik K N Date: Thu, 1 Dec 2016 14:28:06 +0530 Subject: [PATCH 01/14] Fix for CAT-2445 Incorrect Total Cost is showing in Analytics tab --- .../model/resource-costs/resource-costs.js | 43 +++-- server/app/services/resourceService.js | 161 ++++++++++-------- 2 files changed, 124 insertions(+), 80 deletions(-) diff --git a/server/app/model/resource-costs/resource-costs.js b/server/app/model/resource-costs/resource-costs.js index 012c774f6..28a47e042 100644 --- a/server/app/model/resource-costs/resource-costs.js +++ b/server/app/model/resource-costs/resource-costs.js @@ -70,6 +70,11 @@ var ResourceCostsSchema = new Schema({ required: false, trim: true }, + billIntervalId: { + type: String, + required: true, + trim: true + }, billLineItemId: { type: Number, required: true, @@ -132,38 +137,56 @@ var ResourceCostsSchema = new Schema({ }) ResourceCostsSchema.index({'platformDetails.serviceId' : 1}) -ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'billLineItemId': 1, - 'startTime': 1, 'interval': 1}, {'unique': true}) +ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'startTime': 1, + 'billIntervalId': 1, 'billLineItemId': 1, 'interval': 1}, {'unique': true}) -ResourceCostsSchema.statics.saveResourceCost = function saveResourceCost(resourceCostData, callback) { +ResourceCostsSchema.statics.save = function save(resourceCostData, callback) { var resourceCosts = new ResourceCosts(resourceCostData) - resourceCosts.save(function(err, data) { + resourceCosts.save(function(err, result) { + if (err) { + callback(err) + } else { + callback(null, result) + } + }) +} + +ResourceCostsSchema.statics.remove + = function remove(organizationId, providerId, billIntervalId, callback) { + var query = { + organizationId: organizationId, + providerId: providerId, + billIntervalId: billIntervalId + } + + this.find(query).remove(function(err, result) { if (err) { callback(err) } else { - callback(null) + callback(null, result) } }) } -ResourceCostsSchema.statics.upsertResourceCost = function upsertResourceCost(resourceCostData, callback) { +ResourceCostsSchema.statics.upsert = function upsert(resourceCostData, callback) { var query = { organizationId: resourceCostData.organizationId, providerId: resourceCostData.providerId, - billLineItemId: resourceCostData.billLineItemId, startTime: resourceCostData.startTime, + billIntervalId: resourceCostData.billIntervalId, + billLineItemId: resourceCostData.billLineItemId, interval: resourceCostData.interval } - this.findOneAndUpdate(query, resourceCostData, {upsert:true}, + this.findOneAndUpdate(query, resourceCostData, {upsert: true}, function(err, result){ if (err) { callback(null) } else { callback(null, result) } - }); + }) } var ResourceCosts = mongoose.model('ResourceCosts', ResourceCostsSchema) -module.exports = ResourceCosts +module.exports = ResourceCosts \ No newline at end of file diff --git a/server/app/services/resourceService.js b/server/app/services/resourceService.js index f56a7f95c..deb4f322d 100644 --- a/server/app/services/resourceService.js +++ b/server/app/services/resourceService.js @@ -91,91 +91,112 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u var awsZones = appConfig.aws.zones var lineNumber = 0 - var stream = fs.createReadStream(downlaodedCSVPath) - csv.fromStream(stream, {headers: false}).on('data', function(data) { - if(data[awsBillIndexes.totalCost] == 'LineItem') { - var resourceCostEntry = {platformDetails: {}} - - resourceCostEntry.organizationId = provider.orgId - resourceCostEntry.providerId = provider._id - resourceCostEntry.providerType = provider.providerType - resourceCostEntry.cost = data[awsBillIndexes.cost] - resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate]) - resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate]) - resourceCostEntry.lastUpdateTime = Date.parse(updateTime) - resourceCostEntry.interval = 3600 - resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod] - resourceCostEntry.billLineItemId = ++lineNumber - resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - - resourceCostEntry.platformDetails.serviceId - = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] - :resourceCostEntry.platformDetails.serviceId = 'Other' - - resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) - ? 'Global' : data[awsBillIndexes.zone] - - resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) - ? awsZones[data[awsBillIndexes.zone]] : 'Global' - - if (data[awsBillIndexes.instanceId] != null) { - resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] - } + var date = new Date() + var billIntervalId = date.getFullYear() + '-' + date.getMonth() - if(data[awsBillIndexes.usageType] != null) { - resourceCostEntry.platformDetails.usageType = data[awsBillIndexes.usageType] - } + async.waterfall([ + function(next) { + resourceCost.remove(provider.orgId, provider._id, billIntervalId, next) + }, + function(count, next) { + /*var lineNumber = (count == 0)?0:count + var startingLineNumber = (count == 0)?1:(count+2)*/ - if (data[awsBillIndexes.instanceId] in resources) { - var resource = resources[data[awsBillIndexes.instanceId]] + var stream = fs.createReadStream(downlaodedCSVPath) + csv.fromStream(stream).on('data', function(data) { + if(data[awsBillIndexes.totalCost] == 'LineItem') { + var resourceCostEntry = {platformDetails: {}} - resourceCostEntry.resourceId = resource._id + resourceCostEntry.organizationId = provider.orgId + resourceCostEntry.providerId = provider._id + resourceCostEntry.providerType = provider.providerType + resourceCostEntry.cost = data[awsBillIndexes.cost] + resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate]) + resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate]) + resourceCostEntry.lastUpdateTime = Date.parse(updateTime) + resourceCostEntry.interval = 3600 + resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod] + resourceCostEntry.billIntervalId = billIntervalId + resourceCostEntry.billLineItemId = ++lineNumber + resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - if (('bgId' in resource) && (resource.bgId != null)) { - resourceCostEntry.businessGroupId = resource['bgId'] - } + resourceCostEntry.platformDetails.serviceId + = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] + :resourceCostEntry.platformDetails.serviceId = 'Other' - if (('projectId' in resource) && (resource.projectId != null)) { - resourceCostEntry.projectId = resource['projectId'] - } + resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) + ? 'Global' : data[awsBillIndexes.zone] - if (('environmentId' in resource) && (resource.environmentId != null)) { - resourceCostEntry.environmentId = resource['environmentId'] - } + resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) + ? awsZones[data[awsBillIndexes.zone]] : 'Global' - if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { - console.log("BG: " + resource['bgId']) - resourceCostEntry.businessGroupId = resource['bgId'] - } + if (data[awsBillIndexes.instanceId] != null) { + resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] + } - if (('masterDetails.projectId' in resource) - && (resource.masterDetails.projectId != null)) { - resourceCostEntry.projectId = resource['projectId'] - } + if(data[awsBillIndexes.usageType] != null) { + resourceCostEntry.platformDetails.usageType = data[awsBillIndexes.usageType] + } - if (('masterDetails.environmentId' in resource) - && (resource.masterDetails.environmentId != null)) { - resourceCostEntry.environmentId = resource['environmentId'] - } - } + if (data[awsBillIndexes.instanceId] in resources) { + var resource = resources[data[awsBillIndexes.instanceId]] - resourceCostEntry.businessGroupId - = ('businessGroupId' in resourceCostEntry)?resourceCostEntry.businessGroupId:'Unassigned' - resourceCostEntry.projectId - = ('projectId' in resourceCostEntry)?resourceCostEntry.projectId:'Unassigned' - resourceCostEntry.environmentId - = ('environmentId' in resourceCostEntry)?resourceCostEntry.environmentId:'Unassigned' + resourceCostEntry.resourceId = resource._id - resourceCost.upsertResourceCost(resourceCostEntry, function (err, costEntry) { - if (err) { - logger.error(err) - return callback(new Error('Database Error')) + if (('bgId' in resource) && (resource.bgId != null)) { + resourceCostEntry.businessGroupId = resource['bgId'] + } + + if (('projectId' in resource) && (resource.projectId != null)) { + resourceCostEntry.projectId = resource['projectId'] + } + + if (('environmentId' in resource) && (resource.environmentId != null)) { + resourceCostEntry.environmentId = resource['environmentId'] + } + + if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { + console.log("BG: " + resource['bgId']) + resourceCostEntry.businessGroupId = resource['bgId'] + } + + if (('masterDetails.projectId' in resource) + && (resource.masterDetails.projectId != null)) { + resourceCostEntry.projectId = resource['projectId'] + } + + if (('masterDetails.environmentId' in resource) + && (resource.masterDetails.environmentId != null)) { + resourceCostEntry.environmentId = resource['environmentId'] + } + } + + resourceCostEntry.businessGroupId + = ('businessGroupId' in resourceCostEntry)?resourceCostEntry.businessGroupId:'Unassigned' + resourceCostEntry.projectId + = ('projectId' in resourceCostEntry)?resourceCostEntry.projectId:'Unassigned' + resourceCostEntry.environmentId + = ('environmentId' in resourceCostEntry)?resourceCostEntry.environmentId:'Unassigned' + + resourceCost.save(resourceCostEntry, function (err, costEntry) { + if (err) { + logger.error(err) + return next(new Error('Database Error')) + } + }) } + }).on('end', function() { + next() }) } - }).on('end', function() { - callback(null) + ], function(err) { + if(err) { + callback(err) + } else { + callback() + } }) + } function getCostForServices_deprecated(provider,callback) { From 1b2fcfc0845b75d35b3608fe869f701328ff499a Mon Sep 17 00:00:00 2001 From: Karthik K N Date: Thu, 1 Dec 2016 14:28:06 +0530 Subject: [PATCH 02/14] Fix for CAT-2445 Incorrect Total Cost is showing in Analytics tab --- .../model/resource-costs/resource-costs.js | 43 +++-- server/app/services/resourceService.js | 161 ++++++++++-------- 2 files changed, 124 insertions(+), 80 deletions(-) diff --git a/server/app/model/resource-costs/resource-costs.js b/server/app/model/resource-costs/resource-costs.js index 012c774f6..28a47e042 100644 --- a/server/app/model/resource-costs/resource-costs.js +++ b/server/app/model/resource-costs/resource-costs.js @@ -70,6 +70,11 @@ var ResourceCostsSchema = new Schema({ required: false, trim: true }, + billIntervalId: { + type: String, + required: true, + trim: true + }, billLineItemId: { type: Number, required: true, @@ -132,38 +137,56 @@ var ResourceCostsSchema = new Schema({ }) ResourceCostsSchema.index({'platformDetails.serviceId' : 1}) -ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'billLineItemId': 1, - 'startTime': 1, 'interval': 1}, {'unique': true}) +ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'startTime': 1, + 'billIntervalId': 1, 'billLineItemId': 1, 'interval': 1}, {'unique': true}) -ResourceCostsSchema.statics.saveResourceCost = function saveResourceCost(resourceCostData, callback) { +ResourceCostsSchema.statics.save = function save(resourceCostData, callback) { var resourceCosts = new ResourceCosts(resourceCostData) - resourceCosts.save(function(err, data) { + resourceCosts.save(function(err, result) { + if (err) { + callback(err) + } else { + callback(null, result) + } + }) +} + +ResourceCostsSchema.statics.remove + = function remove(organizationId, providerId, billIntervalId, callback) { + var query = { + organizationId: organizationId, + providerId: providerId, + billIntervalId: billIntervalId + } + + this.find(query).remove(function(err, result) { if (err) { callback(err) } else { - callback(null) + callback(null, result) } }) } -ResourceCostsSchema.statics.upsertResourceCost = function upsertResourceCost(resourceCostData, callback) { +ResourceCostsSchema.statics.upsert = function upsert(resourceCostData, callback) { var query = { organizationId: resourceCostData.organizationId, providerId: resourceCostData.providerId, - billLineItemId: resourceCostData.billLineItemId, startTime: resourceCostData.startTime, + billIntervalId: resourceCostData.billIntervalId, + billLineItemId: resourceCostData.billLineItemId, interval: resourceCostData.interval } - this.findOneAndUpdate(query, resourceCostData, {upsert:true}, + this.findOneAndUpdate(query, resourceCostData, {upsert: true}, function(err, result){ if (err) { callback(null) } else { callback(null, result) } - }); + }) } var ResourceCosts = mongoose.model('ResourceCosts', ResourceCostsSchema) -module.exports = ResourceCosts +module.exports = ResourceCosts \ No newline at end of file diff --git a/server/app/services/resourceService.js b/server/app/services/resourceService.js index f56a7f95c..deb4f322d 100644 --- a/server/app/services/resourceService.js +++ b/server/app/services/resourceService.js @@ -91,91 +91,112 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u var awsZones = appConfig.aws.zones var lineNumber = 0 - var stream = fs.createReadStream(downlaodedCSVPath) - csv.fromStream(stream, {headers: false}).on('data', function(data) { - if(data[awsBillIndexes.totalCost] == 'LineItem') { - var resourceCostEntry = {platformDetails: {}} - - resourceCostEntry.organizationId = provider.orgId - resourceCostEntry.providerId = provider._id - resourceCostEntry.providerType = provider.providerType - resourceCostEntry.cost = data[awsBillIndexes.cost] - resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate]) - resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate]) - resourceCostEntry.lastUpdateTime = Date.parse(updateTime) - resourceCostEntry.interval = 3600 - resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod] - resourceCostEntry.billLineItemId = ++lineNumber - resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - - resourceCostEntry.platformDetails.serviceId - = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] - :resourceCostEntry.platformDetails.serviceId = 'Other' - - resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) - ? 'Global' : data[awsBillIndexes.zone] - - resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) - ? awsZones[data[awsBillIndexes.zone]] : 'Global' - - if (data[awsBillIndexes.instanceId] != null) { - resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] - } + var date = new Date() + var billIntervalId = date.getFullYear() + '-' + date.getMonth() - if(data[awsBillIndexes.usageType] != null) { - resourceCostEntry.platformDetails.usageType = data[awsBillIndexes.usageType] - } + async.waterfall([ + function(next) { + resourceCost.remove(provider.orgId, provider._id, billIntervalId, next) + }, + function(count, next) { + /*var lineNumber = (count == 0)?0:count + var startingLineNumber = (count == 0)?1:(count+2)*/ - if (data[awsBillIndexes.instanceId] in resources) { - var resource = resources[data[awsBillIndexes.instanceId]] + var stream = fs.createReadStream(downlaodedCSVPath) + csv.fromStream(stream).on('data', function(data) { + if(data[awsBillIndexes.totalCost] == 'LineItem') { + var resourceCostEntry = {platformDetails: {}} - resourceCostEntry.resourceId = resource._id + resourceCostEntry.organizationId = provider.orgId + resourceCostEntry.providerId = provider._id + resourceCostEntry.providerType = provider.providerType + resourceCostEntry.cost = data[awsBillIndexes.cost] + resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate]) + resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate]) + resourceCostEntry.lastUpdateTime = Date.parse(updateTime) + resourceCostEntry.interval = 3600 + resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod] + resourceCostEntry.billIntervalId = billIntervalId + resourceCostEntry.billLineItemId = ++lineNumber + resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - if (('bgId' in resource) && (resource.bgId != null)) { - resourceCostEntry.businessGroupId = resource['bgId'] - } + resourceCostEntry.platformDetails.serviceId + = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] + :resourceCostEntry.platformDetails.serviceId = 'Other' - if (('projectId' in resource) && (resource.projectId != null)) { - resourceCostEntry.projectId = resource['projectId'] - } + resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) + ? 'Global' : data[awsBillIndexes.zone] - if (('environmentId' in resource) && (resource.environmentId != null)) { - resourceCostEntry.environmentId = resource['environmentId'] - } + resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) + ? awsZones[data[awsBillIndexes.zone]] : 'Global' - if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { - console.log("BG: " + resource['bgId']) - resourceCostEntry.businessGroupId = resource['bgId'] - } + if (data[awsBillIndexes.instanceId] != null) { + resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] + } - if (('masterDetails.projectId' in resource) - && (resource.masterDetails.projectId != null)) { - resourceCostEntry.projectId = resource['projectId'] - } + if(data[awsBillIndexes.usageType] != null) { + resourceCostEntry.platformDetails.usageType = data[awsBillIndexes.usageType] + } - if (('masterDetails.environmentId' in resource) - && (resource.masterDetails.environmentId != null)) { - resourceCostEntry.environmentId = resource['environmentId'] - } - } + if (data[awsBillIndexes.instanceId] in resources) { + var resource = resources[data[awsBillIndexes.instanceId]] - resourceCostEntry.businessGroupId - = ('businessGroupId' in resourceCostEntry)?resourceCostEntry.businessGroupId:'Unassigned' - resourceCostEntry.projectId - = ('projectId' in resourceCostEntry)?resourceCostEntry.projectId:'Unassigned' - resourceCostEntry.environmentId - = ('environmentId' in resourceCostEntry)?resourceCostEntry.environmentId:'Unassigned' + resourceCostEntry.resourceId = resource._id - resourceCost.upsertResourceCost(resourceCostEntry, function (err, costEntry) { - if (err) { - logger.error(err) - return callback(new Error('Database Error')) + if (('bgId' in resource) && (resource.bgId != null)) { + resourceCostEntry.businessGroupId = resource['bgId'] + } + + if (('projectId' in resource) && (resource.projectId != null)) { + resourceCostEntry.projectId = resource['projectId'] + } + + if (('environmentId' in resource) && (resource.environmentId != null)) { + resourceCostEntry.environmentId = resource['environmentId'] + } + + if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { + console.log("BG: " + resource['bgId']) + resourceCostEntry.businessGroupId = resource['bgId'] + } + + if (('masterDetails.projectId' in resource) + && (resource.masterDetails.projectId != null)) { + resourceCostEntry.projectId = resource['projectId'] + } + + if (('masterDetails.environmentId' in resource) + && (resource.masterDetails.environmentId != null)) { + resourceCostEntry.environmentId = resource['environmentId'] + } + } + + resourceCostEntry.businessGroupId + = ('businessGroupId' in resourceCostEntry)?resourceCostEntry.businessGroupId:'Unassigned' + resourceCostEntry.projectId + = ('projectId' in resourceCostEntry)?resourceCostEntry.projectId:'Unassigned' + resourceCostEntry.environmentId + = ('environmentId' in resourceCostEntry)?resourceCostEntry.environmentId:'Unassigned' + + resourceCost.save(resourceCostEntry, function (err, costEntry) { + if (err) { + logger.error(err) + return next(new Error('Database Error')) + } + }) } + }).on('end', function() { + next() }) } - }).on('end', function() { - callback(null) + ], function(err) { + if(err) { + callback(err) + } else { + callback() + } }) + } function getCostForServices_deprecated(provider,callback) { From 0bb74d01a7cb9efa1d2fdd0b66b83663ae674ea0 Mon Sep 17 00:00:00 2001 From: srikanthv02 Date: Thu, 1 Dec 2016 17:42:47 +0530 Subject: [PATCH 03/14] Check for undefined and null --- .../workzone/orchestration/popups/addChefJobEventCtrl.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEventCtrl.js b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEventCtrl.js index 189e99aa7..187ec039b 100644 --- a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEventCtrl.js +++ b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEventCtrl.js @@ -16,7 +16,7 @@ $scope.schedulerEndOn=moment(new Date()).format('MM/DD/YYYY'); }; if(items.type !== 'new'){ - if(items.chefJenkScriptTaskObj !==undefined){ + if(items.chefJenkScriptTaskObj !=null){ if(items.chefJenkScriptTaskObj.cronStartOn && items.chefJenkScriptTaskObj.cronEndOn) { var newStartOn = parseInt(items.chefJenkScriptTaskObj.cronStartOn); var newDate = new Date(newStartOn).toLocaleDateString(); From c6d67c730ae811cf3eebde7bad8e41c783ed47ca Mon Sep 17 00:00:00 2001 From: srikanthv02 Date: Thu, 1 Dec 2016 17:49:43 +0530 Subject: [PATCH 04/14] Review comments taken care --- .../workzone/orchestration/popups/addChefJobEventCtrl.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEventCtrl.js b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEventCtrl.js index 187ec039b..9a3a24b2c 100644 --- a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEventCtrl.js +++ b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEventCtrl.js @@ -16,7 +16,7 @@ $scope.schedulerEndOn=moment(new Date()).format('MM/DD/YYYY'); }; if(items.type !== 'new'){ - if(items.chefJenkScriptTaskObj !=null){ + if(items.chefJenkScriptTaskObj){ if(items.chefJenkScriptTaskObj.cronStartOn && items.chefJenkScriptTaskObj.cronEndOn) { var newStartOn = parseInt(items.chefJenkScriptTaskObj.cronStartOn); var newDate = new Date(newStartOn).toLocaleDateString(); From 5ab8fa933a079e393e86a6af113e42f7b5f28c02 Mon Sep 17 00:00:00 2001 From: Karthik K N Date: Thu, 1 Dec 2016 14:28:06 +0530 Subject: [PATCH 05/14] Fix for CAT-2445 Incorrect Total Cost is showing in Analytics tab --- .../model/resource-costs/resource-costs.js | 43 +++-- server/app/services/resourceService.js | 161 ++++++++++-------- 2 files changed, 124 insertions(+), 80 deletions(-) diff --git a/server/app/model/resource-costs/resource-costs.js b/server/app/model/resource-costs/resource-costs.js index 012c774f6..28a47e042 100644 --- a/server/app/model/resource-costs/resource-costs.js +++ b/server/app/model/resource-costs/resource-costs.js @@ -70,6 +70,11 @@ var ResourceCostsSchema = new Schema({ required: false, trim: true }, + billIntervalId: { + type: String, + required: true, + trim: true + }, billLineItemId: { type: Number, required: true, @@ -132,38 +137,56 @@ var ResourceCostsSchema = new Schema({ }) ResourceCostsSchema.index({'platformDetails.serviceId' : 1}) -ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'billLineItemId': 1, - 'startTime': 1, 'interval': 1}, {'unique': true}) +ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'startTime': 1, + 'billIntervalId': 1, 'billLineItemId': 1, 'interval': 1}, {'unique': true}) -ResourceCostsSchema.statics.saveResourceCost = function saveResourceCost(resourceCostData, callback) { +ResourceCostsSchema.statics.save = function save(resourceCostData, callback) { var resourceCosts = new ResourceCosts(resourceCostData) - resourceCosts.save(function(err, data) { + resourceCosts.save(function(err, result) { + if (err) { + callback(err) + } else { + callback(null, result) + } + }) +} + +ResourceCostsSchema.statics.remove + = function remove(organizationId, providerId, billIntervalId, callback) { + var query = { + organizationId: organizationId, + providerId: providerId, + billIntervalId: billIntervalId + } + + this.find(query).remove(function(err, result) { if (err) { callback(err) } else { - callback(null) + callback(null, result) } }) } -ResourceCostsSchema.statics.upsertResourceCost = function upsertResourceCost(resourceCostData, callback) { +ResourceCostsSchema.statics.upsert = function upsert(resourceCostData, callback) { var query = { organizationId: resourceCostData.organizationId, providerId: resourceCostData.providerId, - billLineItemId: resourceCostData.billLineItemId, startTime: resourceCostData.startTime, + billIntervalId: resourceCostData.billIntervalId, + billLineItemId: resourceCostData.billLineItemId, interval: resourceCostData.interval } - this.findOneAndUpdate(query, resourceCostData, {upsert:true}, + this.findOneAndUpdate(query, resourceCostData, {upsert: true}, function(err, result){ if (err) { callback(null) } else { callback(null, result) } - }); + }) } var ResourceCosts = mongoose.model('ResourceCosts', ResourceCostsSchema) -module.exports = ResourceCosts +module.exports = ResourceCosts \ No newline at end of file diff --git a/server/app/services/resourceService.js b/server/app/services/resourceService.js index f56a7f95c..deb4f322d 100644 --- a/server/app/services/resourceService.js +++ b/server/app/services/resourceService.js @@ -91,91 +91,112 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u var awsZones = appConfig.aws.zones var lineNumber = 0 - var stream = fs.createReadStream(downlaodedCSVPath) - csv.fromStream(stream, {headers: false}).on('data', function(data) { - if(data[awsBillIndexes.totalCost] == 'LineItem') { - var resourceCostEntry = {platformDetails: {}} - - resourceCostEntry.organizationId = provider.orgId - resourceCostEntry.providerId = provider._id - resourceCostEntry.providerType = provider.providerType - resourceCostEntry.cost = data[awsBillIndexes.cost] - resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate]) - resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate]) - resourceCostEntry.lastUpdateTime = Date.parse(updateTime) - resourceCostEntry.interval = 3600 - resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod] - resourceCostEntry.billLineItemId = ++lineNumber - resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - - resourceCostEntry.platformDetails.serviceId - = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] - :resourceCostEntry.platformDetails.serviceId = 'Other' - - resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) - ? 'Global' : data[awsBillIndexes.zone] - - resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) - ? awsZones[data[awsBillIndexes.zone]] : 'Global' - - if (data[awsBillIndexes.instanceId] != null) { - resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] - } + var date = new Date() + var billIntervalId = date.getFullYear() + '-' + date.getMonth() - if(data[awsBillIndexes.usageType] != null) { - resourceCostEntry.platformDetails.usageType = data[awsBillIndexes.usageType] - } + async.waterfall([ + function(next) { + resourceCost.remove(provider.orgId, provider._id, billIntervalId, next) + }, + function(count, next) { + /*var lineNumber = (count == 0)?0:count + var startingLineNumber = (count == 0)?1:(count+2)*/ - if (data[awsBillIndexes.instanceId] in resources) { - var resource = resources[data[awsBillIndexes.instanceId]] + var stream = fs.createReadStream(downlaodedCSVPath) + csv.fromStream(stream).on('data', function(data) { + if(data[awsBillIndexes.totalCost] == 'LineItem') { + var resourceCostEntry = {platformDetails: {}} - resourceCostEntry.resourceId = resource._id + resourceCostEntry.organizationId = provider.orgId + resourceCostEntry.providerId = provider._id + resourceCostEntry.providerType = provider.providerType + resourceCostEntry.cost = data[awsBillIndexes.cost] + resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate]) + resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate]) + resourceCostEntry.lastUpdateTime = Date.parse(updateTime) + resourceCostEntry.interval = 3600 + resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod] + resourceCostEntry.billIntervalId = billIntervalId + resourceCostEntry.billLineItemId = ++lineNumber + resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - if (('bgId' in resource) && (resource.bgId != null)) { - resourceCostEntry.businessGroupId = resource['bgId'] - } + resourceCostEntry.platformDetails.serviceId + = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] + :resourceCostEntry.platformDetails.serviceId = 'Other' - if (('projectId' in resource) && (resource.projectId != null)) { - resourceCostEntry.projectId = resource['projectId'] - } + resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) + ? 'Global' : data[awsBillIndexes.zone] - if (('environmentId' in resource) && (resource.environmentId != null)) { - resourceCostEntry.environmentId = resource['environmentId'] - } + resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) + ? awsZones[data[awsBillIndexes.zone]] : 'Global' - if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { - console.log("BG: " + resource['bgId']) - resourceCostEntry.businessGroupId = resource['bgId'] - } + if (data[awsBillIndexes.instanceId] != null) { + resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] + } - if (('masterDetails.projectId' in resource) - && (resource.masterDetails.projectId != null)) { - resourceCostEntry.projectId = resource['projectId'] - } + if(data[awsBillIndexes.usageType] != null) { + resourceCostEntry.platformDetails.usageType = data[awsBillIndexes.usageType] + } - if (('masterDetails.environmentId' in resource) - && (resource.masterDetails.environmentId != null)) { - resourceCostEntry.environmentId = resource['environmentId'] - } - } + if (data[awsBillIndexes.instanceId] in resources) { + var resource = resources[data[awsBillIndexes.instanceId]] - resourceCostEntry.businessGroupId - = ('businessGroupId' in resourceCostEntry)?resourceCostEntry.businessGroupId:'Unassigned' - resourceCostEntry.projectId - = ('projectId' in resourceCostEntry)?resourceCostEntry.projectId:'Unassigned' - resourceCostEntry.environmentId - = ('environmentId' in resourceCostEntry)?resourceCostEntry.environmentId:'Unassigned' + resourceCostEntry.resourceId = resource._id - resourceCost.upsertResourceCost(resourceCostEntry, function (err, costEntry) { - if (err) { - logger.error(err) - return callback(new Error('Database Error')) + if (('bgId' in resource) && (resource.bgId != null)) { + resourceCostEntry.businessGroupId = resource['bgId'] + } + + if (('projectId' in resource) && (resource.projectId != null)) { + resourceCostEntry.projectId = resource['projectId'] + } + + if (('environmentId' in resource) && (resource.environmentId != null)) { + resourceCostEntry.environmentId = resource['environmentId'] + } + + if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { + console.log("BG: " + resource['bgId']) + resourceCostEntry.businessGroupId = resource['bgId'] + } + + if (('masterDetails.projectId' in resource) + && (resource.masterDetails.projectId != null)) { + resourceCostEntry.projectId = resource['projectId'] + } + + if (('masterDetails.environmentId' in resource) + && (resource.masterDetails.environmentId != null)) { + resourceCostEntry.environmentId = resource['environmentId'] + } + } + + resourceCostEntry.businessGroupId + = ('businessGroupId' in resourceCostEntry)?resourceCostEntry.businessGroupId:'Unassigned' + resourceCostEntry.projectId + = ('projectId' in resourceCostEntry)?resourceCostEntry.projectId:'Unassigned' + resourceCostEntry.environmentId + = ('environmentId' in resourceCostEntry)?resourceCostEntry.environmentId:'Unassigned' + + resourceCost.save(resourceCostEntry, function (err, costEntry) { + if (err) { + logger.error(err) + return next(new Error('Database Error')) + } + }) } + }).on('end', function() { + next() }) } - }).on('end', function() { - callback(null) + ], function(err) { + if(err) { + callback(err) + } else { + callback() + } }) + } function getCostForServices_deprecated(provider,callback) { From d93e2963dd9eb79feade2f256fc9d88ad8202975 Mon Sep 17 00:00:00 2001 From: Durgesh1988 Date: Fri, 2 Dec 2016 17:12:51 +0530 Subject: [PATCH 06/14] Script Executor for Python and bots library Issue --- .../orchestration/popups/addChefJobEvent.html | 9 ++- .../popups/addChefJobEventCtrl.js | 26 ++++-- .../orchestration/popups/addScriptParams.html | 10 ++- .../popups/addScriptParamsCtrl.js | 11 ++- .../orchestration/popups/newTask.html | 11 ++- .../orchestration/popups/newTaskCtrl.js | 16 +++- .../private/ajax/Settings/scriptList.html | 37 ++++++++- client/htmls/private/js/scriptList.js | 31 ++++++- .../catalyst-scheduler/catalystScheduler.js | 2 - server/app/lib/utils/apiUtil.js | 65 +++++++-------- server/app/model/blueprint/blueprint.js | 27 +++++++ .../app/model/classes/tasks/taskTypeScript.js | 81 +++++++++++-------- server/app/model/classes/tasks/tasks.js | 57 ++++++++++--- server/app/model/scripts/scripts.js | 14 +++- server/app/routes/v1.0/routes_blueprints.js | 11 +++ .../app/routes/v1.0/routes_organizations.js | 26 +++--- server/app/routes/v1.0/routes_tasks.js | 21 ++--- server/app/services/instanceService.js | 22 +++-- server/app/services/schedulerService.js | 3 +- server/app/services/scriptService.js | 34 ++++---- server/app/services/taskService.js | 11 +++ 21 files changed, 369 insertions(+), 156 deletions(-) diff --git a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEvent.html b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEvent.html index 6cdbf04b0..d454fa2d5 100644 --- a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEvent.html +++ b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEvent.html @@ -23,12 +23,13 @@

 Schedule your Task Interval
- + - +
@@ -57,7 +58,7 @@

 Schedule your Task --> -
+
-
+
- Add More +
diff --git a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addScriptParamsCtrl.js b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addScriptParamsCtrl.js index 68408183f..4da8650e2 100644 --- a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addScriptParamsCtrl.js +++ b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addScriptParamsCtrl.js @@ -8,11 +8,14 @@ (function(angular){ "use strict"; angular.module('workzone.orchestration') - .controller('addScriptParamsCtrl',['$scope', '$modalInstance', 'toastr',function($scope, $modalInstance, toastr){ + .controller('addScriptParamsCtrl',['$scope', '$modalInstance','items', 'toastr',function($scope, $modalInstance,$items, toastr){ //default selection type - $scope.params=['']; + $scope.params=[]; + for(var i =0; i < $items.noOfParams; i++){ + $scope.params.push({}); + } $scope.add = function() { - $scope.params.push(''); + $scope.params.push({}); }; $scope.removeScriptInputParams = function(paramInput) { if($scope.params.length > 1){ @@ -28,7 +31,7 @@ $scope.ok=function(){ var checkParam = false; for(var i =0; i<$scope.params.length; i++){ - if($scope.params[i] === '' || $scope.params[i] === null){ + if(Object.keys($scope.params[i]).length === 0){ checkParam = false; toastr.error('Please enter parameters'); return false; diff --git a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTask.html b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTask.html index 04c648cd4..e95231bc1 100644 --- a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTask.html +++ b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTask.html @@ -153,6 +153,7 @@

Job Details
@@ -540,8 +541,8 @@

Script Details
@@ -561,13 +562,15 @@
Script Details
Parameter + Description Action - {{param}} - ..... + {{param.paramVal}} + ..... + {{param.paramDesc}} diff --git a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTaskCtrl.js b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTaskCtrl.js index 0c6e70413..b370ef5d6 100644 --- a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTaskCtrl.js +++ b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTaskCtrl.js @@ -152,7 +152,12 @@ templateUrl: 'src/partials/sections/dashboard/workzone/orchestration/popups/addScriptParams.html', controller: 'addScriptParamsCtrl', backdrop: 'static', - keyboard: false + keyboard: false, + resolve: { + items: function () { + return scriptObject; + } + } }).result.then(function (addScriptParams) { $scope.scriptParamsObj[scriptObject._id] = $scope.scriptParamsObj[scriptObject._id].concat(addScriptParams); }, function () { @@ -271,9 +276,12 @@ cronEndOn : $scope.chefJenkScriptTaskObj.cronEnd, cronRepeatEvery : $scope.chefJenkScriptTaskObj.repeatBy, cronFrequency: $scope.chefJenkScriptTaskObj.repeats, - cronTime: typeof startTimeHour !=='undefined'? startTimeHour : new Date().getHours() + ':' + typeof startTimeMinute !=='undefined'? startTimeMinute:new Date().getMinutes(), - cronDays: $scope.chefJenkScriptTaskObj.dayOfWeek, - cronMonth: $scope.chefJenkScriptTaskObj.monthOfYear + cronMinute:startTimeMinute, + cronHour: startTimeHour, + cronWeekDay:dayOfWeek, + cronDate: selectedDayOfTheMonth, + cronMonth: selectedMonth, + cronYear: $scope.chefJenkScriptTaskObj.monthOfYear } }, function () { console.log('Dismiss time is ' + new Date()); diff --git a/client/htmls/private/ajax/Settings/scriptList.html b/client/htmls/private/ajax/Settings/scriptList.html index 426cc4e7d..c00fece50 100644 --- a/client/htmls/private/ajax/Settings/scriptList.html +++ b/client/htmls/private/ajax/Settings/scriptList.html @@ -77,6 +77,7 @@

@@ -84,7 +85,7 @@

-
+
@@ -94,6 +95,40 @@

+
+ +
+
+ +
+
+ +
+
+
+
+
+
+
+
+ +
+ +
+
+
diff --git a/client/htmls/private/js/scriptList.js b/client/htmls/private/js/scriptList.js index 649ede1dc..f76533eb1 100644 --- a/client/htmls/private/js/scriptList.js +++ b/client/htmls/private/js/scriptList.js @@ -7,6 +7,8 @@ function setfilename(val){ //calling the global track functionality when track params are available.. $(document).ready(function(e) { getScriptList(); + $('#divParam').hide(); + $('#checkScriptParam').hide(); }); //when the user clicks on the new button the setting the value to 'new' for the hidden field to know that user is creating the new item.. @@ -23,6 +25,18 @@ $('.addScriptItem').click(function(e) { $('#scriptEditHiddenInput').val('new'); getOrganizationList(); }); + +$("input[name='isParametrized']:radio").change(function(){ + if($(this).val() === 'Yes') { + $('#noOfParams').addClass("required"); + $('#divParam').show(); + $('#checkScriptParam').show(); + } else { + $('#noOfParams').removeClass("required"); + $('#divParam').hide(); + $('#checkScriptParam').hide(); + } +}); //to list down the organization for creating the script item. function getOrganizationList() { $.get('/d4dMasters/readmasterjsonnew/1', function(data) { @@ -46,7 +60,7 @@ var validator = $('#scriptForm').validate({ ignore: [], rules: { scriptFile: { - extension: "sh" + extension: "sh|py" }, scriptName: { maxlength: 15 @@ -54,7 +68,7 @@ var validator = $('#scriptForm').validate({ }, messages: { scriptFile: { - extension: "Only .sh files can be uploaded" + extension: "Only .sh and .py files can be uploaded" }, scriptName: { maxlength: "Limited to 15 chars" @@ -224,6 +238,11 @@ $('#scriptForm').submit(function(e) { var $form = $('#scriptForm'); var scriptData = {}; var $this = $(this); + var isParametrized = false, noOfParams=0; + if($("input[name='isParametrized']:checked").val() === 'Yes'){ + isParametrized =true; + noOfParams=parseInt($('#noOfParams').find(":selected").val()); + } var name = $this.find('#scriptName').val().trim(); var description = $this.find('#scriptDescription').val().trim(); var type = $form.find('#scriptType').val(); @@ -263,7 +282,9 @@ $('#scriptForm').submit(function(e) { "type": type, "description": description, "orgDetails": orgDetails, - "fileId": data.fileId + "fileId": data.fileId, + "isParametrized":isParametrized, + "noOfParams":noOfParams }; } else { url = '../scripts/update/scriptData'; @@ -274,7 +295,9 @@ $('#scriptForm').submit(function(e) { "type": type, "description": description, "orgDetails": orgDetails, - "fileId": data.fileId + "fileId": data.fileId, + "isParametrized":isParametrized, + "noOfParams":noOfParams }; } formSave(methodName,url,reqBody); diff --git a/server/app/cronjobs/catalyst-scheduler/catalystScheduler.js b/server/app/cronjobs/catalyst-scheduler/catalystScheduler.js index 692c57037..bedbc9be7 100644 --- a/server/app/cronjobs/catalyst-scheduler/catalystScheduler.js +++ b/server/app/cronjobs/catalyst-scheduler/catalystScheduler.js @@ -101,8 +101,6 @@ catalystSync.executeSerialScheduledTasks = function executeSerialScheduledTasks( } if(serialTaskList.length === tasks.length){ if(serialTaskList.length > 0) { - console.log(serialTaskList.length); - console.log(serialTaskList); async.waterfall(serialTaskList, function (err, data) { if (err) { logger.error(err); diff --git a/server/app/lib/utils/apiUtil.js b/server/app/lib/utils/apiUtil.js index 0ac9e89cd..17ef5a022 100644 --- a/server/app/lib/utils/apiUtil.js +++ b/server/app/lib/utils/apiUtil.js @@ -34,62 +34,53 @@ var ApiUtil = function() { } this.createCronJobPattern= function(scheduler){ scheduler.cronRepeatEvery = parseInt(scheduler.cronRepeatEvery); + var startOn = null,endOn = null; + if(scheduler.cronStartOn === scheduler.cronEndOn){ + startOn = new Date(); + endOn = new Date() + endOn.setHours(23); + endOn.setMinutes(59); + }else{ + startOn = scheduler.cronStartOn; + endOn = scheduler.cronEndOn; + } if(scheduler.cronFrequency ==='Minutes'){ scheduler.pattern = '*/'+scheduler.cronRepeatEvery+' * * * *'; }else if(scheduler.cronFrequency ==='Hourly'){ scheduler.pattern = '0 */'+scheduler.cronRepeatEvery+' * * *'; }else if(scheduler.cronFrequency ==='Daily'){ - var startOn = Date.parse(scheduler.cronStartOn); - var startHours= startOn.getHours(); - var startMinutes= startOn.getMinutes(); - scheduler.pattern = startMinutes+' '+startHours+' */'+scheduler.cronRepeatEvery+' * *'; + scheduler.pattern = parseInt(scheduler.cronMinute)+' '+parseInt(scheduler.cronHour)+' */'+scheduler.cronRepeatEvery+' * *'; }else if(scheduler.cronFrequency ==='Weekly') { - var startOn = Date.parse(scheduler.cronStartOn); - var startDay= startOn.getDay(); - var startHours= startOn.getHours(); - var startMinutes= startOn.getMinutes(); if(scheduler.cronRepeatEvery === 2) { - scheduler.pattern = startMinutes+' '+startHours+' 8-14 * ' + startDay; + scheduler.pattern = parseInt(scheduler.cronMinute)+' '+parseInt(scheduler.cronHour)+' 8-14 * ' + parseInt(scheduler.cronWeekDay); }else if(scheduler.cronRepeatEvery === 3) { - scheduler.pattern = startMinutes+' '+startHours+' 15-21 * ' + startDay; + scheduler.pattern = parseInt(scheduler.cronMinute)+' '+parseInt(scheduler.cronHour)+' 15-21 * ' + parseInt(scheduler.cronWeekDay); }else if(scheduler.cronRepeatEvery === 4) { - scheduler.pattern = startMinutes+' '+startHours+' 22-28 * ' + startDay; + scheduler.pattern = parseInt(scheduler.cronMinute)+' '+parseInt(scheduler.cronHour)+' 22-28 * ' + parseInt(scheduler.cronWeekDay); }else{ - scheduler.pattern = startMinutes+' '+startHours+' * * ' + startDay; + scheduler.pattern = parseInt(scheduler.cronMinute)+' '+parseInt(scheduler.cronHour)+' * * ' + parseInt(scheduler.cronWeekDay); } - } - if(scheduler.cronFrequency ==='Monthly') { - var startOn = Date.parse(scheduler.cronStartOn); - var startDate= startOn.getDate(); - var startMonth= startOn.getMonth(); - var startDay= startOn.getDay(); - var startHours= startOn.getHours(); - var startMinutes= startOn.getMinutes(); + }else if(scheduler.cronFrequency ==='Monthly') { if(scheduler.cronRepeatEvery === 1) { - scheduler.pattern = startMinutes+' '+startHours+' '+startDate+' * *'; + scheduler.pattern = parseInt(scheduler.cronMinute)+' '+parseInt(scheduler.cronHour)+' '+parseInt(scheduler.cronDate)+' * *'; }else{ - scheduler.pattern = startMinutes+' '+startHours+' '+startDate+' */'+scheduler.cronRepeatEvery+' *'; + scheduler.pattern = parseInt(scheduler.cronMinute)+' '+parseInt(scheduler.cronHour)+' '+parseInt(scheduler.cronDate)+' */'+scheduler.cronRepeatEvery+' *'; } - } - if(scheduler.cronFrequency ==='Yearly') { - var startOn = Date.parse(scheduler.cronStartOn); - var startDate= startOn.getDate(); - var startYear= startOn.getFullYear(); - var startMonth= startOn.getMonth(); - var startHours= startOn.getHours(); - var startMinutes= startOn.getMinutes(); - scheduler.pattern ='0 '+startMinutes+' '+startHours+' '+startDate+' '+startMonth+' ? '+startYear/scheduler.cronRepeatEvery; + }else if(scheduler.cronFrequency ==='Yearly') { + scheduler.pattern ='0 '+parseInt(scheduler.cronMinute)+' '+parseInt(scheduler.cronHour)+' '+parseInt(scheduler.cronDate)+' '+parseInt(scheduler.cronMonth)+' ? '+parseInt(scheduler.cronYear)/scheduler.cronRepeatEvery; } var cronScheduler = { "cronFrequency": scheduler.cronFrequency, "cronRepeatEvery": scheduler.cronRepeatEvery, "cronPattern":scheduler.pattern, - "cronStartOn":Date.parse(scheduler.cronStartOn), - "cronEndOn":Date.parse(scheduler.cronEndOn), - "cronTime":scheduler.cronTime, - "cronDays":scheduler.cronDays, - "cronMonths":scheduler.cronMonths, - "cronYears":scheduler.cronYears + "cronStartOn":Date.parse(startOn), + "cronEndOn":Date.parse(endOn), + "cronHour":scheduler.cronHour, + "cronMinute":scheduler.cronMinute, + "cronDate":scheduler.cronDate, + "cronWeekDay":scheduler.cronWeekDay ? parseInt(scheduler.cronWeekDay):0, + "cronMonth":scheduler.cronMonth ? scheduler.cronMonth: null, + "cronYear":scheduler.cronYear ? scheduler.cronYear: null } return cronScheduler; } diff --git a/server/app/model/blueprint/blueprint.js b/server/app/model/blueprint/blueprint.js index b68228526..63c91ec4b 100755 --- a/server/app/model/blueprint/blueprint.js +++ b/server/app/model/blueprint/blueprint.js @@ -168,6 +168,16 @@ var BlueprintSchema = new Schema({ }, botCategory: { type: String + }, + executionCount:{ + type: Number, + required: false, + default:0 + }, + manualExecutionTime:{ + type: Number, + required: false, + default:10 } }); @@ -1465,6 +1475,23 @@ BlueprintSchema.statics.checkBPDependencyByFieldName = function(fieldName,id, ca callback(null, data); }); }; +BlueprintSchema.statics.updateBlueprintExecutionCount = function updateBlueprintExecutionCount(blueprintId,count,callback) { + Blueprints.update({ + "_id": new ObjectId(blueprintId), + }, { + $set: { + executeCount: count + } + }, { + upsert: false + }, function (err, data) { + if (err) { + callback(err, null); + return; + } + callback(null, data); + }); +}; var Blueprints = mongoose.model('blueprints', BlueprintSchema); diff --git a/server/app/model/classes/tasks/taskTypeScript.js b/server/app/model/classes/tasks/taskTypeScript.js index 8158dbc07..e8bf81c8c 100644 --- a/server/app/model/classes/tasks/taskTypeScript.js +++ b/server/app/model/classes/tasks/taskTypeScript.js @@ -42,9 +42,18 @@ var scriptTaskSchema = taskTypeSchema.extend({ scriptDetails: [{ scriptId: { type: String, - requred: true + required: true }, - scriptParameters: [String] + scriptParameters: [{ + paramVal:{ + type: String, + required: false + }, + paramDesc:{ + type: String, + required: false + } + }] }] }); @@ -62,7 +71,7 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu function getInstances(instanceIds, tagServer, callback) { if (tagServer) { logger.debug('in tagServer', tagServer); - instancesDao.getInstancesByTagServer(tagServer, function(err, instances) { + instancesDao.getInstancesByTagServer(tagServer, function (err, instances) { if (err) { logger.error(err); if (typeof onExecute === 'function') { @@ -83,7 +92,7 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu } return; } - instancesDao.getInstances(instanceIds, function(err, instances) { + instancesDao.getInstances(instanceIds, function (err, instances) { if (err) { logger.error(err); if (typeof onExecute === 'function') { @@ -97,13 +106,13 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu } - getInstances(instanceIds, self.botTagServer, function(err, instances) { + getInstances(instanceIds, self.botTagServer, function (err, instances) { if (err) { logger.error(err); return; } for (var i = 0; i < instances.length; i++) { - (function(instance) { + (function (instance) { var timestampStarted = new Date().getTime(); var actionLog = instancesDao.insertOrchestrationActionLog(instance._id, null, userName, timestampStarted); instance.tempActionLogId = actionLog._id; @@ -146,7 +155,7 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu log: "Instance IP is not defined. Chef Client run failed", timestamp: new Date().getTime() }; - instanceLogModel.createOrUpdate(actionLog._id, instance._id, instanceLog, function(err, logData) { + instanceLogModel.createOrUpdate(actionLog._id, instance._id, instanceLog, function (err, logData) { if (err) { logger.error("Failed to create or update instanceLog: ", err); } @@ -157,7 +166,7 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu }, 1, instance._id, null, actionLog._id); return; } - credentialCryptography.decryptCredential(instance.credentials, function(err, decryptedCredentials) { + credentialCryptography.decryptCredential(instance.credentials, function (err, decryptedCredentials) { var sshOptions = { username: decryptedCredentials.username, host: instance.instanceIP, @@ -169,8 +178,8 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu sshOptions.password = decryptedCredentials.password; } for (var j = 0; j < scriptDetails.length; j++) { - (function(script) { - scriptService.getScriptById(script.scriptId, function(err, scripts) { + (function (script) { + scriptService.getScriptById(script.scriptId, function (err, scripts) { if (err) { logger.error(err); return; @@ -183,7 +192,13 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu if (self.botParams && self.botParams.scriptParams) { params = self.botParams.scriptParams; } - executeBashScript(scripts, sshOptions, logsReferenceIds, params, instanceLog); + executeScriptOnNode(scripts, sshOptions, logsReferenceIds, params, instanceLog,'bash'); + } else if (scripts.type === 'Python') { + var params = script.scriptParameters; + if (self.botParams && self.botParams.scriptParams) { + params = self.botParams.scriptParams; + } + executeScriptOnNode(scripts, sshOptions, logsReferenceIds, params, instanceLog,'python'); } else { return; } @@ -235,19 +250,19 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu } } - function executeBashScript(script, sshOptions, logsReferenceIds, scriptParameters, instanceLog) { + function executeScriptOnNode(script, sshOptions, logsReferenceIds, scriptParameters, instanceLog,scriptType) { var fileName = uuid.v4() + '_' + script.fileName var desPath = appConfig.scriptDir + fileName; var sshExec = new SSHExec(sshOptions); var cryptoConfig = appConfig.cryptoSettings; var cryptography = new Cryptography(cryptoConfig.algorithm, cryptoConfig.password); - fileIo.writeFile(desPath, script.file, false, function(err) { + fileIo.writeFile(desPath, script.file, false, function (err) { if (err) { logger.error("Unable to write file"); return; } else { var scp = new SCP(sshOptions); - scp.upload(desPath, '/tmp', function(err) { + scp.upload(desPath, '/tmp', function (err) { if (err) { var timestampEnded = new Date().getTime(); logsDao.insertLog({ @@ -264,7 +279,7 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu log: "Unable to upload script file " + script.name, timestamp: new Date().getTime() }; - instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function(err, logData) { + instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function (err, logData) { if (err) { logger.error("Failed to create or update instanceLog: ", err); } @@ -273,17 +288,17 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu return; } if (sudoFlag === true) { - var cmdLine = 'sudo bash /tmp/' + fileName; + var cmdLine = 'sudo '+scriptType+' /tmp/' + fileName; } else { - var cmdLine = 'bash /tmp/' + fileName; + var cmdLine = scriptType +' /tmp/' + fileName; } - if (scriptParameters.length > 0) { + if (script.isParameterized === true) { for (var j = 0; j < scriptParameters.length; j++) { - var decryptedText = cryptography.decryptText(scriptParameters[j], cryptoConfig.decryptionEncoding, cryptoConfig.encryptionEncoding); + var decryptedText = cryptography.decryptText(scriptParameters[j].paramVal, cryptoConfig.decryptionEncoding, cryptoConfig.encryptionEncoding); cmdLine = cmdLine + ' "' + decryptedText + '"'; } } - sshExec.exec(cmdLine, function(err, retCode) { + sshExec.exec(cmdLine, function (err, retCode) { if (err) { var timestampEnded = new Date().getTime(); logsDao.insertLog({ @@ -300,7 +315,7 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu log: 'Unable to run script ' + script.name, timestamp: new Date().getTime() }; - instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function(err, logData) { + instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function (err, logData) { if (err) { logger.error("Failed to create or update instanceLog: ", err); } @@ -325,7 +340,7 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu log: 'Task execution success for script ' + script.name, timestamp: new Date().getTime() }; - instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function(err, logData) { + instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function (err, logData) { if (err) { logger.error("Failed to create or update instanceLog: ", err); } @@ -349,7 +364,7 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu log: 'Host Unreachable', timestamp: new Date().getTime() }; - instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function(err, logData) { + instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function (err, logData) { if (err) { logger.error("Failed to create or update instanceLog: ", err); } @@ -370,7 +385,7 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu log: 'Invalid credentials', timestamp: new Date().getTime() }; - instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function(err, logData) { + instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function (err, logData) { if (err) { logger.error("Failed to create or update instanceLog: ", err); } @@ -391,7 +406,7 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu log: 'Unknown error occured. ret code = ' + retCode, timestamp: new Date().getTime() }; - instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function(err, logData) { + instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function (err, logData) { if (err) { logger.error("Failed to create or update instanceLog: ", err); } @@ -414,7 +429,7 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu log: 'Error in running script ' + script.name, timestamp: new Date().getTime() }; - instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function(err, logData) { + instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function (err, logData) { if (err) { logger.error("Failed to create or update instanceLog: ", err); } @@ -422,7 +437,7 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu removeScriptFile(desPath); return; } - }, function(stdOut) { + }, function (stdOut) { logsDao.insertLog({ referenceId: logsReferenceIds, err: false, @@ -434,12 +449,12 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu log: stdOut.toString('ascii'), timestamp: new Date().getTime() }; - instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function(err, logData) { + instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function (err, logData) { if (err) { logger.error("Failed to create or update instanceLog: ", err); } }); - }, function(stdErr) { + }, function (stdErr) { logsDao.insertLog({ referenceId: logsReferenceIds, err: true, @@ -451,17 +466,19 @@ scriptTaskSchema.methods.execute = function(userName, baseUrl, choiceParam, nexu log: stdErr.toString('ascii'), timestamp: new Date().getTime() }; - instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function(err, logData) { + instanceLogModel.createOrUpdate(logsReferenceIds[1], logsReferenceIds[0], instanceLog, function (err, logData) { if (err) { logger.error("Failed to create or update instanceLog: ", err); } }); }); }) - }; + } + ; }); }; -}; +} + function removeScriptFile(filePath) { fileIo.removeFile(filePath, function(err, result) { diff --git a/server/app/model/classes/tasks/tasks.js b/server/app/model/classes/tasks/tasks.js index 8423d4fe6..0e7c6329d 100755 --- a/server/app/model/classes/tasks/tasks.js +++ b/server/app/model/classes/tasks/tasks.js @@ -159,23 +159,29 @@ var taskSchema = new Schema({ }, cronRepeatEvery: { type: Number, - required: false, - trim: true + required: false }, cronFrequency: { type: String, required: false, trim: true }, - cronTime:{ - type: String, + cronMinute:{ + type: Number, required: false, trim: true }, - cronDays:{ - type: String, - required: false, - trim: true + cronHour:{ + type: Number, + required: false + }, + cronWeekDay:{ + type: Number, + required: false + }, + cronDate:{ + type: Number, + required: false }, cronMonth:{ type: String, @@ -183,15 +189,24 @@ var taskSchema = new Schema({ trim: true }, cronYear:{ - type: String, - required: false, - trim: true + type: Number, + required: false } }, cronJobId:{ type: String, required: false, trim: true + }, + executionCount:{ + type: Number, + required: false, + default:0 + }, + manualExecutionTime:{ + type: Number, + required: false, + default:10 } }); taskSchema.plugin(mongoosePaginate); @@ -751,7 +766,8 @@ taskSchema.statics.updateTaskById = function(taskId, taskData, callback) { blueprintIds: taskData.blueprintIds, executionOrder:taskData.executionOrder, taskScheduler:taskData.taskScheduler, - isTaskScheduled:taskData.isTaskScheduled + isTaskScheduled:taskData.isTaskScheduled, + manualExecutionTime:taskData.manualExecutionTime } }, { upsert: false @@ -967,6 +983,23 @@ taskSchema.statics.updateTaskScheduler = function updateTaskScheduler(taskId, ca callback(null, data); }); }; +taskSchema.statics.updateTaskExecutionCount = function updateTaskExecutionCount(taskId,count,callback) { + Tasks.update({ + "_id": new ObjectId(taskId), + }, { + $set: { + executionCount: count + } + }, { + upsert: false + }, function (err, data) { + if (err) { + callback(err, null); + return; + } + callback(null, data); + }); +}; function filterScriptTaskData(data,callback){ var taskList = []; diff --git a/server/app/model/scripts/scripts.js b/server/app/model/scripts/scripts.js index 01f91b714..6f562f0b5 100644 --- a/server/app/model/scripts/scripts.js +++ b/server/app/model/scripts/scripts.js @@ -55,6 +55,16 @@ var ScriptSchema = new Schema({ type:String, required:true, trim:true + }, + isParametrized:{ + type:Boolean, + required:false, + default:false + }, + noOfParams:{ + type:Number, + required:false, + default:0 } }); ScriptSchema.plugin(mongoosePaginate); @@ -88,7 +98,9 @@ ScriptSchema.statics.updateScript = function(scriptData, callback) { $set: { "name": scriptData.name, "description": scriptData.description, - "fileId": scriptData.fileId + "fileId": scriptData.fileId, + "isParametrized":scriptData.isParametrized, + "noOfParams":scriptData.noOfParams }, },{ upsert: false diff --git a/server/app/routes/v1.0/routes_blueprints.js b/server/app/routes/v1.0/routes_blueprints.js index f9298ab30..90b9d8d13 100755 --- a/server/app/routes/v1.0/routes_blueprints.js +++ b/server/app/routes/v1.0/routes_blueprints.js @@ -384,6 +384,17 @@ module.exports.setRoutes = function(app, sessionVerificationFunc) { } var stackName = null; var domainName = null; + var blueprintLaunchCount = 0; + if(blueprint.executionCount) { + blueprintLaunchCount = blueprint.executionCount + 1; + }else{ + blueprintLaunchCount = 1; + } + Blueprints.updateBlueprintExecutionCount(blueprint._id,blueprintLaunchCount,function(err,data){ + if(err){ + logger.error("Error while updating Blueprint Execution Count"); + } + }); if (blueprint.blueprintType === 'aws_cf' || blueprint.blueprintType === 'azure_arm') { stackName = req.query.stackName; if (!stackName) { diff --git a/server/app/routes/v1.0/routes_organizations.js b/server/app/routes/v1.0/routes_organizations.js index dda4b727a..3c458a04f 100755 --- a/server/app/routes/v1.0/routes_organizations.js +++ b/server/app/routes/v1.0/routes_organizations.js @@ -434,6 +434,7 @@ module.exports.setRoutes = function(app, sessionVerification) { var category = 'blueprints'; var permissionto = 'create'; var domainNameCheck = false; + var manualExecutionTime = 10; var orgId = req.params.orgId; var bgId = req.params.bgId; var projectId = req.params.projectId; @@ -452,6 +453,9 @@ module.exports.setRoutes = function(app, sessionVerification) { var botType = req.body.blueprintData.botType; var botCategory = req.body.blueprintData.botCategory; var serviceDeliveryCheck = req.body.blueprintData.serviceDeliveryCheck; + if(req.body.blueprintData.manualExecutionTime && req.body.blueprintData.manualExecutionTime !== null){ + manualExecutionTime = req.body.blueprintData.manualExecutionTime; + } if(req.body.blueprintData.domainNameCheck === 'true'){ domainNameCheck = true; } @@ -502,7 +506,8 @@ module.exports.setRoutes = function(app, sessionVerification) { botType:botType, botCategory:botCategory, serviceDeliveryCheck:serviceDeliveryCheck, - domainNameCheck:domainNameCheck + domainNameCheck:domainNameCheck, + manualExecutionTime:manualExecutionTime }; //adding bluerpintID if present (edit mode) if (blueprintId) @@ -1043,19 +1048,22 @@ module.exports.setRoutes = function(app, sessionVerification) { var count = 0; var encryptedList = []; for(var i = 0; i < paramDetails.length; i++){ - (function(param){ - if(param.scriptParameters.length > 0){ + (function(paramDetail){ + if(paramDetail.scriptParameters.length > 0){ count++; - for(var j = 0; j < param.scriptParameters.length; j++){ + for(var j = 0; j < paramDetail.scriptParameters.length; j++){ (function(scriptParameter){ - var encryptedText = cryptography.encryptText(scriptParameter, cryptoConfig.encryptionEncoding, + var encryptedText = cryptography.encryptText(scriptParameter.paramVal, cryptoConfig.encryptionEncoding, cryptoConfig.decryptionEncoding); - encryptedList.push(encryptedText); - if(encryptedList.length === param.scriptParameters.length){ - param.scriptParameters = encryptedList; + encryptedList.push({ + paramVal:encryptedText, + paramDesc:scriptParameter.paramDesc + }); + if(encryptedList.length === paramDetail.scriptParameters.length){ + paramDetail.scriptParameters = encryptedList; encryptedList = []; } - })(param.scriptParameters[j]); + })(paramDetail.scriptParameters[j]); } }else{ count++; diff --git a/server/app/routes/v1.0/routes_tasks.js b/server/app/routes/v1.0/routes_tasks.js index 0779a60f0..34298868b 100755 --- a/server/app/routes/v1.0/routes_tasks.js +++ b/server/app/routes/v1.0/routes_tasks.js @@ -634,25 +634,28 @@ function encryptedParam(paramDetails, existingParams, callback) { var count = 0; var encryptedList = []; for (var i = 0; i < paramDetails.length; i++) { - (function(param) { - if (param.scriptParameters.length > 0) { + (function(paramDetail) { + if (paramDetail.scriptParameters.length > 0) { count++; - for (var j = 0; j < param.scriptParameters.length; j++) { + for (var j = 0; j < paramDetail.scriptParameters.length; j++) { (function(scriptParameter) { if (scriptParameter === '') { encryptedList.push(existingParams[i].scriptParameters[j]); - } else if (scriptParameter === existingParams[i].scriptParameters[j]) { + } else if (scriptParameter.paramVal === existingParams[i].scriptParameters[j].paramVal) { encryptedList.push(scriptParameter); } else { - var encryptedText = cryptography.encryptText(scriptParameter, cryptoConfig.encryptionEncoding, + var encryptedText = cryptography.encryptText(scriptParameter.paramVal, cryptoConfig.encryptionEncoding, cryptoConfig.decryptionEncoding); - encryptedList.push(encryptedText); + encryptedList.push({ + paramVal:encryptedText, + paramVal:scriptParameter.paramVal + }); } - if (encryptedList.length === param.scriptParameters.length) { - param.scriptParameters = encryptedList; + if (encryptedList.length === paramDetail.scriptParameters.length) { + paramDetail.scriptParameters = encryptedList; encryptedList = []; } - })(param.scriptParameters[j]); + })(paramDetail.scriptParameters[j]); } } else { count++; diff --git a/server/app/services/instanceService.js b/server/app/services/instanceService.js index ed4555535..c165911fc 100644 --- a/server/app/services/instanceService.js +++ b/server/app/services/instanceService.js @@ -1487,12 +1487,22 @@ function updateScheduler(instanceScheduler, callback) { function generateCronPattern(cronInterval,startDate,endDate,callback){ var startIntervalList =[],stopIntervalList=[],count = 0; + var startOn = null,endOn = null; + if(startDate === endDate){ + startOn = new Date(); + endOn = new Date() + endOn.setHours(23); + endOn.setMinutes(59); + }else{ + startOn = startDate; + endOn = endDate; + } if(cronInterval.length === 0){ var scheduler= { instanceStartScheduler: startIntervalList, instanceStopScheduler: stopIntervalList, - schedulerStartOn: Date.parse(startDate), - schedulerEndOn: Date.parse(endDate), + schedulerStartOn: Date.parse(startOn), + schedulerEndOn: Date.parse(endOn), isScheduled: false } callback(null,scheduler); @@ -1522,8 +1532,8 @@ function generateCronPattern(cronInterval,startDate,endDate,callback){ var scheduler= { instanceStartScheduler: startIntervalList, instanceStopScheduler: stopIntervalList, - schedulerStartOn: Date.parse(startDate), - schedulerEndOn: Date.parse(endDate), + schedulerStartOn: Date.parse(startOn), + schedulerEndOn: Date.parse(endOn), isScheduled: true } callback(null,scheduler); @@ -1551,8 +1561,8 @@ function generateCronPattern(cronInterval,startDate,endDate,callback){ var scheduler= { instanceStartScheduler: startIntervalList, instanceStopScheduler: stopIntervalList, - schedulerStartOn: Date.parse(startDate), - schedulerEndOn: Date.parse(endDate), + schedulerStartOn: Date.parse(startOn), + schedulerEndOn: Date.parse(endOn), isScheduled: true } callback(null,scheduler); diff --git a/server/app/services/schedulerService.js b/server/app/services/schedulerService.js index a5b137bad..7b10aefaf 100644 --- a/server/app/services/schedulerService.js +++ b/server/app/services/schedulerService.js @@ -183,14 +183,13 @@ schedulerService.startStopInstance= function startStopInstance(instanceId,catUse }, function(instanceDetails,next){ var currentDate = new Date(); - var dateToString = currentDate.toDateString(); if(instanceDetails[0].instanceState === 'terminated'){ callback({ errCode:201, errMsg:"Instance is already in "+instanceDetails[0].instanceState+" state. So no need to do any action." }) return; - }else if (instanceDetails[0].isScheduled && instanceDetails[0].isScheduled === true && Date.parse(dateToString) > instanceDetails[0].schedulerEndOn) { + }else if (instanceDetails[0].isScheduled && instanceDetails[0].isScheduled === true && currentDate > instanceDetails[0].schedulerEndOn) { instancesDao.updateInstanceScheduler(instanceDetails[0]._id,function(err, updatedData) { if (err) { logger.error("Failed to update Instance Scheduler: ", err); diff --git a/server/app/services/scriptService.js b/server/app/services/scriptService.js index ce2e76910..3d65aea2d 100644 --- a/server/app/services/scriptService.js +++ b/server/app/services/scriptService.js @@ -98,14 +98,16 @@ scriptService.getScriptById=function getScriptById(scriptId,callback){ next(err,null); }else{ var scriptObj= { - scriptId: scripts[0]._id, - name: scripts[0].name, - type: scripts[0].type, - description: scripts[0].description, - orgDetails: scripts[0].orgDetails, - fileId: scripts[0].fileId, - fileName: file.fileName, - file: file.fileData + scriptId: scripts[0]._id, + name: scripts[0].name, + type: scripts[0].type, + description: scripts[0].description, + orgDetails: scripts[0].orgDetails, + isParametrized: scripts[0].isParametrized, + noOfParams: scripts[0].noOfParams, + fileId: scripts[0].fileId, + fileName: file.fileName, + file: file.fileData }; next(null,scriptObj); } @@ -208,13 +210,15 @@ function addFileDetailsForScripts(scripts,callback){ return callback(err,null); }else{ scriptObj = { - scriptId:script._id, - name:script.name, - type:script.type, - description:script.description, - orgDetails:script.orgDetails, - fileId:script.fileId, - fileName:file.filename + scriptId:script._id, + name:script.name, + type:script.type, + description:script.description, + orgDetails:script.orgDetails, + isParametrized: script.isParametrized, + noOfParams: script.noOfParams, + fileId:script.fileId, + fileName:file.filename } scriptList.push(scriptObj); scriptObj={}; diff --git a/server/app/services/taskService.js b/server/app/services/taskService.js index a9186a6c4..afb6f5c96 100644 --- a/server/app/services/taskService.js +++ b/server/app/services/taskService.js @@ -151,6 +151,17 @@ taskService.executeTask = function executeTask(taskId, user, hostProtocol, choic task.botParams = paramOptions; task.botTagServer = botTagServer; var auditTrailId = null; + var taskExecutionCount = 0; + if(task.executionCount){ + taskExecutionCount = task.taskExecutionCount + 1 + }else{ + taskExecutionCount = 1; + } + taskDao.updateTaskExecutionCount(task._id,taskExecutionCount,function(err,data){ + if(err){ + logger.error("Error while updating Task Execution Count"); + } + }); if(task.serviceDeliveryCheck === true){ var actionObj={ auditType:'BOTs', From aeb49c5fa6ff695572edaeecddcea5b8ee264149 Mon Sep 17 00:00:00 2001 From: Durgesh1988 Date: Fri, 2 Dec 2016 17:58:20 +0530 Subject: [PATCH 07/14] Resolved merge conflicts --- .../orchestration/popups/addChefJobEvent.html | 16 +++++-- .../popups/addChefJobEventCtrl.js | 43 ++++++++++++------- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEvent.html b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEvent.html index d454fa2d5..66c49b3b8 100644 --- a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEvent.html +++ b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEvent.html @@ -64,10 +64,14 @@

 Schedule your Taskrequired - + + +

diff --git a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEventCtrl.js b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEventCtrl.js index c467482b2..03d7ed88d 100644 --- a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEventCtrl.js +++ b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEventCtrl.js @@ -10,13 +10,14 @@ angular.module('workzone.orchestration') .controller('addChefJobEventCtrl',['$scope', '$modalInstance', 'items' ,'toastr',function($scope, $modalInstance , items , toastr){ console.log(items); + $scope.checkFrequencyCheck=false; $scope.defaultSelection = function() { $scope.repeatsType = 'Minutes';//default selection. $scope.schedulerStartOn=moment(new Date()).format('MM/DD/YYYY'); $scope.schedulerEndOn=moment(new Date()).format('MM/DD/YYYY'); }; - if(items.type !== 'new'){ - if(items.chefJenkScriptTaskObj) { + if(items.type !== 'new') { + if (items.chefJenkScriptTaskObj) { if (items.chefJenkScriptTaskObj.cronStartOn && items.chefJenkScriptTaskObj.cronEndOn) { var newStartOn = parseInt(items.chefJenkScriptTaskObj.cronStartOn); var newDate = new Date(newStartOn).toLocaleDateString(); @@ -32,19 +33,21 @@ $scope.schedulerStartOn = items.chefJenkScriptTaskObj.cronStart; $scope.schedulerEndOn = items.chefJenkScriptTaskObj.cronEnd; } - } - $scope.repeatBy = items.chefJenkScriptTaskObj.repeatBy || items.chefJenkScriptTaskObj.cronRepeatEvery.toString(); - $scope.repeatsType = items.chefJenkScriptTaskObj.repeats || items.chefJenkScriptTaskObj.cronFrequency; - $scope.timeEventType = items.chefJenkScriptTaskObj.cronHour; - $scope.timeEventMinute = items.chefJenkScriptTaskObj.cronMinute; - $scope.weekOfTheDay = items.chefJenkScriptTaskObj.cronWeekDay; - $scope.currentDate = items.chefJenkScriptTaskObj.cronDate; - $scope.selectedDayOfTheMonth = items.chefJenkScriptTaskObj.cronMonth; - $scope.selectedMonth = items.chefJenkScriptTaskObj.cronYear; - if($scope.repeatsType === 'Minutes' || $scope.repeatsType === 'Hourly'){ - $scope.checkFrequencyCheck = false; - }else{ - $scope.checkFrequencyCheck = true; + $scope.repeatBy = items.chefJenkScriptTaskObj.repeatBy || items.chefJenkScriptTaskObj.cronRepeatEvery.toString(); + $scope.repeatsType = items.chefJenkScriptTaskObj.repeats || items.chefJenkScriptTaskObj.cronFrequency; + $scope.timeEventHour = items.chefJenkScriptTaskObj.cronHour; + $scope.timeEventMinute = items.chefJenkScriptTaskObj.cronMinute; + $scope.weekOfTheDay = items.chefJenkScriptTaskObj.cronWeekDay; + $scope.currentDate = items.chefJenkScriptTaskObj.cronDate; + $scope.selectedDayOfTheMonth = items.chefJenkScriptTaskObj.cronMonth; + $scope.selectedMonth = items.chefJenkScriptTaskObj.cronYear; + if ($scope.repeatsType === 'Minutes' || $scope.repeatsType === 'Hourly') { + $scope.checkFrequencyCheck = false; + } else { + $scope.checkFrequencyCheck = true; + } + } else { + $scope.defaultSelection(); } } else { $scope.defaultSelection(); @@ -77,6 +80,14 @@ } return input; }; + $scope.repeatHourCount = function(max, step) { + step = step || 1; + var input = []; + for (var i = 0; i <= max; i += step) { + input.push(i); + } + return input; + }; $scope.isDaySelected = { flag:true } @@ -93,7 +104,7 @@ repeatBy: $scope.repeatBy, cronStart: $scope.schedulerStartOn, cronEnd: $scope.schedulerEndOn, - startTime: $scope.timeEventType, + startTime: $scope.timeEventHour, startTimeMinute: $scope.timeEventMinute, dayOfWeek: $scope.weekOfTheDay, selectedDayOfTheMonth: $scope.selectedDayOfTheMonth, From 4394ff1ae190a76dfea313554518bd7bd901f668 Mon Sep 17 00:00:00 2001 From: Durgesh1988 Date: Mon, 5 Dec 2016 11:57:36 +0530 Subject: [PATCH 08/14] Script Executor for Python Implementation --- .../htmls/private/ajax/Settings/scriptList.html | 2 +- client/htmls/private/js/scriptList.js | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/client/htmls/private/ajax/Settings/scriptList.html b/client/htmls/private/ajax/Settings/scriptList.html index c00fece50..1e7d4c93c 100644 --- a/client/htmls/private/ajax/Settings/scriptList.html +++ b/client/htmls/private/ajax/Settings/scriptList.html @@ -99,7 +99,7 @@

- +
diff --git a/client/htmls/private/js/scriptList.js b/client/htmls/private/js/scriptList.js index f76533eb1..133118463 100644 --- a/client/htmls/private/js/scriptList.js +++ b/client/htmls/private/js/scriptList.js @@ -98,7 +98,10 @@ function getScriptList() { "serverSide": true, "destroy":true, "createdRow": function( row, data ) { - $( row ).attr({"scriptId" : data.scriptId,"scriptName":data.name,"scriptType":data.type, "scriptDesc" : data.description, "orgId" : data.orgDetails.id ,"orgName" : data.orgDetails.name,"scriptFileName" : data.fileName,"scriptFileId" : data.fileId}); + $( row ).attr({"scriptId" : data.scriptId,"scriptName":data.name,"scriptType":data.type, + "scriptDesc" : data.description, "orgId" : data.orgDetails.id ,"orgName" : data.orgDetails.name, + "scriptFileName" : data.fileName,"scriptFileId" : data.fileId, + "isParametrized" : data.isParametrized,"noOfParams":data.noOfParams}); }, "ajax": '/scripts', "columns": [ @@ -157,6 +160,18 @@ $('#scriptListTable tbody').on( 'click', 'button.editRowScriptItem', function(){ $editModal.find('#scriptHiddenInputId').val($this.parents('tr').attr('scriptId')); $editModal.find('#fileHiddenInputId').val($this.parents('tr').attr('scriptFileId')); $editModal.find('#fileNameDisplay').append($this.parents('tr').attr('scriptFileName')); + if($this.parents('tr').attr('isParametrized') === true || $this.parents('tr').attr('isParametrized') === 'true'){ + $('input:radio[name="isParametrized"][value="Yes"]').prop('checked', true); + $('#noOfParams').addClass("required"); + $('#divParam').show(); + $('#checkScriptParam').show(); + $editModal.find('#noOfParams').val($this.parents('tr').attr('noOfParams')); + }else{ + $('input:radio[name="isParametrized"][value="No"]').prop('checked', true); + $('#noOfParams').removeClass("required"); + $('#divParam').hide(); + $('#checkScriptParam').hide(); + } $editModal.find('#scriptFileNameHidden').empty().append($this.parents('tr').attr('scriptFileName')); return false; }); From d66b94616e10ab80eab72c65307a35c141700abd Mon Sep 17 00:00:00 2001 From: Karthik K N Date: Thu, 1 Dec 2016 14:28:06 +0530 Subject: [PATCH 09/14] Fix for CAT-2445 Incorrect Total Cost is showing in Analytics tab --- .../model/resource-costs/resource-costs.js | 43 +++-- server/app/services/resourceService.js | 161 ++++++++++-------- 2 files changed, 124 insertions(+), 80 deletions(-) diff --git a/server/app/model/resource-costs/resource-costs.js b/server/app/model/resource-costs/resource-costs.js index 012c774f6..28a47e042 100644 --- a/server/app/model/resource-costs/resource-costs.js +++ b/server/app/model/resource-costs/resource-costs.js @@ -70,6 +70,11 @@ var ResourceCostsSchema = new Schema({ required: false, trim: true }, + billIntervalId: { + type: String, + required: true, + trim: true + }, billLineItemId: { type: Number, required: true, @@ -132,38 +137,56 @@ var ResourceCostsSchema = new Schema({ }) ResourceCostsSchema.index({'platformDetails.serviceId' : 1}) -ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'billLineItemId': 1, - 'startTime': 1, 'interval': 1}, {'unique': true}) +ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'startTime': 1, + 'billIntervalId': 1, 'billLineItemId': 1, 'interval': 1}, {'unique': true}) -ResourceCostsSchema.statics.saveResourceCost = function saveResourceCost(resourceCostData, callback) { +ResourceCostsSchema.statics.save = function save(resourceCostData, callback) { var resourceCosts = new ResourceCosts(resourceCostData) - resourceCosts.save(function(err, data) { + resourceCosts.save(function(err, result) { + if (err) { + callback(err) + } else { + callback(null, result) + } + }) +} + +ResourceCostsSchema.statics.remove + = function remove(organizationId, providerId, billIntervalId, callback) { + var query = { + organizationId: organizationId, + providerId: providerId, + billIntervalId: billIntervalId + } + + this.find(query).remove(function(err, result) { if (err) { callback(err) } else { - callback(null) + callback(null, result) } }) } -ResourceCostsSchema.statics.upsertResourceCost = function upsertResourceCost(resourceCostData, callback) { +ResourceCostsSchema.statics.upsert = function upsert(resourceCostData, callback) { var query = { organizationId: resourceCostData.organizationId, providerId: resourceCostData.providerId, - billLineItemId: resourceCostData.billLineItemId, startTime: resourceCostData.startTime, + billIntervalId: resourceCostData.billIntervalId, + billLineItemId: resourceCostData.billLineItemId, interval: resourceCostData.interval } - this.findOneAndUpdate(query, resourceCostData, {upsert:true}, + this.findOneAndUpdate(query, resourceCostData, {upsert: true}, function(err, result){ if (err) { callback(null) } else { callback(null, result) } - }); + }) } var ResourceCosts = mongoose.model('ResourceCosts', ResourceCostsSchema) -module.exports = ResourceCosts +module.exports = ResourceCosts \ No newline at end of file diff --git a/server/app/services/resourceService.js b/server/app/services/resourceService.js index f56a7f95c..deb4f322d 100644 --- a/server/app/services/resourceService.js +++ b/server/app/services/resourceService.js @@ -91,91 +91,112 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u var awsZones = appConfig.aws.zones var lineNumber = 0 - var stream = fs.createReadStream(downlaodedCSVPath) - csv.fromStream(stream, {headers: false}).on('data', function(data) { - if(data[awsBillIndexes.totalCost] == 'LineItem') { - var resourceCostEntry = {platformDetails: {}} - - resourceCostEntry.organizationId = provider.orgId - resourceCostEntry.providerId = provider._id - resourceCostEntry.providerType = provider.providerType - resourceCostEntry.cost = data[awsBillIndexes.cost] - resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate]) - resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate]) - resourceCostEntry.lastUpdateTime = Date.parse(updateTime) - resourceCostEntry.interval = 3600 - resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod] - resourceCostEntry.billLineItemId = ++lineNumber - resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - - resourceCostEntry.platformDetails.serviceId - = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] - :resourceCostEntry.platformDetails.serviceId = 'Other' - - resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) - ? 'Global' : data[awsBillIndexes.zone] - - resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) - ? awsZones[data[awsBillIndexes.zone]] : 'Global' - - if (data[awsBillIndexes.instanceId] != null) { - resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] - } + var date = new Date() + var billIntervalId = date.getFullYear() + '-' + date.getMonth() - if(data[awsBillIndexes.usageType] != null) { - resourceCostEntry.platformDetails.usageType = data[awsBillIndexes.usageType] - } + async.waterfall([ + function(next) { + resourceCost.remove(provider.orgId, provider._id, billIntervalId, next) + }, + function(count, next) { + /*var lineNumber = (count == 0)?0:count + var startingLineNumber = (count == 0)?1:(count+2)*/ - if (data[awsBillIndexes.instanceId] in resources) { - var resource = resources[data[awsBillIndexes.instanceId]] + var stream = fs.createReadStream(downlaodedCSVPath) + csv.fromStream(stream).on('data', function(data) { + if(data[awsBillIndexes.totalCost] == 'LineItem') { + var resourceCostEntry = {platformDetails: {}} - resourceCostEntry.resourceId = resource._id + resourceCostEntry.organizationId = provider.orgId + resourceCostEntry.providerId = provider._id + resourceCostEntry.providerType = provider.providerType + resourceCostEntry.cost = data[awsBillIndexes.cost] + resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate]) + resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate]) + resourceCostEntry.lastUpdateTime = Date.parse(updateTime) + resourceCostEntry.interval = 3600 + resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod] + resourceCostEntry.billIntervalId = billIntervalId + resourceCostEntry.billLineItemId = ++lineNumber + resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - if (('bgId' in resource) && (resource.bgId != null)) { - resourceCostEntry.businessGroupId = resource['bgId'] - } + resourceCostEntry.platformDetails.serviceId + = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] + :resourceCostEntry.platformDetails.serviceId = 'Other' - if (('projectId' in resource) && (resource.projectId != null)) { - resourceCostEntry.projectId = resource['projectId'] - } + resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) + ? 'Global' : data[awsBillIndexes.zone] - if (('environmentId' in resource) && (resource.environmentId != null)) { - resourceCostEntry.environmentId = resource['environmentId'] - } + resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) + ? awsZones[data[awsBillIndexes.zone]] : 'Global' - if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { - console.log("BG: " + resource['bgId']) - resourceCostEntry.businessGroupId = resource['bgId'] - } + if (data[awsBillIndexes.instanceId] != null) { + resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] + } - if (('masterDetails.projectId' in resource) - && (resource.masterDetails.projectId != null)) { - resourceCostEntry.projectId = resource['projectId'] - } + if(data[awsBillIndexes.usageType] != null) { + resourceCostEntry.platformDetails.usageType = data[awsBillIndexes.usageType] + } - if (('masterDetails.environmentId' in resource) - && (resource.masterDetails.environmentId != null)) { - resourceCostEntry.environmentId = resource['environmentId'] - } - } + if (data[awsBillIndexes.instanceId] in resources) { + var resource = resources[data[awsBillIndexes.instanceId]] - resourceCostEntry.businessGroupId - = ('businessGroupId' in resourceCostEntry)?resourceCostEntry.businessGroupId:'Unassigned' - resourceCostEntry.projectId - = ('projectId' in resourceCostEntry)?resourceCostEntry.projectId:'Unassigned' - resourceCostEntry.environmentId - = ('environmentId' in resourceCostEntry)?resourceCostEntry.environmentId:'Unassigned' + resourceCostEntry.resourceId = resource._id - resourceCost.upsertResourceCost(resourceCostEntry, function (err, costEntry) { - if (err) { - logger.error(err) - return callback(new Error('Database Error')) + if (('bgId' in resource) && (resource.bgId != null)) { + resourceCostEntry.businessGroupId = resource['bgId'] + } + + if (('projectId' in resource) && (resource.projectId != null)) { + resourceCostEntry.projectId = resource['projectId'] + } + + if (('environmentId' in resource) && (resource.environmentId != null)) { + resourceCostEntry.environmentId = resource['environmentId'] + } + + if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { + console.log("BG: " + resource['bgId']) + resourceCostEntry.businessGroupId = resource['bgId'] + } + + if (('masterDetails.projectId' in resource) + && (resource.masterDetails.projectId != null)) { + resourceCostEntry.projectId = resource['projectId'] + } + + if (('masterDetails.environmentId' in resource) + && (resource.masterDetails.environmentId != null)) { + resourceCostEntry.environmentId = resource['environmentId'] + } + } + + resourceCostEntry.businessGroupId + = ('businessGroupId' in resourceCostEntry)?resourceCostEntry.businessGroupId:'Unassigned' + resourceCostEntry.projectId + = ('projectId' in resourceCostEntry)?resourceCostEntry.projectId:'Unassigned' + resourceCostEntry.environmentId + = ('environmentId' in resourceCostEntry)?resourceCostEntry.environmentId:'Unassigned' + + resourceCost.save(resourceCostEntry, function (err, costEntry) { + if (err) { + logger.error(err) + return next(new Error('Database Error')) + } + }) } + }).on('end', function() { + next() }) } - }).on('end', function() { - callback(null) + ], function(err) { + if(err) { + callback(err) + } else { + callback() + } }) + } function getCostForServices_deprecated(provider,callback) { From 071711988d67c363f4fae0e9fc53ff771a01ad18 Mon Sep 17 00:00:00 2001 From: Durgesh1988 Date: Mon, 5 Dec 2016 12:10:26 +0530 Subject: [PATCH 10/14] Add some debug points for debugging --- server/app/cronjobs/catalyst-scheduler/catalystScheduler.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/app/cronjobs/catalyst-scheduler/catalystScheduler.js b/server/app/cronjobs/catalyst-scheduler/catalystScheduler.js index bedbc9be7..552bdaf7a 100644 --- a/server/app/cronjobs/catalyst-scheduler/catalystScheduler.js +++ b/server/app/cronjobs/catalyst-scheduler/catalystScheduler.js @@ -7,12 +7,14 @@ var cronTab = require('node-crontab'); var catalystSync = module.exports = {}; catalystSync.executeScheduledInstances = function executeScheduledInstances() { + logger.debug("Instance Scheduler is started. "); instancesDao.getScheduledInstances(function(err, instances) { if (err) { logger.error("Failed to fetch Instance: ", err); return; } if (instances && instances.length) { + logger.debug("Schedule Instance length>>"+instances.length); var resultList =[]; for (var i = 0; i < instances.length; i++) { (function(instance) { @@ -21,6 +23,7 @@ catalystSync.executeScheduledInstances = function executeScheduledInstances() { } resultList.push(function(callback){schedulerService.executeSchedulerForInstances(instance,callback);}); if(resultList.length === instances.length){ + logger.debug("Schedule Instance length for Scheduler Start>>"+resultList.length); async.parallel(resultList,function(err,results){ if(err){ logger.error(err); From 52f566b7744f561a04bcf2c0469d799da1e709f7 Mon Sep 17 00:00:00 2001 From: Karthik K N Date: Mon, 5 Dec 2016 12:32:44 +0530 Subject: [PATCH 11/14] cost aggregation fixes --- server/app/model/resource-costs/resource-costs.js | 2 +- server/app/services/resourceService.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/app/model/resource-costs/resource-costs.js b/server/app/model/resource-costs/resource-costs.js index 28a47e042..9a0761fd4 100644 --- a/server/app/model/resource-costs/resource-costs.js +++ b/server/app/model/resource-costs/resource-costs.js @@ -24,7 +24,7 @@ var Schema = mongoose.Schema // @TODO Date field types to be revised var ResourceCostsSchema = new Schema({ cost: { - type: SchemaTypes.Double, + type: Number, required: true }, currency: { diff --git a/server/app/services/resourceService.js b/server/app/services/resourceService.js index deb4f322d..20a38dafe 100644 --- a/server/app/services/resourceService.js +++ b/server/app/services/resourceService.js @@ -92,7 +92,7 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u var lineNumber = 0 var date = new Date() - var billIntervalId = date.getFullYear() + '-' + date.getMonth() + var billIntervalId = date.getFullYear() + '-' + (date.getMonth() + 1) async.waterfall([ function(next) { From fcaf6794183ef9dedb082f84abfe9e10a9f48ec4 Mon Sep 17 00:00:00 2001 From: Durgesh1988 Date: Mon, 5 Dec 2016 13:01:46 +0530 Subject: [PATCH 12/14] Add Task Execution Standard Time in Task/History Schema --- server/app/model/classes/tasks/taskHistory.js | 6 +++++- server/app/model/classes/tasks/tasks.js | 7 +++---- server/app/routes/v1.0/routes_organizations.js | 5 +++++ server/app/routes/v1.0/routes_tasks.js | 5 +++++ 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/server/app/model/classes/tasks/taskHistory.js b/server/app/model/classes/tasks/taskHistory.js index 715b05158..c78748847 100755 --- a/server/app/model/classes/tasks/taskHistory.js +++ b/server/app/model/classes/tasks/taskHistory.js @@ -56,7 +56,11 @@ var taskHistorySchema = new Schema({ bgName: String, projectName: String, envName: String, - taskName: String + taskName: String, + manualExecutionTime: { + type: Number, + required: false + } }); taskHistorySchema.plugin(mongoosePaginate); diff --git a/server/app/model/classes/tasks/tasks.js b/server/app/model/classes/tasks/tasks.js index 0e7c6329d..17d65b945 100755 --- a/server/app/model/classes/tasks/tasks.js +++ b/server/app/model/classes/tasks/tasks.js @@ -205,8 +205,7 @@ var taskSchema = new Schema({ }, manualExecutionTime:{ type: Number, - required: false, - default:10 + required: false } }); taskSchema.plugin(mongoosePaginate); @@ -227,9 +226,9 @@ taskSchema.methods.execute = function(userName, baseUrl, choiceParam, appData, b orgName: self.orgName, bgName: self.bgName, projectName: self.projectName, - envName: self.envName + envName: self.envName, + manualExecutionTime:self.manualExecutionTime }; - if (this.taskType === TASK_TYPE.CHEF_TASK) { task = new ChefTask(this.taskConfig); diff --git a/server/app/routes/v1.0/routes_organizations.js b/server/app/routes/v1.0/routes_organizations.js index 3c458a04f..ab1879030 100755 --- a/server/app/routes/v1.0/routes_organizations.js +++ b/server/app/routes/v1.0/routes_organizations.js @@ -987,6 +987,11 @@ module.exports.setRoutes = function(app, sessionVerification) { if(taskData.taskType === 'jenkins'){ taskData.executionOrder= 'PARALLEL'; } + if(taskData.manualExecutionTime && taskData.manualExecutionTime !== null){ + taskData.manualExecutionTime = parseInt(taskData.manualExecutionTime); + }else{ + taskData.manualExecutionTime = 10; + } configmgmtDao.getEnvNameFromEnvId(req.params.envId, function(err, envName) { if (err) { res.status(500).send("Failed to fetch ENV: ", err); diff --git a/server/app/routes/v1.0/routes_tasks.js b/server/app/routes/v1.0/routes_tasks.js index 34298868b..b8b48a41e 100755 --- a/server/app/routes/v1.0/routes_tasks.js +++ b/server/app/routes/v1.0/routes_tasks.js @@ -521,6 +521,11 @@ module.exports.setRoutes = function(app, sessionVerification) { taskData.taskScheduler = apiUtil.createCronJobPattern(taskData.taskScheduler); taskData.isTaskScheduled = true; } + if(taskData.manualExecutionTime && taskData.manualExecutionTime !== null){ + taskData.manualExecutionTime = parseInt(taskData.manualExecutionTime); + }else{ + taskData.manualExecutionTime = 10; + } if (taskData.taskType === 'script') { Tasks.getTaskById(req.params.taskId, function(err, scriptTask) { if (err) { From 7741989745ab3ba23f3faafae710faf15295e80b Mon Sep 17 00:00:00 2001 From: Durgesh1988 Date: Mon, 5 Dec 2016 13:13:02 +0530 Subject: [PATCH 13/14] Add Task Execution Standard Time in Task/History Schema --- .../workzone/orchestration/popups/addChefJobEvent.html | 4 ++-- .../dashboard/workzone/orchestration/popups/newTask.html | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEvent.html b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEvent.html index 66c49b3b8..fbd8ef8f6 100644 --- a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEvent.html +++ b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/addChefJobEvent.html @@ -68,7 +68,7 @@

 Schedule your TaskChoose Start Hour --> - @@ -83,7 +83,7 @@

 Schedule your TaskChoose Start Minute --> - diff --git a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTask.html b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTask.html index e95231bc1..7744b30b7 100644 --- a/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTask.html +++ b/client/cat3/src/partials/sections/dashboard/workzone/orchestration/popups/newTask.html @@ -534,7 +534,7 @@

Script Details
From 1bc6a96bb9362d89b2d571066c72d6c6385a3390 Mon Sep 17 00:00:00 2001 From: Udesh Date: Mon, 5 Dec 2016 15:13:29 +0530 Subject: [PATCH 14/14] added filterby in instances api and actionLogs api --- server/app/model/classes/instance/instance.js | 4 +- server/app/routes/v1.0/routes_instances.js | 28 +- server/app/services/instanceService.js | 342 +++++++++++------- 3 files changed, 216 insertions(+), 158 deletions(-) diff --git a/server/app/model/classes/instance/instance.js b/server/app/model/classes/instance/instance.js index 3291514c5..075bd3400 100755 --- a/server/app/model/classes/instance/instance.js +++ b/server/app/model/classes/instance/instance.js @@ -1515,11 +1515,11 @@ var InstancesDao = function () { return actionLog._id; } - this.getAllActionLogs = function (instanceId, callback) { + this.getAllActionLogs = function (instanceId,filterByQuery, callback) { logger.debug("Enter getAllActionLogs (%s)", instanceId); Instances.find({ "_id": new ObjectId(instanceId) - }, function (err, data) { + },filterByQuery, function (err, data) { if (err) { logger.error("Failed getAllActionLogs (%s)", instanceId, err); callback(err, null); diff --git a/server/app/routes/v1.0/routes_instances.js b/server/app/routes/v1.0/routes_instances.js index 624e4af31..131297c14 100755 --- a/server/app/routes/v1.0/routes_instances.js +++ b/server/app/routes/v1.0/routes_instances.js @@ -256,15 +256,11 @@ module.exports.setRoutes = function (app, sessionVerificationFunc) { function (next) { apiUtil.paginationRequest(req.query, 'instances', next); }, + function (paginationReq, next) { + instanceService.parseInstanceMonitorQuery(paginationReq, next); + }, function (paginationReq, next) { reqData = paginationReq; - if (paginationReq.filterBy && paginationReq.filterBy.monitor) { - if (paginationReq.filterBy.monitor === "true") { - paginationReq.filterBy.monitor = {$ne: null}; - } else { - paginationReq.filterBy.monitor = null; - } - } instancesDao.getInstanceList(paginationReq, next); }, function (instances, next) { @@ -2674,21 +2670,21 @@ module.exports.setRoutes = function (app, sessionVerificationFunc) { app.get('/instances/:instanceId/actionLogs', function (req, res) { logger.debug("Enter get() for /instances/%s/actionLogs", req.params.instanceId); - instancesDao.getAllActionLogs(req.params.instanceId, function (err, actionLogs) { + async.waterfall([ + function (next) { + instanceService.parseActionLogsQuery(req.query, next) + }, + function (filterByQuery, next) { + instanceService.getInstanceActionLogs(req.params.instanceId, filterByQuery, next); + } + ], function (err, actionLogs) { if (err) { logger.error("Failed to fetch ActionLogs: ", err); res.send(500); return; - } - - if (actionLogs && actionLogs.length) { - logger.debug("Enter get() for /instances/%s/actionLogs", req.params.instanceId); - res.send(actionLogs); } else { - logger.debug("Exit get() for /instances/%s/actionLogs", req.params.instanceId); - res.send([]); + res.status(200).send(actionLogs) } - }); }); diff --git a/server/app/services/instanceService.js b/server/app/services/instanceService.js index ed4555535..4fb8422f5 100644 --- a/server/app/services/instanceService.js +++ b/server/app/services/instanceService.js @@ -1,12 +1,12 @@ /* Copyright [2016] [Relevance Lab] - + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -67,10 +67,13 @@ instanceService.removeInstanceById = removeInstanceById; instanceService.removeInstancesByProviderId = removeInstancesByProviderId; instanceService.instanceSyncWithAWS = instanceSyncWithAWS; instanceService.updateScheduler = updateScheduler; +instanceService.parseInstanceMonitorQuery = parseInstanceMonitorQuery; +instanceService.getInstanceActionLogs = getInstanceActionLogs; +instanceService.parseActionLogsQuery = parseActionLogsQuery; function checkIfUnassignedInstanceExists(providerId, instanceId, callback) { unassignedInstancesModel.getById(instanceId, - function(err, instance) { + function (err, instance) { if (err) { var err = new Error('Internal server error'); err.status = 500; @@ -95,7 +98,7 @@ function validateListInstancesQuery(orgs, filterQuery, callback) { var queryObjectAndCondition = filterQuery.queryObj['$and'][0]; if (('orgName' in queryObjectAndCondition) && ('$in' in queryObjectAndCondition)) { - var validOrgs = queryObjectAndCondition['orgName']['$in'].filter(function(orgName) { + var validOrgs = queryObjectAndCondition['orgName']['$in'].filter(function (orgName) { return (orgName in orgs); }); @@ -104,7 +107,7 @@ function validateListInstancesQuery(orgs, filterQuery, callback) { err.status = 403; return callback(err); } else { - orgIds = validOrgs.reduce(function(a, b) { + orgIds = validOrgs.reduce(function (a, b) { return a.concat(orgs[b].rowid); }, orgIds); } @@ -117,7 +120,7 @@ function validateListInstancesQuery(orgs, filterQuery, callback) { return callback(err); } } else { - orgIds = Object.keys(orgs).reduce(function(a, b) { + orgIds = Object.keys(orgs).reduce(function (a, b) { return a.concat(orgs[b].rowid); }, orgIds); } @@ -127,10 +130,10 @@ function validateListInstancesQuery(orgs, filterQuery, callback) { if (orgIds.length > 0) { if (queryObjectAndCondition.providerId) { - filterQuery.queryObj['$and'][0].orgId = { '$in': orgIds } + filterQuery.queryObj['$and'][0].orgId = {'$in': orgIds} } else { - filterQuery.queryObj['$and'][0].providerId = { '$ne': null }; - filterQuery.queryObj['$and'][0].orgId = { '$in': orgIds }; + filterQuery.queryObj['$and'][0].providerId = {'$ne': null}; + filterQuery.queryObj['$and'][0].orgId = {'$in': orgIds}; } } @@ -138,7 +141,7 @@ function validateListInstancesQuery(orgs, filterQuery, callback) { } function getUnassignedInstancesByProvider(jsonData, callback) { - unassignedInstancesModel.getByProviderId(jsonData, function(err, assignedInstances) { + unassignedInstancesModel.getByProviderId(jsonData, function (err, assignedInstances) { if (err) { var err = new Error('Internal server error'); err.status = 500; @@ -160,10 +163,10 @@ function bulkUpdateInstanceProviderTags(provider, instances, callback) { } else { var unassignedInstances = []; for (var i = 0; i < instances.length; i++) { - (function(j) { + (function (j) { // @TODO replace with single query // @TODO Improve error handling - unassignedInstancesModel.getById(instances[j].id, function(err, unassignedInstance) { + unassignedInstancesModel.getById(instances[j].id, function (err, unassignedInstance) { if (err) { var err = new Error('Internal server error'); err.status = 500; @@ -225,14 +228,14 @@ function bulkUpdateAWSInstanceTags(provider, instances, callback) { } for (var i = 0; i < instances.length; i++) { - (function(j) { + (function (j) { awsSettings.region = instances[j].providerData.region; var ec2 = new EC2(awsSettings); logger.debug('Updating tags for instance ', instances[j]._id); // @TODO Improve error handling ec2.createTags(instances[j].platformId, instances[j].tags, - function(err, data) { + function (err, data) { if (err) { logger.error(err); if (err.code === 'AccessDenied') { @@ -256,7 +259,7 @@ function bulkUpdateUnassignedInstanceTags(instances, callback) { var catalystEntityTypes = appConfig.catalystEntityTypes; for (var i = 0; i < instances.length; i++) { - (function(j) { + (function (j) { var params = { '_id': instances[j]._id } @@ -264,7 +267,7 @@ function bulkUpdateUnassignedInstanceTags(instances, callback) { 'tags': instances[j].tags } unassignedInstancesModel.updateInstance(params, fields, - function(err, instanceUpdated) { + function (err, instanceUpdated) { if (err) { logger.error(err); var err = new Error('Internal server error'); @@ -283,7 +286,7 @@ function updateUnassignedInstanceProviderTags(provider, instanceId, tags, callba var providerTypes = appConfig.providerTypes; unassignedInstancesModel.getById(instanceId, - function(err, instance) { + function (err, instance) { if (err) { logger.error(err); var err = new Error('Internal server error'); @@ -341,7 +344,7 @@ function updateAWSInstanceTag(provider, instance, tags, callback) { ec2.createTags(instance.platformId, tags, - function(err, data) { + function (err, data) { if (err) { logger.error(err); if (err.code === 'AccessDenied') { @@ -366,7 +369,7 @@ function updateUnassignedInstanceTags(instance, tags, tagMappingsList, callback) var catalystEntityTypes = appConfig.catalystEntityTypes; var tagMappings = {}; - tagMappingsList.forEach(function(tagMapping) { + tagMappingsList.forEach(function (tagMapping) { tagMappings[tagMapping.name] = tagMapping; }); @@ -398,7 +401,7 @@ function updateUnassignedInstanceTags(instance, tags, tagMappingsList, callback) }; unassignedInstancesModel.updateInstance(params, fields, - function(err, instanceUpdated) { + function (err, instanceUpdated) { if (err) { logger.error(err); var err = new Error('Internal server error'); @@ -413,18 +416,18 @@ function updateUnassignedInstanceTags(instance, tags, tagMappingsList, callback) function getTrackedInstancesForProvider(provider, next) { async.parallel({ - managed: function(callback) { - instancesModel.getInstanceByProviderId(provider._id, callback); - }, - unmanaged: function(callback) { - //@TODO Duplicate function of getByProviderId, to be cleaned up - unManagedInstancesModel.getInstanceByProviderId(provider._id, callback); - }, - unassigned: function(callback) { - unassignedInstancesModel.getUnAssignedInstancesByProviderId(provider._id, callback); - } + managed: function (callback) { + instancesModel.getInstanceByProviderId(provider._id, callback); + }, + unmanaged: function (callback) { + //@TODO Duplicate function of getByProviderId, to be cleaned up + unManagedInstancesModel.getInstanceByProviderId(provider._id, callback); }, - function(err, results) { + unassigned: function (callback) { + unassignedInstancesModel.getUnAssignedInstancesByProviderId(provider._id, callback); + } + }, + function (err, results) { if (err) { var err = new Error('Internal server error'); err.status = 500; @@ -438,25 +441,25 @@ function getTrackedInstancesForProvider(provider, next) { function getTrackedInstances(query, category, next) { async.parallel([ - function(callback) { - if (category === 'managed') { - instancesModel.getAll(query, callback); - } else if (category === 'assigned') { - unManagedInstancesModel.getAll(query, callback); - } else if (category === 'unassigned') { - unassignedInstancesModel.getAll(query, callback); - } else { - callback(null, [{ docs: [], total: 0 }]); - } + function (callback) { + if (category === 'managed') { + instancesModel.getAll(query, callback); + } else if (category === 'assigned') { + unManagedInstancesModel.getAll(query, callback); + } else if (category === 'unassigned') { + unassignedInstancesModel.getAll(query, callback); + } else { + callback(null, [{docs: [], total: 0}]); } - ], - function(err, results) { + } + ], + function (err, results) { if (err) { var err = new Error('Internal server error'); err.status = 500; next(err) } else { - var instances = results.reduce(function(a, b) { + var instances = results.reduce(function (a, b) { return a.concat(b); }, []); @@ -491,7 +494,7 @@ function createUnassignedInstancesList(instances, callback) { var instancesListObject = {}; var instancesList = []; - instances.forEach(function(instance) { + instances.forEach(function (instance) { // @TODO Copy object, reduce code var tempInstance = {}; var provider = { @@ -520,7 +523,7 @@ function createTrackedInstancesResponse(instances, callback) { var instancesListObject = {}; var instancesList = []; - instancesListObject.trackedInstances = instances.map(function(instance) { + instancesListObject.trackedInstances = instances.map(function (instance) { var instanceObj = {}; instanceObj.id = instance._id; instanceObj.category = ('chefNodeName' in instance) ? 'managed' : 'unmanaged'; @@ -579,7 +582,7 @@ instanceService.createInstance = function createInstance(instanceObj, callback) } var buffer = new Buffer(instanceObj.provider.providerDetails.sshPrivateKey, 'base64'); var toAscii = buffer.toString('ascii'); - fs.writeFile(tempLocation, toAscii, function(err) { + fs.writeFile(tempLocation, toAscii, function (err) { if (err) { var error = new Error("Unable to create pem file."); error.status = 500; @@ -590,8 +593,8 @@ instanceService.createInstance = function createInstance(instanceObj, callback) username: blueprint.vmImage.userName, pemFileLocation: tempLocation, password: blueprint.vmImage.password - }, function(err, encryptedCredential) { - fs.unlink(tempLocation, function() { + }, function (err, encryptedCredential) { + fs.unlink(tempLocation, function () { logger.debug('temp file deleted'); }) if (err) { @@ -647,7 +650,7 @@ instanceService.createInstance = function createInstance(instanceObj, callback) break; defaut: break; } - instancesModel.createInstance(instances, function(err, instanceData) { + instancesModel.createInstance(instances, function (err, instanceData) { if (err) { logger.debug("Failed to createInstance.", err); var error = new Error("Failed to createInstance."); @@ -663,8 +666,8 @@ instanceService.createInstance = function createInstance(instanceObj, callback) instanceService.bootstrapInstance = function bootstrapInstance(bootstrapData, callback) { var blueprintObj = bootstrapData.blueprint; - credentialCryptography.decryptCredential(bootstrapData.credentials, function(err, decryptedCredential) { - configmgmtDao.getEnvNameFromEnvId(bootstrapData.envId, function(err, envName) { + credentialCryptography.decryptCredential(bootstrapData.credentials, function (err, decryptedCredential) { + configmgmtDao.getEnvNameFromEnvId(bootstrapData.envId, function (err, envName) { if (err) { callback({ message: "Failed to get env name from env id" @@ -678,7 +681,7 @@ instanceService.bootstrapInstance = function bootstrapInstance(bootstrapData, ca return; } - getCookBookAttributes(bootstrapData, function(err, jsonAttributes) { + getCookBookAttributes(bootstrapData, function (err, jsonAttributes) { logger.debug("jsonAttributes::::: ", JSON.stringify(jsonAttributes)); var runlist = bootstrapData.runlist; //logger.debug("launchParams.blueprintData.extraRunlist: ", JSON.stringify(launchParams.blueprintData.extraRunlist)); @@ -696,7 +699,7 @@ instanceService.bootstrapInstance = function bootstrapInstance(bootstrapData, ca jsonAttributes: jsonAttributes, instancePassword: decryptedCredential.password }; - configmgmtDao.getChefServerDetails(bootstrapData.chef.serverId, function(err, chefDetails) { + configmgmtDao.getChefServerDetails(bootstrapData.chef.serverId, function (err, chefDetails) { if (err) { var error = new Error("Failed to getChefServerDetails"); error.status = 500; @@ -715,7 +718,7 @@ instanceService.bootstrapInstance = function bootstrapInstance(bootstrapData, ca hostedChefUrl: chefDetails.url }); - chef.getEnvironment(envName, function(err, env) { + chef.getEnvironment(envName, function (err, env) { if (err) { var error = new Error("Failed chef.getEnvironment"); error.status = 500; @@ -723,7 +726,7 @@ instanceService.bootstrapInstance = function bootstrapInstance(bootstrapData, ca } if (!env) { - chef.createEnvironment(envName, function(err) { + chef.createEnvironment(envName, function (err) { if (err) { logger.error("Failed chef.createEnvironment", err); var error = new Error("Failed to create environment in chef server."); @@ -740,7 +743,7 @@ instanceService.bootstrapInstance = function bootstrapInstance(bootstrapData, ca var timestampStarted = new Date().getTime(); var actionLog = instancesModel.insertBootstrapActionLog(bootstrapData.id, bootstrapData.runlist, "admin", timestampStarted); var logsReferenceIds = [bootstrapData.id, actionLog._id]; - chef.bootstrapInstance(bootstrapInstanceParams, function(err, code) { + chef.bootstrapInstance(bootstrapInstanceParams, function (err, code) { if (bootstrapInstanceParams.pemFilePath) { //fs.unlink(bootstrapInstanceParams.pemFilePath); } @@ -748,7 +751,7 @@ instanceService.bootstrapInstance = function bootstrapInstance(bootstrapData, ca logger.error('process stopped ==> ', err, code); if (err) { logger.error("knife launch err ==>", err); - instancesModel.updateInstanceBootstrapStatus(bootstrapData.id, 'failed', function(err, updateData) {}); + instancesModel.updateInstanceBootstrapStatus(bootstrapData.id, 'failed', function (err, updateData) {}); var timestampEnded = new Date().getTime(); logsDao.insertLog({ referenceId: logsReferenceIds, @@ -760,7 +763,7 @@ instanceService.bootstrapInstance = function bootstrapInstance(bootstrapData, ca return callback(err); } else { if (code == 0) { - instancesModel.updateInstanceBootstrapStatus(bootstrapData.id, 'success', function(err, updateData) { + instancesModel.updateInstanceBootstrapStatus(bootstrapData.id, 'success', function (err, updateData) { if (err) { logger.error("Unable to set instance bootstarp status. code 0", err); } else { @@ -777,7 +780,7 @@ instanceService.bootstrapInstance = function bootstrapInstance(bootstrapData, ca instancesModel.updateActionLog(bootstrapData.id, actionLog._id, true, timestampEnded); - chef.getNode(bootstrapData.chefNodeName, function(err, nodeData) { + chef.getNode(bootstrapData.chefNodeName, function (err, nodeData) { if (err) { logger.error("Failed chef.getNode", err); var error = new Error("Failed to get Node from chef server."); @@ -797,7 +800,7 @@ instanceService.bootstrapInstance = function bootstrapInstance(bootstrapData, ca hardwareData.memory.free = nodeData.automatic.memory.free; } hardwareData.os = bootstrapData.hardware.os; - instancesModel.setHardwareDetails(bootstrapData.id, hardwareData, function(err, updateData) { + instancesModel.setHardwareDetails(bootstrapData.id, hardwareData, function (err, updateData) { if (err) { logger.error("Unable to set instance hardware details code (setHardwareDetails)", err); } else { @@ -806,7 +809,7 @@ instanceService.bootstrapInstance = function bootstrapInstance(bootstrapData, ca }); //Checking docker status and updating var docker = new Docker(); - docker.checkDockerStatus(bootstrapData.id, function(err, retCode) { + docker.checkDockerStatus(bootstrapData.id, function (err, retCode) { if (err) { logger.error("Failed docker.checkDockerStatus", err); var error = new Error("Failed to check Status from Docker."); @@ -816,7 +819,7 @@ instanceService.bootstrapInstance = function bootstrapInstance(bootstrapData, ca } logger.debug('Docker Check Returned:' + retCode); if (retCode == '0') { - instancesModel.updateInstanceDockerStatus(bootstrapData.id, "success", '', function(data) { + instancesModel.updateInstanceDockerStatus(bootstrapData.id, "success", '', function (data) { logger.debug('Instance Docker Status set to Success'); }); } @@ -828,7 +831,7 @@ instanceService.bootstrapInstance = function bootstrapInstance(bootstrapData, ca }); } else { - instancesModel.updateInstanceBootstrapStatus(bootstrapData.id, 'failed', function(err, updateData) { + instancesModel.updateInstanceBootstrapStatus(bootstrapData.id, 'failed', function (err, updateData) { if (err) { logger.error("Unable to set instance bootstarp status code != 0", err); } else { @@ -850,7 +853,7 @@ instanceService.bootstrapInstance = function bootstrapInstance(bootstrapData, ca } } - }, function(stdOutData) { + }, function (stdOutData) { logsDao.insertLog({ referenceId: logsReferenceIds, @@ -859,7 +862,7 @@ instanceService.bootstrapInstance = function bootstrapInstance(bootstrapData, ca timestamp: new Date().getTime() }); - }, function(stdErrData) { + }, function (stdErrData) { //retrying 4 times before giving up. logsDao.insertLog({ @@ -886,7 +889,7 @@ function getCookBookAttributes(instance, callback) { var attr = []; if (instance.runlist && instance.runlist.length) { for (var j = 0; j < instance.runlist.length; j++) { - (function(j) { + (function (j) { // Attributes which are configures in blueprint. attr = instance.runlist[j].attributes; if (attr && attr.length) { @@ -917,7 +920,7 @@ function getCookBookAttributes(instance, callback) { } }); - nexus.getNexusArtifactVersions(blueprint.applications[0].repoId, repoName, groupId, artifactId, function(err, data) { + nexus.getNexusArtifactVersions(blueprint.applications[0].repoId, repoName, groupId, artifactId, function (err, data) { if (err) { logger.debug("Failed to fetch Repository from Mongo: ", err); objectArray.push({ @@ -939,7 +942,7 @@ function getCookBookAttributes(instance, callback) { var latestVersion = versions[latestVersionIndex - 1]; //logger.debug("Got latest catalyst version from nexus: ", latestVersion); - nexus.getNexusArtifact(blueprint.applications[0].repoId, repoName, groupId, function(err, artifacts) { + nexus.getNexusArtifact(blueprint.applications[0].repoId, repoName, groupId, function (err, artifacts) { if (err) { logger.debug("Failed to get artifacts."); objectArray.push({ @@ -1010,7 +1013,7 @@ function getCookBookAttributes(instance, callback) { // Update app-data for promote var nodeIp = []; nodeIp.push(instance.instanceIP); - configmgmtDao.getEnvNameFromEnvId(instance.envId, function(err, envName) { + configmgmtDao.getEnvNameFromEnvId(instance.envId, function (err, envName) { if (err) { callback({ message: "Failed to get env name from env id" @@ -1033,7 +1036,7 @@ function getCookBookAttributes(instance, callback) { "nodeIps": nodeIp } }; - AppData.createNewOrUpdate(appData, function(err, data) { + AppData.createNewOrUpdate(appData, function (err, data) { if (err) { logger.debug("Failed to create or update app-data: ", err); } @@ -1131,7 +1134,7 @@ function getCookBookAttributes(instance, callback) { // Update app-data for promote var nodeIp = []; nodeIp.push(instance.instanceIP); - configmgmtDao.getEnvNameFromEnvId(instance.envId, function(err, envName) { + configmgmtDao.getEnvNameFromEnvId(instance.envId, function (err, envName) { if (err) { callback({ message: "Failed to get env name from env id" @@ -1164,7 +1167,7 @@ function getCookBookAttributes(instance, callback) { "version": blueprint.applications[0].repoDetails.imageTag, "docker": actualDocker }; - AppData.createNewOrUpdate(appData, function(err, data) { + AppData.createNewOrUpdate(appData, function (err, data) { if (err) { logger.debug("Failed to create or update app-data: ", err); } @@ -1182,11 +1185,12 @@ function getCookBookAttributes(instance, callback) { callback(null, attributeObj); return; } -}; +} +; instanceService.getInstanceActionList = function getInstanceActionList(callback) { - instancesDao.listInstances(function(err, list) { + instancesDao.listInstances(function (err, list) { if (err) { logger.debug("Error while fetching instance actionLog: ", err); return callback(err, null); @@ -1195,7 +1199,7 @@ instanceService.getInstanceActionList = function getInstanceActionList(callback) if (list && list.length) { var actionLogs = []; for (var i = 0; i < list.length; i++) { - (function(i) { + (function (i) { if (list[i].instanceState != "terminated" && list[i].actionLogs && list[i].actionLogs.length) { for (var j = 0; j < list[i].actionLogs.length; j++) { if (list[i].actionLogs[j].name != "Orchestration") { @@ -1232,14 +1236,14 @@ instanceService.getInstanceActionList = function getInstanceActionList(callback) }; instanceService.getInstanceAction = function getInstanceAction(actionId, callback) { - instancesDao.getActionLogsById(actionId, function(err, action) { + instancesDao.getActionLogsById(actionId, function (err, action) { if (err) { logger.error("Failed to fetch Instance Action: ", err); return callback(err, null); } if (action && action.length) { var instanceAction = JSON.parse(JSON.stringify(action[0].actionLogs[0])); - logsDao.getLogsByReferenceId(actionId, null, function(err, data) { + logsDao.getLogsByReferenceId(actionId, null, function (err, data) { if (err) { logger.error("Failed to fetch Logs: ", err); callback(500, null); @@ -1258,13 +1262,13 @@ instanceService.getInstanceAction = function getInstanceAction(actionId, callbac }; function removeInstanceById(instanceId, callback) { - containerModel.deleteContainerByInstanceId(instanceId, function(err, container) { + containerModel.deleteContainerByInstanceId(instanceId, function (err, container) { if (err) { logger.error("Container deletion Failed >> ", err); callback(err, null); return; } else { - instancesModel.removeInstanceById(instanceId, function(err, data) { + instancesModel.removeInstanceById(instanceId, function (err, data) { if (err) { logger.error("Instance deletion Failed >> ", err); callback(err, null); @@ -1274,32 +1278,33 @@ function removeInstanceById(instanceId, callback) { }); } }); -}; +} +; function removeInstancesByProviderId(providerId, callback) { async.parallel({ - managedInstance: function(callback) { + managedInstance: function (callback) { instancesModel.removeInstancesByProviderId(providerId, callback); }, - assignedInstance: function(callback) { + assignedInstance: function (callback) { unManagedInstancesModel.removeInstancesByProviderId(providerId, callback); }, - unassignedInstance: function(callback) { + unassignedInstance: function (callback) { unassignedInstancesModel.removeInstancesByProviderId(providerId, callback); }, - resources: function(callback) { + resources: function (callback) { resources.removeResourcesByProviderId(providerId, callback); }, - resourcesCost: function(callback) { + resourcesCost: function (callback) { resourceCost.removeResourceCostByProviderId(providerId, callback); }, - resourcesUsage: function(callback) { + resourcesUsage: function (callback) { resourceUsage.removeResourceUsageByProviderId(providerId, callback); }, - resourcesTags: function(callback) { + resourcesTags: function (callback) { tagsModel.removeTagsByProviderId(providerId, callback); } - }, function(err, results) { + }, function (err, results) { if (err) { callback(err, null); } else { @@ -1310,10 +1315,10 @@ function removeInstancesByProviderId(providerId, callback) { function instanceSyncWithAWS(instanceId, instanceData, providerDetails, callback) { async.waterfall([ - function(next) { + function (next) { instancesModel.getInstanceById(instanceId, next); }, - function(instances, next) { + function (instances, next) { var instance = instances[0]; var routeHostedZoneParamList = []; if (instance.instanceState !== instanceData.state && instance.bootStrapStatus === 'success') { @@ -1331,9 +1336,10 @@ function instanceSyncWithAWS(instanceId, instanceData, providerDetails, callback action = 'Shutting-Down'; } else { action = 'Start'; - }; + } + ; if (instanceData.state === 'terminated' && instance.instanceState === 'shutting-down') { - instanceLogModel.getLogsByInstanceIdStatus(instance._id, instance.instanceState, function(err, data) { + instanceLogModel.getLogsByInstanceIdStatus(instance._id, instance.instanceState, function (err, data) { if (err) { logger.error("Failed to get Instance Logs: ", err); next(err, null); @@ -1349,7 +1355,7 @@ function instanceSyncWithAWS(instanceId, instanceData, providerDetails, callback log: "Instance " + instanceData.state, timestamp: timestampStarted }); - instanceLogModel.createOrUpdate(data.actionId, instance._id, data, function(err, logData) { + instanceLogModel.createOrUpdate(data.actionId, instance._id, data, function (err, logData) { if (err) { logger.error("Failed to create or update instanceLog: ", err); next(err, null); @@ -1364,12 +1370,12 @@ function instanceSyncWithAWS(instanceId, instanceData, providerDetails, callback next(null, routeHostedZoneParamList); } }, - function(paramList, next) { + function (paramList, next) { async.parallel({ - instanceDataSync: function(callback) { + instanceDataSync: function (callback) { instancesModel.updateInstanceStatus(instanceId, instanceData, callback); }, - route53Sync: function(callback) { + route53Sync: function (callback) { if (paramList.length > 0) { var cryptoConfig = appConfig.cryptoSettings; var cryptography = new Cryptography(cryptoConfig.algorithm, cryptoConfig.password); @@ -1385,9 +1391,9 @@ function instanceSyncWithAWS(instanceId, instanceData, providerDetails, callback var route53 = new Route53(route53Config); var count = 0; for (var i = 0; i < paramList.length; i++) { - (function(params) { + (function (params) { params.ChangeBatch.Changes[0].Action = 'DELETE'; - route53.changeResourceRecordSets(params, function(err, data) { + route53.changeResourceRecordSets(params, function (err, data) { count++; if (err) { callback(err, null); @@ -1403,7 +1409,7 @@ function instanceSyncWithAWS(instanceId, instanceData, providerDetails, callback callback(null, paramList); } } - }, function(err, results) { + }, function (err, results) { if (err) { next(err); } else { @@ -1412,7 +1418,7 @@ function instanceSyncWithAWS(instanceId, instanceData, providerDetails, callback }) } - ], function(err, results) { + ], function (err, results) { if (err) { callback(err, null); } else { @@ -1456,7 +1462,7 @@ function createOrUpdateInstanceLogs(instance, instanceState, action, user, times action: action, logs: [] }; - instanceLogModel.createOrUpdate(actionLog._id, instance._id, instanceLog, function(err, logData) { + instanceLogModel.createOrUpdate(actionLog._id, instance._id, instanceLog, function (err, logData) { if (err) { logger.error("Failed to create or update instanceLog: ", err); next(err, null); @@ -1467,17 +1473,17 @@ function createOrUpdateInstanceLogs(instance, instanceState, action, user, times function updateScheduler(instanceScheduler, callback) { async.waterfall([ - function(next){ - generateCronPattern(instanceScheduler.interval,instanceScheduler.schedulerStartOn,instanceScheduler.schedulerEndOn,next); + function (next) { + generateCronPattern(instanceScheduler.interval, instanceScheduler.schedulerStartOn, instanceScheduler.schedulerEndOn, next); }, - function(schedulerDetails,next){ + function (schedulerDetails, next) { schedulerDetails.interval = instanceScheduler.interval; - instancesDao.updateScheduler(instanceScheduler.instanceIds, schedulerDetails,next); + instancesDao.updateScheduler(instanceScheduler.instanceIds, schedulerDetails, next); } - ],function(err,results){ - if(err){ + ], function (err, results) { + if (err) { return callback(err, null); - }else{ + } else { callback(null, {"message": "Scheduler Updated."}); catalystSync.executeScheduledInstances(); return; @@ -1485,77 +1491,133 @@ function updateScheduler(instanceScheduler, callback) { }); } -function generateCronPattern(cronInterval,startDate,endDate,callback){ - var startIntervalList =[],stopIntervalList=[],count = 0; - if(cronInterval.length === 0){ - var scheduler= { +function parseInstanceMonitorQuery(paginationReq, callback) { + if (paginationReq.filterBy && paginationReq.filterBy.monitor) { + if (paginationReq.filterBy.monitor === "true") { + paginationReq.filterBy.monitor = {$ne: null}; + } else { + paginationReq.filterBy.monitor = null; + } + } + return callback(null, paginationReq); +} + +function getInstanceActionLogs(instanceId, filterByQuery, callback) { + logger.debug("filterByQuery------>>>>", JSON.stringify(filterByQuery)); + instancesDao.getAllActionLogs(instanceId, filterByQuery, function (err, actionLogs) { + if (err) { + callback(err); + return; + } + + if (actionLogs && actionLogs.length) { + logger.debug("Enter get() for /instances/%s/actionLogs", instanceId); + callback(null, actionLogs); + } else { + logger.debug("Exit get() for /instances/%s/actionLogs", instanceId); + callback(null, []); + } + + }); +} + +function parseActionLogsQuery(requestQuery, callback) { + logger.debug("requestQuery------>>>>", JSON.stringify(requestQuery)); + var query = {}; + if (requestQuery.fromTime || requestQuery.toTime) { + query = { + "actionLogs": { + "$elemMatch": { + "timeStarted": { + } + } + } + }; + if (requestQuery.fromTime) { + query.actionLogs.$elemMatch.timeStarted['$gte'] = Date.parse(requestQuery.fromTime); + } + if (requestQuery.toTime) { + query.actionLogs.$elemMatch.timeStarted['$lte'] = Date.parse(requestQuery.toTime); + } + } + return callback(null, query); +} + +function generateCronPattern(cronInterval, startDate, endDate, callback) { + var startIntervalList = [], stopIntervalList = [], count = 0; + if (cronInterval.length === 0) { + var scheduler = { instanceStartScheduler: startIntervalList, instanceStopScheduler: stopIntervalList, schedulerStartOn: Date.parse(startDate), schedulerEndOn: Date.parse(endDate), isScheduled: false } - callback(null,scheduler); + callback(null, scheduler); return; - }else{ - for(var i = 0; i < cronInterval.length; i++){ - (function(interval){ - if(interval.action==='start'){ + } else { + for (var i = 0; i < cronInterval.length; i++) { + (function (interval) { + if (interval.action === 'start') { count++; var timeSplit = interval.time.split(":"); var hours = parseInt(timeSplit[0]); var minutes = parseInt(timeSplit[1]); - var sortedDays = interval.days.sort(function(a, b){return a-b}); + var sortedDays = interval.days.sort(function (a, b) { + return a - b + }); var strDays = ''; - for(var j = 0; j < sortedDays.length; j++){ - if(strDays !== '') - strDays = strDays+','+sortedDays[j]; + for (var j = 0; j < sortedDays.length; j++) { + if (strDays !== '') + strDays = strDays + ',' + sortedDays[j]; else strDays = sortedDays[j]; } startIntervalList.push({ - cronTime:interval.time, - cronDays:sortedDays, - cronPattern:minutes +' '+ hours +' '+ '* * '+ strDays + cronTime: interval.time, + cronDays: sortedDays, + cronPattern: minutes + ' ' + hours + ' ' + '* * ' + strDays }); - if(count === cronInterval.length){ - var scheduler= { + if (count === cronInterval.length) { + var scheduler = { instanceStartScheduler: startIntervalList, instanceStopScheduler: stopIntervalList, schedulerStartOn: Date.parse(startDate), schedulerEndOn: Date.parse(endDate), isScheduled: true } - callback(null,scheduler); + callback(null, scheduler); return; } - }else{ + } else { count++; var timeSplit = interval.time.split(":"); var hours = parseInt(timeSplit[0]); var minutes = parseInt(timeSplit[1]); - var sortedDays = interval.days.sort(function(a, b){return a-b}); + var sortedDays = interval.days.sort(function (a, b) { + return a - b + }); var strDays = ''; - for(var j = 0; j < sortedDays.length; j++){ - if(strDays !== '') - strDays = strDays+','+sortedDays[j]; + for (var j = 0; j < sortedDays.length; j++) { + if (strDays !== '') + strDays = strDays + ',' + sortedDays[j]; else strDays = sortedDays[j]; } stopIntervalList.push({ - cronTime:interval.time, - cronDays:sortedDays, - cronPattern:minutes +' '+ hours +' '+ '* * '+ strDays + cronTime: interval.time, + cronDays: sortedDays, + cronPattern: minutes + ' ' + hours + ' ' + '* * ' + strDays }); - if(count === cronInterval.length){ - var scheduler= { + if (count === cronInterval.length) { + var scheduler = { instanceStartScheduler: startIntervalList, instanceStopScheduler: stopIntervalList, schedulerStartOn: Date.parse(startDate), schedulerEndOn: Date.parse(endDate), isScheduled: true } - callback(null,scheduler); + callback(null, scheduler); return; } }