diff --git a/server/app/cronjobs/aws-cost-aggregation/AWSResourceCostsAggregation.js b/server/app/cronjobs/aws-cost-aggregation/AWSResourceCostsAggregation.js index 257661091..8fe204bc3 100644 --- a/server/app/cronjobs/aws-cost-aggregation/AWSResourceCostsAggregation.js +++ b/server/app/cronjobs/aws-cost-aggregation/AWSResourceCostsAggregation.js @@ -50,6 +50,7 @@ AWSResourceCostsAggregation.aggregateEntityCostsByOrg = aggregateEntityCostsByOr AWSResourceCostsAggregation.aggregateEntityCostsByProvider = aggregateEntityCostsByProvider AWSResourceCostsAggregation.aggregateEntityCostTrendByOrg = aggregateEntityCostTrendByOrg AWSResourceCostsAggregation.aggregateEntityCostTrendByProvider = aggregateEntityCostTrendByProvider +AWSResourceCostsAggregation.aggregateResourceCostsForAllPeriods = aggregateResourceCostsForAllPeriods // AWSResourceCostsAggregation.execute() @@ -206,8 +207,7 @@ function updateResourceCosts(provider, downloadedCSVPath, callback) { resourceService.getAllResourcesForProvider(provider, next) }, function(resources, next) { - resourceService.updateAWSResourceCostsFromCSV(provider, resources, downloadedCSVPath, - AWSResourceCostsAggregation.currentCronRunTime, next) + AWSResourceCostsAggregation.aggregateResourceCostsForAllPeriods(provider, resources, next) }, function(next) { AWSProvider.updateLastBillUpdateTime(provider._id, @@ -223,6 +223,13 @@ function updateResourceCosts(provider, downloadedCSVPath, callback) { }) } +// Only aggregates monthly cost as of now and updates resource/instance collections +// @TODO To be aggregated for all periods +function aggregateResourceCostsForAllPeriods(provider, resources, callback) { + resourceService.aggregateResourceCostsForPeriod(provider, resources, 'month', + AWSResourceCostsAggregation.currentCronRunTime, callback) +} + function aggregateEntityCostsByOrg(orgs, callback) { // var catalystEntityHierarchy = appConfig.catalystEntityHierarchy var costAggregationPeriods = appConfig.costAggregationPeriods diff --git a/server/app/model/resource-costs/resource-costs.js b/server/app/model/resource-costs/resource-costs.js index 9a0761fd4..2c8f9a306 100644 --- a/server/app/model/resource-costs/resource-costs.js +++ b/server/app/model/resource-costs/resource-costs.js @@ -77,7 +77,7 @@ var ResourceCostsSchema = new Schema({ }, billLineItemId: { type: Number, - required: true, + required: false, trim: true }, platformDetails: { @@ -137,8 +137,6 @@ var ResourceCostsSchema = new Schema({ }) ResourceCostsSchema.index({'platformDetails.serviceId' : 1}) -ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'startTime': 1, - 'billIntervalId': 1, 'billLineItemId': 1, 'interval': 1}, {'unique': true}) ResourceCostsSchema.statics.save = function save(resourceCostData, callback) { var resourceCosts = new ResourceCosts(resourceCostData) diff --git a/server/app/services/resourceService.js b/server/app/services/resourceService.js index 20a38dafe..e9a54edc6 100644 --- a/server/app/services/resourceService.js +++ b/server/app/services/resourceService.js @@ -47,6 +47,7 @@ resourceService.getEC2InstancesInfo=getEC2InstancesInfo; resourceService.getAllResourcesForProvider = getAllResourcesForProvider; resourceService.updateAWSResourceCostsFromCSV = updateAWSResourceCostsFromCSV resourceService.updateDomainNameForInstance = updateDomainNameForInstance +resourceService.aggregateResourceCostsForPeriod = aggregateResourceCostsForPeriod // @TODO To be cached if needed. In memory data will not exceed 200MB for upto 2000 instances. function getAllResourcesForProvider(provider, next) { @@ -193,10 +194,56 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u if(err) { callback(err) } else { - callback() + callback(null, resources) } }) +} + +function aggregateResourceCostsForPeriod(provider, resources, period, endTime, callback) { + var catalystEntityHierarchy = appConfig.catalystEntityHierarchy + var date = new Date() + var billIntervalId = date.getFullYear() + '-' + (date.getMonth() + 1) + + var offset = (new Date()).getTimezoneOffset()*60000 + var startTime = dateUtil.getStartOfPeriod(period, endTime) + var query = { 'providerId': provider._id.toString() } + query.startTime = {$gte: Date.parse(startTime) + offset} + query.endTime = {$lte: Date.parse(endTime) + offset} + + async.waterfall([ + function(next) { + resourceCost.aggregate([ + {$match: query}, + {$group: {_id: "$" + catalystEntityHierarchy['resource'].key, + totalCost: {$sum: "$cost"}}} + ],next) + }, + function(resourceCosts, next) { + async.forEach(resourceCosts, function(resourceCost, next1) { + if(resourceCost._id in resources) { + resources[resourceCost._id].cost = { + aggregateInstanceCost: Math.round(resourceCost.totalCost * 100) / 100, + currency:'USD', + symbol:"$" + } + resources[resourceCost._id].save(next1) + } + }, function(err) { + if(err) { + next(err) + } else { + next() + } + }) + } + ], function(err) { + if(err) { + callback(err) + } else { + callback() + } + }) } function getCostForServices_deprecated(provider,callback) { diff --git a/server/install.js b/server/install.js index 06b7d6b7b..4e15ebef8 100755 --- a/server/install.js +++ b/server/install.js @@ -471,7 +471,7 @@ function getDefaultsConfig() { key: 'platformDetails.region' }, resource: { - key: 'resourceId' + key: 'platformDetails.instanceId' } }, costAggregationPeriods: {