diff --git a/server/app/cronjobs/aws-cost-aggregation/AWSResourceCostsAggregation.js b/server/app/cronjobs/aws-cost-aggregation/AWSResourceCostsAggregation.js index cab81a0b6..293dd2943 100644 --- a/server/app/cronjobs/aws-cost-aggregation/AWSResourceCostsAggregation.js +++ b/server/app/cronjobs/aws-cost-aggregation/AWSResourceCostsAggregation.js @@ -175,17 +175,18 @@ function downloadLatestBill(provider, callback) { } }, function(billDownloaded, next) { + var downloadedCSVPath = appConfig.aws.s3BucketDownloadFileLocation + + AWSResourceCostsAggregation.csvFileName + if(billDownloaded) { var downloadedZipPath = appConfig.aws.s3BucketDownloadFileLocation + appConfig.aws.s3BucketFileName var zip = new AdmZip(downloadedZipPath) zip.extractAllTo(appConfig.aws.s3BucketDownloadFileLocation, true) - var downloadedCSVPath = appConfig.aws.s3BucketDownloadFileLocation - + AWSResourceCostsAggregation.csvFileName next(null, downloadedCSVPath) } else { - next(null, null) + next(null, downloadedCSVPath) } } ], diff --git a/server/app/model/entity-costs/entity-costs.js b/server/app/model/entity-costs/entity-costs.js index dda01b2de..1a9ef742e 100644 --- a/server/app/model/entity-costs/entity-costs.js +++ b/server/app/model/entity-costs/entity-costs.js @@ -67,6 +67,9 @@ var EntityCostsSchema = new Schema({ } }) +EntityCostsSchema.index({'entity.id': 1, 'entity.type': 1, 'parentEntity.id': 1, 'startTime': 1, + 'period': 1}, {'unique': true}) + EntityCostsSchema.statics.saveEntityCost = function saveEntityCost(entityCostData, callback) { var entityCosts = new EntityCosts(entityCostData) entityCosts.save(function(err, data) { diff --git a/server/app/model/resource-costs/resource-costs.js b/server/app/model/resource-costs/resource-costs.js index d692c961d..cc2b75e85 100644 --- a/server/app/model/resource-costs/resource-costs.js +++ b/server/app/model/resource-costs/resource-costs.js @@ -121,12 +121,12 @@ var ResourceCostsSchema = new Schema({ type: Number, required: true } -}); -/*ResourceCostsSchema.index({'platformDetails.instanceId': 1, 'platformDetails.usageType': 1, - 'startTime': 1, 'endTime': 1}, {'unique': true, 'sparse': true, 'partialFilterExpression': - {'platformDetails.usageType': { $exists: true }}})*/ +}) -ResourceCostsSchema.statics.saveResourceCost = function(resourceCostData, callback) { +ResourceCostsSchema.index({'organizationId': 1, 'resourceId': 1, 'billLineRecordId': 1, + 'startTime': 1, 'interval': 1}, {'unique': true}) + +ResourceCostsSchema.statics.saveResourceCost = function saveResourceCost(resourceCostData, callback) { var resourceCosts = new ResourceCosts(resourceCostData) resourceCosts.save(function(err, data) { if (err) { @@ -137,5 +137,24 @@ ResourceCostsSchema.statics.saveResourceCost = function(resourceCostData, callba }) } +ResourceCostsSchema.statics.upsertResourceCost = function upsertResourceCost(resourceCostData, callback) { + var query = { + organizationId: resourceCostData.organizationId, + resourceId: resourceCostData.billLineRecordId, + billLineRecordId: resourceCostData.billLineRecordId, + startTime: resourceCostData.startTime, + interval: resourceCostData.interval + } + + this.findOneAndUpdate(query, resourceCostData, {upsert:true}, + function(err, result){ + if (err) { + callback(null) + } else { + callback(null, result) + } + }); +} + var ResourceCosts = mongoose.model('ResourceCosts', ResourceCostsSchema) module.exports = ResourceCosts \ No newline at end of file diff --git a/server/app/services/analyticsService.js b/server/app/services/analyticsService.js index 398340b94..9c8770394 100644 --- a/server/app/services/analyticsService.js +++ b/server/app/services/analyticsService.js @@ -97,7 +97,9 @@ analyticsService.aggregateEntityCosts if(err) { next0(err) } else { + //@TODO Blocking call to be avoided var entityCosts = {} + for(var i = 0; i < aggregateCosts.totalCosts.length; i++) { entityCosts[aggregateCosts.totalCosts[i]._id] = { entity: { @@ -122,6 +124,7 @@ analyticsService.aggregateEntityCosts } } + // @TODO Blocking call to be avoided for(var i = 0; i < aggregateCosts.serviceCosts.length; i++) { for(var j = 0; j < aggregateCosts.serviceCosts[i].length; j++) { if(aggregateCosts.serviceCosts[i][j]._id in entityCosts) { @@ -289,17 +292,25 @@ analyticsService.formatAggregateCost = function formatAggregateCost(entityCosts, splitUpCosts: {} } - analyticsService.getEntityDetails(formattedAggregateCost.entity.type, - formattedAggregateCost.entity.id, - function(err, entityDetails) { - if(err) { - next(err) - } else { - formattedAggregateCost.entity.name = entityDetails.name - next(null, formattedAggregateCost) + if (formattedAggregateCost.entity.id != 'Unassigned' + && formattedAggregateCost.entity.id != 'Other' + && formattedAggregateCost.entity.id != 'Global' + && formattedAggregateCost.entity.id != 'Unknown') { + analyticsService.getEntityDetails(formattedAggregateCost.entity.type, + formattedAggregateCost.entity.id, + function (err, entityDetails) { + if (err) { + next(err) + } else { + formattedAggregateCost.entity.name = entityDetails.name + next(null, formattedAggregateCost) + } } - } - ) + ) + } else { + formattedAggregateCost.entity.name = formattedAggregateCost.entity.id + next(null, formattedAggregateCost) + } }, function(formattedAggregateCost, next) { async.forEach(entityCosts.splitUpCosts, @@ -319,7 +330,7 @@ analyticsService.formatAggregateCost = function formatAggregateCost(entityCosts, } if (costEntry.entity.id != 'Unassigned' && costEntry.entity.id != 'Other' - && costEntry.entity.id != 'Unknown') { + && costEntry.entity.id != 'Unknown' && costEntry.entity.id != 'Global') { analyticsService.getEntityDetails(costEntry.entity.type, costEntry.entity.id, function (err, entityDetails) { if (err) { diff --git a/server/app/services/resourceService.js b/server/app/services/resourceService.js index 6c641b673..04de2eb2f 100644 --- a/server/app/services/resourceService.js +++ b/server/app/services/resourceService.js @@ -92,9 +92,7 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u var stream = fs.createReadStream(downlaodedCSVPath) csv.fromStream(stream, {headers: false}).on('data', function(data) { - if((data[awsBillIndexes.totalCost] == 'LineItem') - && ((provider.lastBillUpdateTime == null) - || (Date.parse(data[awsBillIndexes.endDate]) > Date.parse(provider.lastBillUpdateTime)))) { + if(data[awsBillIndexes.totalCost] == 'LineItem') { var resourceCostEntry = {platformDetails: {}} resourceCostEntry.organizationId = provider.orgId @@ -113,10 +111,10 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u } resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) - ? 'Unknown' : data[awsBillIndexes.zone] + ? 'Global' : data[awsBillIndexes.zone] resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) - ? awsZones[data[awsBillIndexes.zone]] : 'Unknown' + ? awsZones[data[awsBillIndexes.zone]] : 'Global' if (data[awsBillIndexes.instanceId] != null) { resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] @@ -131,32 +129,38 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u resourceCostEntry.resourceId = resource._id - if ('bgId' in resource) { + if (('bgId' in resource) && (resource.bgId != null)) { resourceCostEntry.businessGroupId = resource['bgId'] } - if ('projectId' in resource) { + if (('projectId' in resource) && (resource.projectId != null)) { resourceCostEntry.projectId = resource['projectId'] } - if ('environmentId' in resource) { + if (('environmentId' in resource) && (resource.environmentId != null)) { resourceCostEntry.environmentId = resource['environmentId'] } - if ('masterDetails.bgId' in resource) { + if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { resourceCostEntry.businessGroupId = resource['bgId'] } - if ('masterDetails.projectId' in resource) { + if (('masterDetails.projectId' in resource) + && (resource.masterDetails.projectId != null)) { resourceCostEntry.projectId = resource['projectId'] } - if ('masterDetails.environmentId' in resource) { + if (('masterDetails.environmentId' in resource) + && (resource.masterDetails.environmentId != null)) { resourceCostEntry.environmentId = resource['environmentId'] } + } else { + resourceCostEntry.businessGroupId = 'Unassigned' + resourceCostEntry.projectId = 'Unassigned' + resourceCostEntry.environmentId = 'Unassigned' } - resourceCost.saveResourceCost(resourceCostEntry, function (err, costEntry) { + resourceCost.upsertResourceCost(resourceCostEntry, function (err, costEntry) { if (err) { logger.error(err) return callback(new Error('Database Error')) diff --git a/server/install.js b/server/install.js index 8780419b9..1a03a053f 100755 --- a/server/install.js +++ b/server/install.js @@ -477,6 +477,12 @@ function getDefaultsConfig() { } }, }, + costDefaultIds: { + businessGroupId: 'Unassigned', + environmentId: 'Unassigned', + projectId: 'Unassigned', + region: 'Global' + }, // cronjobTimeDelay: '"* * * * * *"',