Cost aggregation enhancements and bug fixes

server/app/cronjobs/aws-cost-aggregation/AWSResourceCostsAggregation.js

@@ -175,17 +175,18 @@ function downloadLatestBill(provider, callback) {
             }
         },
         function(billDownloaded, next) {
+            var downloadedCSVPath = appConfig.aws.s3BucketDownloadFileLocation
+                + AWSResourceCostsAggregation.csvFileName
+
             if(billDownloaded) {
                 var downloadedZipPath = appConfig.aws.s3BucketDownloadFileLocation
                     + appConfig.aws.s3BucketFileName
                 var zip = new AdmZip(downloadedZipPath)
                 zip.extractAllTo(appConfig.aws.s3BucketDownloadFileLocation, true)
 
-                var downloadedCSVPath = appConfig.aws.s3BucketDownloadFileLocation
-                    + AWSResourceCostsAggregation.csvFileName
                 next(null, downloadedCSVPath)
             } else {
-                next(null, null)
+                next(null, downloadedCSVPath)
             }
         }
     ],

server/app/model/entity-costs/entity-costs.js

@@ -67,6 +67,9 @@ var EntityCostsSchema = new Schema({
     }
 })
 
+EntityCostsSchema.index({'entity.id': 1, 'entity.type': 1, 'parentEntity.id': 1, 'startTime': 1,
+    'period': 1}, {'unique': true})
+
 EntityCostsSchema.statics.saveEntityCost = function saveEntityCost(entityCostData, callback) {
     var entityCosts = new EntityCosts(entityCostData)
     entityCosts.save(function(err, data) {

server/app/model/resource-costs/resource-costs.js

@@ -121,12 +121,12 @@ var ResourceCostsSchema = new Schema({
         type: Number,
         required: true
     }
-});
-/*ResourceCostsSchema.index({'platformDetails.instanceId': 1, 'platformDetails.usageType': 1,
-    'startTime': 1, 'endTime': 1}, {'unique': true, 'sparse': true, 'partialFilterExpression':
-    {'platformDetails.usageType': { $exists: true }}})*/
+})
 
-ResourceCostsSchema.statics.saveResourceCost = function(resourceCostData, callback) {
+ResourceCostsSchema.index({'organizationId': 1, 'resourceId': 1, 'billLineRecordId': 1,
+    'startTime': 1, 'interval': 1}, {'unique': true})
+
+ResourceCostsSchema.statics.saveResourceCost = function saveResourceCost(resourceCostData, callback) {
     var resourceCosts = new ResourceCosts(resourceCostData)
     resourceCosts.save(function(err, data) {
         if (err) {
@@ -137,5 +137,24 @@ ResourceCostsSchema.statics.saveResourceCost = function(resourceCostData, callba
     })
 }
 
+ResourceCostsSchema.statics.upsertResourceCost = function upsertResourceCost(resourceCostData, callback) {
+    var query = {
+        organizationId: resourceCostData.organizationId,
+        resourceId: resourceCostData.billLineRecordId,
+        billLineRecordId: resourceCostData.billLineRecordId,
+        startTime: resourceCostData.startTime,
+        interval: resourceCostData.interval
+    }
+
+    this.findOneAndUpdate(query, resourceCostData, {upsert:true},
+        function(err, result){
+            if (err) {
+                callback(null)
+            } else {
+                callback(null, result)
+            }
+        });
+}
+
 var ResourceCosts = mongoose.model('ResourceCosts', ResourceCostsSchema)
 module.exports = ResourceCosts

server/app/services/analyticsService.js

@@ -97,7 +97,9 @@ analyticsService.aggregateEntityCosts
 			if(err) {
 				next0(err)
 			} else {
+				//@TODO Blocking call to be avoided
 				var entityCosts = {}
+
 				for(var i = 0; i < aggregateCosts.totalCosts.length; i++) {
 					entityCosts[aggregateCosts.totalCosts[i]._id] = {
 						entity: {
@@ -122,6 +124,7 @@ analyticsService.aggregateEntityCosts
 					}
 				}
 
+				// @TODO Blocking call to be avoided
 				for(var i = 0; i < aggregateCosts.serviceCosts.length; i++) {
 					for(var j = 0; j < aggregateCosts.serviceCosts[i].length; j++) {
 						if(aggregateCosts.serviceCosts[i][j]._id in entityCosts) {
@@ -289,17 +292,25 @@ analyticsService.formatAggregateCost = function formatAggregateCost(entityCosts,
 				splitUpCosts: {}
 			}
 
-			analyticsService.getEntityDetails(formattedAggregateCost.entity.type,
-				formattedAggregateCost.entity.id,
-				function(err, entityDetails) {
-					if(err) {
-						next(err)
-					} else {
-						formattedAggregateCost.entity.name = entityDetails.name
-						next(null, formattedAggregateCost)
+			if (formattedAggregateCost.entity.id != 'Unassigned'
+				&& formattedAggregateCost.entity.id != 'Other'
+				&& formattedAggregateCost.entity.id != 'Global'
+				&& formattedAggregateCost.entity.id != 'Unknown') {
+				analyticsService.getEntityDetails(formattedAggregateCost.entity.type,
+					formattedAggregateCost.entity.id,
+					function (err, entityDetails) {
+						if (err) {
+							next(err)
+						} else {
+							formattedAggregateCost.entity.name = entityDetails.name
+							next(null, formattedAggregateCost)
+						}
 					}
-				}
-			)
+				)
+			} else {
+				formattedAggregateCost.entity.name = formattedAggregateCost.entity.id
+				next(null, formattedAggregateCost)
+			}
 		},
 		function(formattedAggregateCost, next) {
 			async.forEach(entityCosts.splitUpCosts,
@@ -319,7 +330,7 @@ analyticsService.formatAggregateCost = function formatAggregateCost(entityCosts,
 					}
 
 					if (costEntry.entity.id != 'Unassigned' && costEntry.entity.id != 'Other'
-						&& costEntry.entity.id != 'Unknown') {
+						&& costEntry.entity.id != 'Unknown' && costEntry.entity.id != 'Global') {
 						analyticsService.getEntityDetails(costEntry.entity.type, costEntry.entity.id,
 							function (err, entityDetails) {
 								if (err) {

server/app/services/resourceService.js

@@ -92,9 +92,7 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u
 
     var stream = fs.createReadStream(downlaodedCSVPath)
     csv.fromStream(stream, {headers: false}).on('data', function(data) {
-        if((data[awsBillIndexes.totalCost] == 'LineItem')
-            && ((provider.lastBillUpdateTime == null)
-            || (Date.parse(data[awsBillIndexes.endDate]) > Date.parse(provider.lastBillUpdateTime)))) {
+        if(data[awsBillIndexes.totalCost] == 'LineItem') {
             var resourceCostEntry = {platformDetails: {}}
 
             resourceCostEntry.organizationId = provider.orgId
@@ -113,10 +111,10 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u
             }
 
             resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null)
-                ? 'Unknown' : data[awsBillIndexes.zone]
+                ? 'Global' : data[awsBillIndexes.zone]
 
             resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones)
-                ? awsZones[data[awsBillIndexes.zone]] : 'Unknown'
+                ? awsZones[data[awsBillIndexes.zone]] : 'Global'
 
             if (data[awsBillIndexes.instanceId] != null) {
                 resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId]
@@ -131,32 +129,38 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u
 
                 resourceCostEntry.resourceId = resource._id
 
-                if ('bgId' in resource) {
+                if (('bgId' in resource) && (resource.bgId != null)) {
                     resourceCostEntry.businessGroupId = resource['bgId']
                 }
 
-                if ('projectId' in resource) {
+                if (('projectId' in resource) && (resource.projectId != null)) {
                     resourceCostEntry.projectId = resource['projectId']
                 }
 
-                if ('environmentId' in resource) {
+                if (('environmentId' in resource) && (resource.environmentId != null)) {
                     resourceCostEntry.environmentId = resource['environmentId']
                 }
 
-                if ('masterDetails.bgId' in resource) {
+                if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) {
                     resourceCostEntry.businessGroupId = resource['bgId']
                 }
 
-                if ('masterDetails.projectId' in resource) {
+                if (('masterDetails.projectId' in resource)
+                    && (resource.masterDetails.projectId != null)) {
                     resourceCostEntry.projectId = resource['projectId']
                 }
 
-                if ('masterDetails.environmentId' in resource) {
+                if (('masterDetails.environmentId' in resource)
+                    && (resource.masterDetails.environmentId != null)) {
                     resourceCostEntry.environmentId = resource['environmentId']
                 }
+            } else {
+                resourceCostEntry.businessGroupId = 'Unassigned'
+                resourceCostEntry.projectId = 'Unassigned'
+                resourceCostEntry.environmentId = 'Unassigned'
             }
 
-            resourceCost.saveResourceCost(resourceCostEntry, function (err, costEntry) {
+            resourceCost.upsertResourceCost(resourceCostEntry, function (err, costEntry) {
                 if (err) {
                     logger.error(err)
                     return callback(new Error('Database Error'))

server/install.js

@@ -477,6 +477,12 @@ function getDefaultsConfig() {
                 }
             },
         },
+        costDefaultIds: {
+            businessGroupId: 'Unassigned',
+            environmentId: 'Unassigned',
+            projectId: 'Unassigned',
+            region: 'Global'
+        },
 
         // cronjobTimeDelay: '"* * * * * *"',