diff --git a/server/app/lib/utils/dateUtil.js b/server/app/lib/utils/dateUtil.js index 41a78fc3f..724d68b09 100644 --- a/server/app/lib/utils/dateUtil.js +++ b/server/app/lib/utils/dateUtil.js @@ -49,6 +49,18 @@ function getStartOfAMonthInUTC(date) { } } +function getStartOfAWeekInUTC(date) { + if( typeof date === 'undefined' || date === null ) { + return null; + }else{ + var dateStartOfWeek = moment.utc(date).seconds(0); + dateStartOfWeek = moment.utc(dateStartOfWeek).minute(0); + dateStartOfWeek = moment.utc(dateStartOfWeek).hour(0); + dateStartOfWeek = moment.utc(dateStartOfWeek).day(0); + return dateStartOfWeek.format(); + } +} + function getStartOfADayInUTC(date) { if( typeof date === 'undefined' || date === null ){ var err = new Error("Invalid date"); @@ -61,6 +73,23 @@ function getStartOfADayInUTC(date) { } } +function getStartOfPeriod(period, date) { + var startTime = null + switch(period) { + case 'month': + startTime = getStartOfAMonthInUTC(date); + break; + case 'week': + startTime = getStartOfAWeekInUTC(date); + break; + case 'day': + startTime = getStartOfADayInUTC(date); + break; + } + + return startTime; +} + /* * Get Date in UTC Format * date - JavaScript Date @@ -186,3 +215,5 @@ momentDateUtil.getStartOfAYearInUTCAsync = getStartOfAYearInUTCAsync; momentDateUtil.getStartOfAMonthInUTC = getStartOfAMonthInUTC; momentDateUtil.getDateDifferenceInDays = getDateDifferenceInDays; momentDateUtil.getStartOfADayInUTC = getStartOfADayInUTC; +momentDateUtil.getStartOfAWeekInUTC = getStartOfAWeekInUTC; +momentDateUtil.getStartOfPeriod = getStartOfPeriod; diff --git a/server/app/model/resource-costs/resource-costs.js b/server/app/model/resource-costs/resource-costs.js index bcf24bdd5..1d6151d7c 100644 --- a/server/app/model/resource-costs/resource-costs.js +++ b/server/app/model/resource-costs/resource-costs.js @@ -71,7 +71,7 @@ var ResourceCostsSchema = new Schema({ trim: true }, billLineItemId: { - type: String, + type: Number, required: true, trim: true }, @@ -131,7 +131,8 @@ var ResourceCostsSchema = new Schema({ } }) -ResourceCostsSchema.index({'organizationId': 1, 'billLineItemId': 1, +ResourceCostsSchema.index({'platformDetails.serviceId' : 1}) +ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'billLineItemId': 1, 'startTime': 1, 'interval': 1}, {'unique': true}) ResourceCostsSchema.statics.saveResourceCost = function saveResourceCost(resourceCostData, callback) { @@ -148,6 +149,7 @@ ResourceCostsSchema.statics.saveResourceCost = function saveResourceCost(resourc ResourceCostsSchema.statics.upsertResourceCost = function upsertResourceCost(resourceCostData, callback) { var query = { organizationId: resourceCostData.organizationId, + providerId: resourceCostData.providerId, billLineItemId: resourceCostData.billLineItemId, startTime: resourceCostData.startTime, interval: resourceCostData.interval diff --git a/server/app/services/analyticsService.js b/server/app/services/analyticsService.js index fe9651ac5..ae55cc55a 100644 --- a/server/app/services/analyticsService.js +++ b/server/app/services/analyticsService.js @@ -35,18 +35,8 @@ analyticsService.aggregateEntityCosts }) var offset = (new Date()).getTimezoneOffset()*60000 - var startTime - var interval - switch (period) { - case 'month': - startTime = dateUtil.getStartOfAMonthInUTC(endTime) - interval = costAggregationPeriods.month.intervalInSeconds - break - case 'day': - startTime = dateUtil.getStartOfADayInUTC(endTime) - interval = costAggregationPeriods.day.intervalInSeconds - break - } + var interval = costAggregationPeriods[period].intervalInSeconds + var startTime = dateUtil.getStartOfPeriod(period, endTime) async.forEach(catalystEntityHierarchy[parentEntity].children, function (childEntity, next0) { var query = parentEntityQuery @@ -62,36 +52,28 @@ analyticsService.aggregateEntityCosts ], next1) }, function(totalCosts, next1) { - var serviceCosts = [] - async.forEach(platformServices, - function(service, next2) { - query['platformDetails.serviceId'] = service - - resourceCostsModel.aggregate([ - {$match: query}, - {$group: {_id: "$" + catalystEntityHierarchy[childEntity].key, - totalCost: {$sum: "$cost"}, - service: {$first: "$platformDetails.serviceId"}}} - ], function(err, serviceCost) { - if(err) { - next2(err) - } else { - serviceCosts.push(serviceCost) - next2() - } - }) - }, - function(err) { - if(err) { - return next1(err) - } else { - var aggregatedCosts = {totalCosts: totalCosts} - aggregatedCosts.serviceCosts = serviceCosts - - next1(null, aggregatedCosts) + resourceCostsModel.aggregate([ + {$match: query}, + { + $group: { + _id: { + "entityId": "$" + catalystEntityHierarchy[childEntity].key, + "service": "$platformDetails.serviceId" + }, + totalCost: {$sum: "$cost"}, + service: {$first: "$platformDetails.serviceId"} } } - ) + ], function (err, serviceCosts) { + if (err) { + return next1(err) + } else { + var aggregatedCosts = {totalCosts: totalCosts} + aggregatedCosts.serviceCosts = serviceCosts + + next1(null, aggregatedCosts) + } + }) } ], function(err, aggregateCosts) { @@ -100,17 +82,10 @@ analyticsService.aggregateEntityCosts } else { //@TODO Blocking call to be avoided var entityCosts = {} - for(var i = 0; i < aggregateCosts.totalCosts.length; i++) { entityCosts[aggregateCosts.totalCosts[i]._id] = { - entity: { - id: aggregateCosts.totalCosts[i]._id, - type: childEntity - }, - parentEntity: { - id: parentEntityId, - type: parentEntity - }, + entity: { id: aggregateCosts.totalCosts[i]._id, type: childEntity}, + parentEntity: {id: parentEntityId, type: parentEntity}, costs: { totalCost: Math.round(aggregateCosts.totalCosts[i].totalCost * 100) / 100, AWS: { @@ -127,12 +102,10 @@ analyticsService.aggregateEntityCosts // @TODO Blocking call to be avoided for(var i = 0; i < aggregateCosts.serviceCosts.length; i++) { - for(var j = 0; j < aggregateCosts.serviceCosts[i].length; j++) { - if(aggregateCosts.serviceCosts[i][j]._id in entityCosts) { - entityCosts[aggregateCosts.serviceCosts[i][j]._id] - .costs.AWS.serviceCosts[aggregateCosts.serviceCosts[i][j].service] - = Math.round(aggregateCosts.serviceCosts[i][j].totalCost * 100) / 100 - } + if(aggregateCosts.serviceCosts[i]._id.entityId in entityCosts) { + entityCosts[aggregateCosts.serviceCosts[i]._id.entityId] + .costs.AWS.serviceCosts[aggregateCosts.serviceCosts[i]._id.service] + = Math.round(aggregateCosts.serviceCosts[i].totalCost * 100) / 100 } } @@ -176,23 +149,15 @@ analyticsService.validateAndParseCostQuery var err = new Error('Invalid request') err.errors = [{messages: 'Mandatory fields missing'}] err.status = 400 - callback(err) + return callback(err) } - var startTime - switch (requestQuery.period) { - case 'month': - startTime = dateUtil.getStartOfAMonthInUTC(requestQuery.toTimeStamp) - break - /*case 'year': - startTime = dateUtil.getStartOfAMonthInUTC(requestQuery.toTimeStamp) - break*/ - default: - var err = new Error('Invalid request') - err.errors = [{messages: 'Period is invalid'}] - err.status = 400 - return callback(err) - break + var startTime = dateUtil.getStartOfPeriod(requestQuery.period, requestQuery.toTimeStamp) + if(startTime == null) { + var err = new Error('Invalid request') + err.errors = [{messages: 'Period is invalid'}] + err.status = 400 + return callback(err) } //@TODO Query object format to be changed diff --git a/server/app/services/resourceService.js b/server/app/services/resourceService.js index 7c2e5851b..7feddace3 100644 --- a/server/app/services/resourceService.js +++ b/server/app/services/resourceService.js @@ -108,9 +108,9 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u resourceCostEntry.billLineItemId = ++lineNumber resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - if (data[awsBillIndexes.prod] in awsServices) { - resourceCostEntry.platformDetails.serviceId = awsServices[data[awsBillIndexes.prod]] - } + resourceCostEntry.platformDetails.serviceId + = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] + :resourceCostEntry.platformDetails.serviceId = 'Other' resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) ? 'Global' : data[awsBillIndexes.zone] diff --git a/server/install.js b/server/install.js index 0596ff667..33995c1bd 100755 --- a/server/install.js +++ b/server/install.js @@ -404,7 +404,8 @@ function getDefaultsConfig() { 'Amazon Route 53': 'R53', 'Amazon Redshift': 'RedShift', 'Amazon ElastiCache': 'ElastiCache', - 'Amazon CloudFront': 'CloudFront' + 'Amazon CloudFront': 'CloudFront', + 'Other': 'Other' }, billIndexes: { @@ -462,30 +463,40 @@ function getDefaultsConfig() { }, region: { key: 'platformDetails.region' + }, + resource: { + key: 'resourceId' } }, costAggregationPeriods: { - 'year': { + /*'year': { intervalInSeconds: null, childInterval: { name: 'monthly', intervalInSeconds: 2592000 } - }, + },*/ 'month': { intervalInSeconds: 2592000, childInterval: { - name: 'daily', + name: 'week', + intervalInSeconds: 86400 + } + }, + week: { + intervalInSeconds: 604800, + childInterval: { + name: 'day', intervalInSeconds: 86400 } }, day: { intervalInSeconds: 86400, childInterval: { - name: 'hourly', + name: 'hour', intervalInSeconds: 3600 } - }, + } }, costDefaultIds: { businessGroupId: 'Unassigned',