Skip to content

Cost aggregation optimization and enhancements #825

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 17, 2016
Merged

Cost aggregation optimization and enhancements #825

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8231ca4
code update to generate weekly aggregate cost
Oct 14, 2016
f6d6d83
entity cost aggregation optimization
Oct 14, 2016
dceb444
service cost precision and dateUtil changes
Oct 17, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions server/app/lib/utils/dateUtil.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,18 @@ function getStartOfAMonthInUTC(date) {
}
}

function getStartOfAWeekInUTC(date) {
if( typeof date === 'undefined' || date === null ) {
return null;
}else{
var dateStartOfWeek = moment.utc(date).seconds(0);
dateStartOfWeek = moment.utc(dateStartOfWeek).minute(0);
dateStartOfWeek = moment.utc(dateStartOfWeek).hour(0);
dateStartOfWeek = moment.utc(dateStartOfWeek).day(0);
return dateStartOfWeek.format();
}
}

function getStartOfADayInUTC(date) {
if( typeof date === 'undefined' || date === null ){
var err = new Error("Invalid date");
Expand All @@ -61,6 +73,23 @@ function getStartOfADayInUTC(date) {
}
}

function getStartOfPeriod(period, date) {
var startTime = null
switch(period) {
case 'month':
startTime = getStartOfAMonthInUTC(date);
break;
case 'week':
startTime = getStartOfAWeekInUTC(date);
break;
case 'day':
startTime = getStartOfADayInUTC(date);
break;
}

return startTime;
}

/*
* Get Date in UTC Format
* date - JavaScript Date
Expand Down Expand Up @@ -186,3 +215,5 @@ momentDateUtil.getStartOfAYearInUTCAsync = getStartOfAYearInUTCAsync;
momentDateUtil.getStartOfAMonthInUTC = getStartOfAMonthInUTC;
momentDateUtil.getDateDifferenceInDays = getDateDifferenceInDays;
momentDateUtil.getStartOfADayInUTC = getStartOfADayInUTC;
momentDateUtil.getStartOfAWeekInUTC = getStartOfAWeekInUTC;
momentDateUtil.getStartOfPeriod = getStartOfPeriod;
6 changes: 4 additions & 2 deletions server/app/model/resource-costs/resource-costs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ var ResourceCostsSchema = new Schema({
trim: true
},
billLineItemId: {
type: String,
type: Number,
required: true,
trim: true
},
Expand Down Expand Up @@ -131,7 +131,8 @@ var ResourceCostsSchema = new Schema({
}
})

ResourceCostsSchema.index({'organizationId': 1, 'billLineItemId': 1,
ResourceCostsSchema.index({'platformDetails.serviceId' : 1})
ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'billLineItemId': 1,
'startTime': 1, 'interval': 1}, {'unique': true})

ResourceCostsSchema.statics.saveResourceCost = function saveResourceCost(resourceCostData, callback) {
Expand All @@ -148,6 +149,7 @@ ResourceCostsSchema.statics.saveResourceCost = function saveResourceCost(resourc
ResourceCostsSchema.statics.upsertResourceCost = function upsertResourceCost(resourceCostData, callback) {
var query = {
organizationId: resourceCostData.organizationId,
providerId: resourceCostData.providerId,
billLineItemId: resourceCostData.billLineItemId,
startTime: resourceCostData.startTime,
interval: resourceCostData.interval
Expand Down
105 changes: 35 additions & 70 deletions server/app/services/analyticsService.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,18 +35,8 @@ analyticsService.aggregateEntityCosts
})

var offset = (new Date()).getTimezoneOffset()*60000
var startTime
var interval
switch (period) {
case 'month':
startTime = dateUtil.getStartOfAMonthInUTC(endTime)
interval = costAggregationPeriods.month.intervalInSeconds
break
case 'day':
startTime = dateUtil.getStartOfADayInUTC(endTime)
interval = costAggregationPeriods.day.intervalInSeconds
break
}
var interval = costAggregationPeriods[period].intervalInSeconds
var startTime = dateUtil.getStartOfPeriod(period, endTime)

async.forEach(catalystEntityHierarchy[parentEntity].children, function (childEntity, next0) {
var query = parentEntityQuery
Expand All @@ -62,36 +52,28 @@ analyticsService.aggregateEntityCosts
], next1)
},
function(totalCosts, next1) {
var serviceCosts = []
async.forEach(platformServices,
function(service, next2) {
query['platformDetails.serviceId'] = service

resourceCostsModel.aggregate([
{$match: query},
{$group: {_id: "$" + catalystEntityHierarchy[childEntity].key,
totalCost: {$sum: "$cost"},
service: {$first: "$platformDetails.serviceId"}}}
], function(err, serviceCost) {
if(err) {
next2(err)
} else {
serviceCosts.push(serviceCost)
next2()
}
})
},
function(err) {
if(err) {
return next1(err)
} else {
var aggregatedCosts = {totalCosts: totalCosts}
aggregatedCosts.serviceCosts = serviceCosts

next1(null, aggregatedCosts)
resourceCostsModel.aggregate([
{$match: query},
{
$group: {
_id: {
"entityId": "$" + catalystEntityHierarchy[childEntity].key,
"service": "$platformDetails.serviceId"
},
totalCost: {$sum: "$cost"},
service: {$first: "$platformDetails.serviceId"}
}
}
)
], function (err, serviceCosts) {
if (err) {
return next1(err)
} else {
var aggregatedCosts = {totalCosts: totalCosts}
aggregatedCosts.serviceCosts = serviceCosts

next1(null, aggregatedCosts)
}
})
}
],
function(err, aggregateCosts) {
Expand All @@ -100,17 +82,10 @@ analyticsService.aggregateEntityCosts
} else {
//@TODO Blocking call to be avoided
var entityCosts = {}

for(var i = 0; i < aggregateCosts.totalCosts.length; i++) {
entityCosts[aggregateCosts.totalCosts[i]._id] = {
entity: {
id: aggregateCosts.totalCosts[i]._id,
type: childEntity
},
parentEntity: {
id: parentEntityId,
type: parentEntity
},
entity: { id: aggregateCosts.totalCosts[i]._id, type: childEntity},
parentEntity: {id: parentEntityId, type: parentEntity},
costs: {
totalCost: Math.round(aggregateCosts.totalCosts[i].totalCost * 100) / 100,
AWS: {
Expand All @@ -127,12 +102,10 @@ analyticsService.aggregateEntityCosts

// @TODO Blocking call to be avoided
for(var i = 0; i < aggregateCosts.serviceCosts.length; i++) {
for(var j = 0; j < aggregateCosts.serviceCosts[i].length; j++) {
if(aggregateCosts.serviceCosts[i][j]._id in entityCosts) {
entityCosts[aggregateCosts.serviceCosts[i][j]._id]
.costs.AWS.serviceCosts[aggregateCosts.serviceCosts[i][j].service]
= Math.round(aggregateCosts.serviceCosts[i][j].totalCost * 100) / 100
}
if(aggregateCosts.serviceCosts[i]._id.entityId in entityCosts) {
entityCosts[aggregateCosts.serviceCosts[i]._id.entityId]
.costs.AWS.serviceCosts[aggregateCosts.serviceCosts[i]._id.service]
= Math.round(aggregateCosts.serviceCosts[i].totalCost * 100) / 100
}
}

Expand Down Expand Up @@ -176,23 +149,15 @@ analyticsService.validateAndParseCostQuery
var err = new Error('Invalid request')
err.errors = [{messages: 'Mandatory fields missing'}]
err.status = 400
callback(err)
return callback(err)
}

var startTime
switch (requestQuery.period) {
case 'month':
startTime = dateUtil.getStartOfAMonthInUTC(requestQuery.toTimeStamp)
break
/*case 'year':
startTime = dateUtil.getStartOfAMonthInUTC(requestQuery.toTimeStamp)
break*/
default:
var err = new Error('Invalid request')
err.errors = [{messages: 'Period is invalid'}]
err.status = 400
return callback(err)
break
var startTime = dateUtil.getStartOfPeriod(requestQuery.period, requestQuery.toTimeStamp)
if(startTime == null) {
var err = new Error('Invalid request')
err.errors = [{messages: 'Period is invalid'}]
err.status = 400
return callback(err)
}

//@TODO Query object format to be changed
Expand Down
6 changes: 3 additions & 3 deletions server/app/services/resourceService.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,9 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u
resourceCostEntry.billLineItemId = ++lineNumber
resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId]

if (data[awsBillIndexes.prod] in awsServices) {
resourceCostEntry.platformDetails.serviceId = awsServices[data[awsBillIndexes.prod]]
}
resourceCostEntry.platformDetails.serviceId
= (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]]
:resourceCostEntry.platformDetails.serviceId = 'Other'

resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null)
? 'Global' : data[awsBillIndexes.zone]
Expand Down
23 changes: 17 additions & 6 deletions server/install.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,8 @@ function getDefaultsConfig() {
'Amazon Route 53': 'R53',
'Amazon Redshift': 'RedShift',
'Amazon ElastiCache': 'ElastiCache',
'Amazon CloudFront': 'CloudFront'
'Amazon CloudFront': 'CloudFront',
'Other': 'Other'
},

billIndexes: {
Expand Down Expand Up @@ -462,30 +463,40 @@ function getDefaultsConfig() {
},
region: {
key: 'platformDetails.region'
},
resource: {
key: 'resourceId'
}
},
costAggregationPeriods: {
'year': {
/*'year': {
intervalInSeconds: null,
childInterval: {
name: 'monthly',
intervalInSeconds: 2592000
}
},
},*/
'month': {
intervalInSeconds: 2592000,
childInterval: {
name: 'daily',
name: 'week',
intervalInSeconds: 86400
}
},
week: {
intervalInSeconds: 604800,
childInterval: {
name: 'day',
intervalInSeconds: 86400
}
},
day: {
intervalInSeconds: 86400,
childInterval: {
name: 'hourly',
name: 'hour',
intervalInSeconds: 3600
}
},
}
},
costDefaultIds: {
businessGroupId: 'Unassigned',
Expand Down