From 6b9bcfc9b546ef52a10bbad08fda95949e33da40 Mon Sep 17 00:00:00 2001 From: Karthik K N Date: Thu, 1 Dec 2016 14:28:06 +0530 Subject: [PATCH 1/5] Fix for CAT-2445 Incorrect Total Cost is showing in Analytics tab --- .../model/resource-costs/resource-costs.js | 43 +++-- server/app/services/resourceService.js | 161 ++++++++++-------- 2 files changed, 124 insertions(+), 80 deletions(-) diff --git a/server/app/model/resource-costs/resource-costs.js b/server/app/model/resource-costs/resource-costs.js index 012c774f6..28a47e042 100644 --- a/server/app/model/resource-costs/resource-costs.js +++ b/server/app/model/resource-costs/resource-costs.js @@ -70,6 +70,11 @@ var ResourceCostsSchema = new Schema({ required: false, trim: true }, + billIntervalId: { + type: String, + required: true, + trim: true + }, billLineItemId: { type: Number, required: true, @@ -132,38 +137,56 @@ var ResourceCostsSchema = new Schema({ }) ResourceCostsSchema.index({'platformDetails.serviceId' : 1}) -ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'billLineItemId': 1, - 'startTime': 1, 'interval': 1}, {'unique': true}) +ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'startTime': 1, + 'billIntervalId': 1, 'billLineItemId': 1, 'interval': 1}, {'unique': true}) -ResourceCostsSchema.statics.saveResourceCost = function saveResourceCost(resourceCostData, callback) { +ResourceCostsSchema.statics.save = function save(resourceCostData, callback) { var resourceCosts = new ResourceCosts(resourceCostData) - resourceCosts.save(function(err, data) { + resourceCosts.save(function(err, result) { + if (err) { + callback(err) + } else { + callback(null, result) + } + }) +} + +ResourceCostsSchema.statics.remove + = function remove(organizationId, providerId, billIntervalId, callback) { + var query = { + organizationId: organizationId, + providerId: providerId, + billIntervalId: billIntervalId + } + + this.find(query).remove(function(err, result) { if (err) { callback(err) } else { - callback(null) + callback(null, result) } }) } -ResourceCostsSchema.statics.upsertResourceCost = function upsertResourceCost(resourceCostData, callback) { +ResourceCostsSchema.statics.upsert = function upsert(resourceCostData, callback) { var query = { organizationId: resourceCostData.organizationId, providerId: resourceCostData.providerId, - billLineItemId: resourceCostData.billLineItemId, startTime: resourceCostData.startTime, + billIntervalId: resourceCostData.billIntervalId, + billLineItemId: resourceCostData.billLineItemId, interval: resourceCostData.interval } - this.findOneAndUpdate(query, resourceCostData, {upsert:true}, + this.findOneAndUpdate(query, resourceCostData, {upsert: true}, function(err, result){ if (err) { callback(null) } else { callback(null, result) } - }); + }) } var ResourceCosts = mongoose.model('ResourceCosts', ResourceCostsSchema) -module.exports = ResourceCosts +module.exports = ResourceCosts \ No newline at end of file diff --git a/server/app/services/resourceService.js b/server/app/services/resourceService.js index f56a7f95c..deb4f322d 100644 --- a/server/app/services/resourceService.js +++ b/server/app/services/resourceService.js @@ -91,91 +91,112 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u var awsZones = appConfig.aws.zones var lineNumber = 0 - var stream = fs.createReadStream(downlaodedCSVPath) - csv.fromStream(stream, {headers: false}).on('data', function(data) { - if(data[awsBillIndexes.totalCost] == 'LineItem') { - var resourceCostEntry = {platformDetails: {}} - - resourceCostEntry.organizationId = provider.orgId - resourceCostEntry.providerId = provider._id - resourceCostEntry.providerType = provider.providerType - resourceCostEntry.cost = data[awsBillIndexes.cost] - resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate]) - resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate]) - resourceCostEntry.lastUpdateTime = Date.parse(updateTime) - resourceCostEntry.interval = 3600 - resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod] - resourceCostEntry.billLineItemId = ++lineNumber - resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - - resourceCostEntry.platformDetails.serviceId - = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] - :resourceCostEntry.platformDetails.serviceId = 'Other' - - resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) - ? 'Global' : data[awsBillIndexes.zone] - - resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) - ? awsZones[data[awsBillIndexes.zone]] : 'Global' - - if (data[awsBillIndexes.instanceId] != null) { - resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] - } + var date = new Date() + var billIntervalId = date.getFullYear() + '-' + date.getMonth() - if(data[awsBillIndexes.usageType] != null) { - resourceCostEntry.platformDetails.usageType = data[awsBillIndexes.usageType] - } + async.waterfall([ + function(next) { + resourceCost.remove(provider.orgId, provider._id, billIntervalId, next) + }, + function(count, next) { + /*var lineNumber = (count == 0)?0:count + var startingLineNumber = (count == 0)?1:(count+2)*/ - if (data[awsBillIndexes.instanceId] in resources) { - var resource = resources[data[awsBillIndexes.instanceId]] + var stream = fs.createReadStream(downlaodedCSVPath) + csv.fromStream(stream).on('data', function(data) { + if(data[awsBillIndexes.totalCost] == 'LineItem') { + var resourceCostEntry = {platformDetails: {}} - resourceCostEntry.resourceId = resource._id + resourceCostEntry.organizationId = provider.orgId + resourceCostEntry.providerId = provider._id + resourceCostEntry.providerType = provider.providerType + resourceCostEntry.cost = data[awsBillIndexes.cost] + resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate]) + resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate]) + resourceCostEntry.lastUpdateTime = Date.parse(updateTime) + resourceCostEntry.interval = 3600 + resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod] + resourceCostEntry.billIntervalId = billIntervalId + resourceCostEntry.billLineItemId = ++lineNumber + resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - if (('bgId' in resource) && (resource.bgId != null)) { - resourceCostEntry.businessGroupId = resource['bgId'] - } + resourceCostEntry.platformDetails.serviceId + = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] + :resourceCostEntry.platformDetails.serviceId = 'Other' - if (('projectId' in resource) && (resource.projectId != null)) { - resourceCostEntry.projectId = resource['projectId'] - } + resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) + ? 'Global' : data[awsBillIndexes.zone] - if (('environmentId' in resource) && (resource.environmentId != null)) { - resourceCostEntry.environmentId = resource['environmentId'] - } + resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) + ? awsZones[data[awsBillIndexes.zone]] : 'Global' - if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { - console.log("BG: " + resource['bgId']) - resourceCostEntry.businessGroupId = resource['bgId'] - } + if (data[awsBillIndexes.instanceId] != null) { + resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] + } - if (('masterDetails.projectId' in resource) - && (resource.masterDetails.projectId != null)) { - resourceCostEntry.projectId = resource['projectId'] - } + if(data[awsBillIndexes.usageType] != null) { + resourceCostEntry.platformDetails.usageType = data[awsBillIndexes.usageType] + } - if (('masterDetails.environmentId' in resource) - && (resource.masterDetails.environmentId != null)) { - resourceCostEntry.environmentId = resource['environmentId'] - } - } + if (data[awsBillIndexes.instanceId] in resources) { + var resource = resources[data[awsBillIndexes.instanceId]] - resourceCostEntry.businessGroupId - = ('businessGroupId' in resourceCostEntry)?resourceCostEntry.businessGroupId:'Unassigned' - resourceCostEntry.projectId - = ('projectId' in resourceCostEntry)?resourceCostEntry.projectId:'Unassigned' - resourceCostEntry.environmentId - = ('environmentId' in resourceCostEntry)?resourceCostEntry.environmentId:'Unassigned' + resourceCostEntry.resourceId = resource._id - resourceCost.upsertResourceCost(resourceCostEntry, function (err, costEntry) { - if (err) { - logger.error(err) - return callback(new Error('Database Error')) + if (('bgId' in resource) && (resource.bgId != null)) { + resourceCostEntry.businessGroupId = resource['bgId'] + } + + if (('projectId' in resource) && (resource.projectId != null)) { + resourceCostEntry.projectId = resource['projectId'] + } + + if (('environmentId' in resource) && (resource.environmentId != null)) { + resourceCostEntry.environmentId = resource['environmentId'] + } + + if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { + console.log("BG: " + resource['bgId']) + resourceCostEntry.businessGroupId = resource['bgId'] + } + + if (('masterDetails.projectId' in resource) + && (resource.masterDetails.projectId != null)) { + resourceCostEntry.projectId = resource['projectId'] + } + + if (('masterDetails.environmentId' in resource) + && (resource.masterDetails.environmentId != null)) { + resourceCostEntry.environmentId = resource['environmentId'] + } + } + + resourceCostEntry.businessGroupId + = ('businessGroupId' in resourceCostEntry)?resourceCostEntry.businessGroupId:'Unassigned' + resourceCostEntry.projectId + = ('projectId' in resourceCostEntry)?resourceCostEntry.projectId:'Unassigned' + resourceCostEntry.environmentId + = ('environmentId' in resourceCostEntry)?resourceCostEntry.environmentId:'Unassigned' + + resourceCost.save(resourceCostEntry, function (err, costEntry) { + if (err) { + logger.error(err) + return next(new Error('Database Error')) + } + }) } + }).on('end', function() { + next() }) } - }).on('end', function() { - callback(null) + ], function(err) { + if(err) { + callback(err) + } else { + callback() + } }) + } function getCostForServices_deprecated(provider,callback) { From 1b2fcfc0845b75d35b3608fe869f701328ff499a Mon Sep 17 00:00:00 2001 From: Karthik K N Date: Thu, 1 Dec 2016 14:28:06 +0530 Subject: [PATCH 2/5] Fix for CAT-2445 Incorrect Total Cost is showing in Analytics tab --- .../model/resource-costs/resource-costs.js | 43 +++-- server/app/services/resourceService.js | 161 ++++++++++-------- 2 files changed, 124 insertions(+), 80 deletions(-) diff --git a/server/app/model/resource-costs/resource-costs.js b/server/app/model/resource-costs/resource-costs.js index 012c774f6..28a47e042 100644 --- a/server/app/model/resource-costs/resource-costs.js +++ b/server/app/model/resource-costs/resource-costs.js @@ -70,6 +70,11 @@ var ResourceCostsSchema = new Schema({ required: false, trim: true }, + billIntervalId: { + type: String, + required: true, + trim: true + }, billLineItemId: { type: Number, required: true, @@ -132,38 +137,56 @@ var ResourceCostsSchema = new Schema({ }) ResourceCostsSchema.index({'platformDetails.serviceId' : 1}) -ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'billLineItemId': 1, - 'startTime': 1, 'interval': 1}, {'unique': true}) +ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'startTime': 1, + 'billIntervalId': 1, 'billLineItemId': 1, 'interval': 1}, {'unique': true}) -ResourceCostsSchema.statics.saveResourceCost = function saveResourceCost(resourceCostData, callback) { +ResourceCostsSchema.statics.save = function save(resourceCostData, callback) { var resourceCosts = new ResourceCosts(resourceCostData) - resourceCosts.save(function(err, data) { + resourceCosts.save(function(err, result) { + if (err) { + callback(err) + } else { + callback(null, result) + } + }) +} + +ResourceCostsSchema.statics.remove + = function remove(organizationId, providerId, billIntervalId, callback) { + var query = { + organizationId: organizationId, + providerId: providerId, + billIntervalId: billIntervalId + } + + this.find(query).remove(function(err, result) { if (err) { callback(err) } else { - callback(null) + callback(null, result) } }) } -ResourceCostsSchema.statics.upsertResourceCost = function upsertResourceCost(resourceCostData, callback) { +ResourceCostsSchema.statics.upsert = function upsert(resourceCostData, callback) { var query = { organizationId: resourceCostData.organizationId, providerId: resourceCostData.providerId, - billLineItemId: resourceCostData.billLineItemId, startTime: resourceCostData.startTime, + billIntervalId: resourceCostData.billIntervalId, + billLineItemId: resourceCostData.billLineItemId, interval: resourceCostData.interval } - this.findOneAndUpdate(query, resourceCostData, {upsert:true}, + this.findOneAndUpdate(query, resourceCostData, {upsert: true}, function(err, result){ if (err) { callback(null) } else { callback(null, result) } - }); + }) } var ResourceCosts = mongoose.model('ResourceCosts', ResourceCostsSchema) -module.exports = ResourceCosts +module.exports = ResourceCosts \ No newline at end of file diff --git a/server/app/services/resourceService.js b/server/app/services/resourceService.js index f56a7f95c..deb4f322d 100644 --- a/server/app/services/resourceService.js +++ b/server/app/services/resourceService.js @@ -91,91 +91,112 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u var awsZones = appConfig.aws.zones var lineNumber = 0 - var stream = fs.createReadStream(downlaodedCSVPath) - csv.fromStream(stream, {headers: false}).on('data', function(data) { - if(data[awsBillIndexes.totalCost] == 'LineItem') { - var resourceCostEntry = {platformDetails: {}} - - resourceCostEntry.organizationId = provider.orgId - resourceCostEntry.providerId = provider._id - resourceCostEntry.providerType = provider.providerType - resourceCostEntry.cost = data[awsBillIndexes.cost] - resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate]) - resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate]) - resourceCostEntry.lastUpdateTime = Date.parse(updateTime) - resourceCostEntry.interval = 3600 - resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod] - resourceCostEntry.billLineItemId = ++lineNumber - resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - - resourceCostEntry.platformDetails.serviceId - = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] - :resourceCostEntry.platformDetails.serviceId = 'Other' - - resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) - ? 'Global' : data[awsBillIndexes.zone] - - resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) - ? awsZones[data[awsBillIndexes.zone]] : 'Global' - - if (data[awsBillIndexes.instanceId] != null) { - resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] - } + var date = new Date() + var billIntervalId = date.getFullYear() + '-' + date.getMonth() - if(data[awsBillIndexes.usageType] != null) { - resourceCostEntry.platformDetails.usageType = data[awsBillIndexes.usageType] - } + async.waterfall([ + function(next) { + resourceCost.remove(provider.orgId, provider._id, billIntervalId, next) + }, + function(count, next) { + /*var lineNumber = (count == 0)?0:count + var startingLineNumber = (count == 0)?1:(count+2)*/ - if (data[awsBillIndexes.instanceId] in resources) { - var resource = resources[data[awsBillIndexes.instanceId]] + var stream = fs.createReadStream(downlaodedCSVPath) + csv.fromStream(stream).on('data', function(data) { + if(data[awsBillIndexes.totalCost] == 'LineItem') { + var resourceCostEntry = {platformDetails: {}} - resourceCostEntry.resourceId = resource._id + resourceCostEntry.organizationId = provider.orgId + resourceCostEntry.providerId = provider._id + resourceCostEntry.providerType = provider.providerType + resourceCostEntry.cost = data[awsBillIndexes.cost] + resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate]) + resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate]) + resourceCostEntry.lastUpdateTime = Date.parse(updateTime) + resourceCostEntry.interval = 3600 + resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod] + resourceCostEntry.billIntervalId = billIntervalId + resourceCostEntry.billLineItemId = ++lineNumber + resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - if (('bgId' in resource) && (resource.bgId != null)) { - resourceCostEntry.businessGroupId = resource['bgId'] - } + resourceCostEntry.platformDetails.serviceId + = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] + :resourceCostEntry.platformDetails.serviceId = 'Other' - if (('projectId' in resource) && (resource.projectId != null)) { - resourceCostEntry.projectId = resource['projectId'] - } + resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) + ? 'Global' : data[awsBillIndexes.zone] - if (('environmentId' in resource) && (resource.environmentId != null)) { - resourceCostEntry.environmentId = resource['environmentId'] - } + resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) + ? awsZones[data[awsBillIndexes.zone]] : 'Global' - if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { - console.log("BG: " + resource['bgId']) - resourceCostEntry.businessGroupId = resource['bgId'] - } + if (data[awsBillIndexes.instanceId] != null) { + resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] + } - if (('masterDetails.projectId' in resource) - && (resource.masterDetails.projectId != null)) { - resourceCostEntry.projectId = resource['projectId'] - } + if(data[awsBillIndexes.usageType] != null) { + resourceCostEntry.platformDetails.usageType = data[awsBillIndexes.usageType] + } - if (('masterDetails.environmentId' in resource) - && (resource.masterDetails.environmentId != null)) { - resourceCostEntry.environmentId = resource['environmentId'] - } - } + if (data[awsBillIndexes.instanceId] in resources) { + var resource = resources[data[awsBillIndexes.instanceId]] - resourceCostEntry.businessGroupId - = ('businessGroupId' in resourceCostEntry)?resourceCostEntry.businessGroupId:'Unassigned' - resourceCostEntry.projectId - = ('projectId' in resourceCostEntry)?resourceCostEntry.projectId:'Unassigned' - resourceCostEntry.environmentId - = ('environmentId' in resourceCostEntry)?resourceCostEntry.environmentId:'Unassigned' + resourceCostEntry.resourceId = resource._id - resourceCost.upsertResourceCost(resourceCostEntry, function (err, costEntry) { - if (err) { - logger.error(err) - return callback(new Error('Database Error')) + if (('bgId' in resource) && (resource.bgId != null)) { + resourceCostEntry.businessGroupId = resource['bgId'] + } + + if (('projectId' in resource) && (resource.projectId != null)) { + resourceCostEntry.projectId = resource['projectId'] + } + + if (('environmentId' in resource) && (resource.environmentId != null)) { + resourceCostEntry.environmentId = resource['environmentId'] + } + + if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { + console.log("BG: " + resource['bgId']) + resourceCostEntry.businessGroupId = resource['bgId'] + } + + if (('masterDetails.projectId' in resource) + && (resource.masterDetails.projectId != null)) { + resourceCostEntry.projectId = resource['projectId'] + } + + if (('masterDetails.environmentId' in resource) + && (resource.masterDetails.environmentId != null)) { + resourceCostEntry.environmentId = resource['environmentId'] + } + } + + resourceCostEntry.businessGroupId + = ('businessGroupId' in resourceCostEntry)?resourceCostEntry.businessGroupId:'Unassigned' + resourceCostEntry.projectId + = ('projectId' in resourceCostEntry)?resourceCostEntry.projectId:'Unassigned' + resourceCostEntry.environmentId + = ('environmentId' in resourceCostEntry)?resourceCostEntry.environmentId:'Unassigned' + + resourceCost.save(resourceCostEntry, function (err, costEntry) { + if (err) { + logger.error(err) + return next(new Error('Database Error')) + } + }) } + }).on('end', function() { + next() }) } - }).on('end', function() { - callback(null) + ], function(err) { + if(err) { + callback(err) + } else { + callback() + } }) + } function getCostForServices_deprecated(provider,callback) { From 5ab8fa933a079e393e86a6af113e42f7b5f28c02 Mon Sep 17 00:00:00 2001 From: Karthik K N Date: Thu, 1 Dec 2016 14:28:06 +0530 Subject: [PATCH 3/5] Fix for CAT-2445 Incorrect Total Cost is showing in Analytics tab --- .../model/resource-costs/resource-costs.js | 43 +++-- server/app/services/resourceService.js | 161 ++++++++++-------- 2 files changed, 124 insertions(+), 80 deletions(-) diff --git a/server/app/model/resource-costs/resource-costs.js b/server/app/model/resource-costs/resource-costs.js index 012c774f6..28a47e042 100644 --- a/server/app/model/resource-costs/resource-costs.js +++ b/server/app/model/resource-costs/resource-costs.js @@ -70,6 +70,11 @@ var ResourceCostsSchema = new Schema({ required: false, trim: true }, + billIntervalId: { + type: String, + required: true, + trim: true + }, billLineItemId: { type: Number, required: true, @@ -132,38 +137,56 @@ var ResourceCostsSchema = new Schema({ }) ResourceCostsSchema.index({'platformDetails.serviceId' : 1}) -ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'billLineItemId': 1, - 'startTime': 1, 'interval': 1}, {'unique': true}) +ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'startTime': 1, + 'billIntervalId': 1, 'billLineItemId': 1, 'interval': 1}, {'unique': true}) -ResourceCostsSchema.statics.saveResourceCost = function saveResourceCost(resourceCostData, callback) { +ResourceCostsSchema.statics.save = function save(resourceCostData, callback) { var resourceCosts = new ResourceCosts(resourceCostData) - resourceCosts.save(function(err, data) { + resourceCosts.save(function(err, result) { + if (err) { + callback(err) + } else { + callback(null, result) + } + }) +} + +ResourceCostsSchema.statics.remove + = function remove(organizationId, providerId, billIntervalId, callback) { + var query = { + organizationId: organizationId, + providerId: providerId, + billIntervalId: billIntervalId + } + + this.find(query).remove(function(err, result) { if (err) { callback(err) } else { - callback(null) + callback(null, result) } }) } -ResourceCostsSchema.statics.upsertResourceCost = function upsertResourceCost(resourceCostData, callback) { +ResourceCostsSchema.statics.upsert = function upsert(resourceCostData, callback) { var query = { organizationId: resourceCostData.organizationId, providerId: resourceCostData.providerId, - billLineItemId: resourceCostData.billLineItemId, startTime: resourceCostData.startTime, + billIntervalId: resourceCostData.billIntervalId, + billLineItemId: resourceCostData.billLineItemId, interval: resourceCostData.interval } - this.findOneAndUpdate(query, resourceCostData, {upsert:true}, + this.findOneAndUpdate(query, resourceCostData, {upsert: true}, function(err, result){ if (err) { callback(null) } else { callback(null, result) } - }); + }) } var ResourceCosts = mongoose.model('ResourceCosts', ResourceCostsSchema) -module.exports = ResourceCosts +module.exports = ResourceCosts \ No newline at end of file diff --git a/server/app/services/resourceService.js b/server/app/services/resourceService.js index f56a7f95c..deb4f322d 100644 --- a/server/app/services/resourceService.js +++ b/server/app/services/resourceService.js @@ -91,91 +91,112 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u var awsZones = appConfig.aws.zones var lineNumber = 0 - var stream = fs.createReadStream(downlaodedCSVPath) - csv.fromStream(stream, {headers: false}).on('data', function(data) { - if(data[awsBillIndexes.totalCost] == 'LineItem') { - var resourceCostEntry = {platformDetails: {}} - - resourceCostEntry.organizationId = provider.orgId - resourceCostEntry.providerId = provider._id - resourceCostEntry.providerType = provider.providerType - resourceCostEntry.cost = data[awsBillIndexes.cost] - resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate]) - resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate]) - resourceCostEntry.lastUpdateTime = Date.parse(updateTime) - resourceCostEntry.interval = 3600 - resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod] - resourceCostEntry.billLineItemId = ++lineNumber - resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - - resourceCostEntry.platformDetails.serviceId - = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] - :resourceCostEntry.platformDetails.serviceId = 'Other' - - resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) - ? 'Global' : data[awsBillIndexes.zone] - - resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) - ? awsZones[data[awsBillIndexes.zone]] : 'Global' - - if (data[awsBillIndexes.instanceId] != null) { - resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] - } + var date = new Date() + var billIntervalId = date.getFullYear() + '-' + date.getMonth() - if(data[awsBillIndexes.usageType] != null) { - resourceCostEntry.platformDetails.usageType = data[awsBillIndexes.usageType] - } + async.waterfall([ + function(next) { + resourceCost.remove(provider.orgId, provider._id, billIntervalId, next) + }, + function(count, next) { + /*var lineNumber = (count == 0)?0:count + var startingLineNumber = (count == 0)?1:(count+2)*/ - if (data[awsBillIndexes.instanceId] in resources) { - var resource = resources[data[awsBillIndexes.instanceId]] + var stream = fs.createReadStream(downlaodedCSVPath) + csv.fromStream(stream).on('data', function(data) { + if(data[awsBillIndexes.totalCost] == 'LineItem') { + var resourceCostEntry = {platformDetails: {}} - resourceCostEntry.resourceId = resource._id + resourceCostEntry.organizationId = provider.orgId + resourceCostEntry.providerId = provider._id + resourceCostEntry.providerType = provider.providerType + resourceCostEntry.cost = data[awsBillIndexes.cost] + resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate]) + resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate]) + resourceCostEntry.lastUpdateTime = Date.parse(updateTime) + resourceCostEntry.interval = 3600 + resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod] + resourceCostEntry.billIntervalId = billIntervalId + resourceCostEntry.billLineItemId = ++lineNumber + resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - if (('bgId' in resource) && (resource.bgId != null)) { - resourceCostEntry.businessGroupId = resource['bgId'] - } + resourceCostEntry.platformDetails.serviceId + = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] + :resourceCostEntry.platformDetails.serviceId = 'Other' - if (('projectId' in resource) && (resource.projectId != null)) { - resourceCostEntry.projectId = resource['projectId'] - } + resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) + ? 'Global' : data[awsBillIndexes.zone] - if (('environmentId' in resource) && (resource.environmentId != null)) { - resourceCostEntry.environmentId = resource['environmentId'] - } + resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) + ? awsZones[data[awsBillIndexes.zone]] : 'Global' - if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { - console.log("BG: " + resource['bgId']) - resourceCostEntry.businessGroupId = resource['bgId'] - } + if (data[awsBillIndexes.instanceId] != null) { + resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] + } - if (('masterDetails.projectId' in resource) - && (resource.masterDetails.projectId != null)) { - resourceCostEntry.projectId = resource['projectId'] - } + if(data[awsBillIndexes.usageType] != null) { + resourceCostEntry.platformDetails.usageType = data[awsBillIndexes.usageType] + } - if (('masterDetails.environmentId' in resource) - && (resource.masterDetails.environmentId != null)) { - resourceCostEntry.environmentId = resource['environmentId'] - } - } + if (data[awsBillIndexes.instanceId] in resources) { + var resource = resources[data[awsBillIndexes.instanceId]] - resourceCostEntry.businessGroupId - = ('businessGroupId' in resourceCostEntry)?resourceCostEntry.businessGroupId:'Unassigned' - resourceCostEntry.projectId - = ('projectId' in resourceCostEntry)?resourceCostEntry.projectId:'Unassigned' - resourceCostEntry.environmentId - = ('environmentId' in resourceCostEntry)?resourceCostEntry.environmentId:'Unassigned' + resourceCostEntry.resourceId = resource._id - resourceCost.upsertResourceCost(resourceCostEntry, function (err, costEntry) { - if (err) { - logger.error(err) - return callback(new Error('Database Error')) + if (('bgId' in resource) && (resource.bgId != null)) { + resourceCostEntry.businessGroupId = resource['bgId'] + } + + if (('projectId' in resource) && (resource.projectId != null)) { + resourceCostEntry.projectId = resource['projectId'] + } + + if (('environmentId' in resource) && (resource.environmentId != null)) { + resourceCostEntry.environmentId = resource['environmentId'] + } + + if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { + console.log("BG: " + resource['bgId']) + resourceCostEntry.businessGroupId = resource['bgId'] + } + + if (('masterDetails.projectId' in resource) + && (resource.masterDetails.projectId != null)) { + resourceCostEntry.projectId = resource['projectId'] + } + + if (('masterDetails.environmentId' in resource) + && (resource.masterDetails.environmentId != null)) { + resourceCostEntry.environmentId = resource['environmentId'] + } + } + + resourceCostEntry.businessGroupId + = ('businessGroupId' in resourceCostEntry)?resourceCostEntry.businessGroupId:'Unassigned' + resourceCostEntry.projectId + = ('projectId' in resourceCostEntry)?resourceCostEntry.projectId:'Unassigned' + resourceCostEntry.environmentId + = ('environmentId' in resourceCostEntry)?resourceCostEntry.environmentId:'Unassigned' + + resourceCost.save(resourceCostEntry, function (err, costEntry) { + if (err) { + logger.error(err) + return next(new Error('Database Error')) + } + }) } + }).on('end', function() { + next() }) } - }).on('end', function() { - callback(null) + ], function(err) { + if(err) { + callback(err) + } else { + callback() + } }) + } function getCostForServices_deprecated(provider,callback) { From d66b94616e10ab80eab72c65307a35c141700abd Mon Sep 17 00:00:00 2001 From: Karthik K N Date: Thu, 1 Dec 2016 14:28:06 +0530 Subject: [PATCH 4/5] Fix for CAT-2445 Incorrect Total Cost is showing in Analytics tab --- .../model/resource-costs/resource-costs.js | 43 +++-- server/app/services/resourceService.js | 161 ++++++++++-------- 2 files changed, 124 insertions(+), 80 deletions(-) diff --git a/server/app/model/resource-costs/resource-costs.js b/server/app/model/resource-costs/resource-costs.js index 012c774f6..28a47e042 100644 --- a/server/app/model/resource-costs/resource-costs.js +++ b/server/app/model/resource-costs/resource-costs.js @@ -70,6 +70,11 @@ var ResourceCostsSchema = new Schema({ required: false, trim: true }, + billIntervalId: { + type: String, + required: true, + trim: true + }, billLineItemId: { type: Number, required: true, @@ -132,38 +137,56 @@ var ResourceCostsSchema = new Schema({ }) ResourceCostsSchema.index({'platformDetails.serviceId' : 1}) -ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'billLineItemId': 1, - 'startTime': 1, 'interval': 1}, {'unique': true}) +ResourceCostsSchema.index({'organizationId': 1, 'providerId': 1, 'startTime': 1, + 'billIntervalId': 1, 'billLineItemId': 1, 'interval': 1}, {'unique': true}) -ResourceCostsSchema.statics.saveResourceCost = function saveResourceCost(resourceCostData, callback) { +ResourceCostsSchema.statics.save = function save(resourceCostData, callback) { var resourceCosts = new ResourceCosts(resourceCostData) - resourceCosts.save(function(err, data) { + resourceCosts.save(function(err, result) { + if (err) { + callback(err) + } else { + callback(null, result) + } + }) +} + +ResourceCostsSchema.statics.remove + = function remove(organizationId, providerId, billIntervalId, callback) { + var query = { + organizationId: organizationId, + providerId: providerId, + billIntervalId: billIntervalId + } + + this.find(query).remove(function(err, result) { if (err) { callback(err) } else { - callback(null) + callback(null, result) } }) } -ResourceCostsSchema.statics.upsertResourceCost = function upsertResourceCost(resourceCostData, callback) { +ResourceCostsSchema.statics.upsert = function upsert(resourceCostData, callback) { var query = { organizationId: resourceCostData.organizationId, providerId: resourceCostData.providerId, - billLineItemId: resourceCostData.billLineItemId, startTime: resourceCostData.startTime, + billIntervalId: resourceCostData.billIntervalId, + billLineItemId: resourceCostData.billLineItemId, interval: resourceCostData.interval } - this.findOneAndUpdate(query, resourceCostData, {upsert:true}, + this.findOneAndUpdate(query, resourceCostData, {upsert: true}, function(err, result){ if (err) { callback(null) } else { callback(null, result) } - }); + }) } var ResourceCosts = mongoose.model('ResourceCosts', ResourceCostsSchema) -module.exports = ResourceCosts +module.exports = ResourceCosts \ No newline at end of file diff --git a/server/app/services/resourceService.js b/server/app/services/resourceService.js index f56a7f95c..deb4f322d 100644 --- a/server/app/services/resourceService.js +++ b/server/app/services/resourceService.js @@ -91,91 +91,112 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u var awsZones = appConfig.aws.zones var lineNumber = 0 - var stream = fs.createReadStream(downlaodedCSVPath) - csv.fromStream(stream, {headers: false}).on('data', function(data) { - if(data[awsBillIndexes.totalCost] == 'LineItem') { - var resourceCostEntry = {platformDetails: {}} - - resourceCostEntry.organizationId = provider.orgId - resourceCostEntry.providerId = provider._id - resourceCostEntry.providerType = provider.providerType - resourceCostEntry.cost = data[awsBillIndexes.cost] - resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate]) - resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate]) - resourceCostEntry.lastUpdateTime = Date.parse(updateTime) - resourceCostEntry.interval = 3600 - resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod] - resourceCostEntry.billLineItemId = ++lineNumber - resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - - resourceCostEntry.platformDetails.serviceId - = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] - :resourceCostEntry.platformDetails.serviceId = 'Other' - - resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) - ? 'Global' : data[awsBillIndexes.zone] - - resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) - ? awsZones[data[awsBillIndexes.zone]] : 'Global' - - if (data[awsBillIndexes.instanceId] != null) { - resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] - } + var date = new Date() + var billIntervalId = date.getFullYear() + '-' + date.getMonth() - if(data[awsBillIndexes.usageType] != null) { - resourceCostEntry.platformDetails.usageType = data[awsBillIndexes.usageType] - } + async.waterfall([ + function(next) { + resourceCost.remove(provider.orgId, provider._id, billIntervalId, next) + }, + function(count, next) { + /*var lineNumber = (count == 0)?0:count + var startingLineNumber = (count == 0)?1:(count+2)*/ - if (data[awsBillIndexes.instanceId] in resources) { - var resource = resources[data[awsBillIndexes.instanceId]] + var stream = fs.createReadStream(downlaodedCSVPath) + csv.fromStream(stream).on('data', function(data) { + if(data[awsBillIndexes.totalCost] == 'LineItem') { + var resourceCostEntry = {platformDetails: {}} - resourceCostEntry.resourceId = resource._id + resourceCostEntry.organizationId = provider.orgId + resourceCostEntry.providerId = provider._id + resourceCostEntry.providerType = provider.providerType + resourceCostEntry.cost = data[awsBillIndexes.cost] + resourceCostEntry.startTime = Date.parse(data[awsBillIndexes.startDate]) + resourceCostEntry.endTime = Date.parse(data[awsBillIndexes.endDate]) + resourceCostEntry.lastUpdateTime = Date.parse(updateTime) + resourceCostEntry.interval = 3600 + resourceCostEntry.platformDetails.serviceName = data[awsBillIndexes.prod] + resourceCostEntry.billIntervalId = billIntervalId + resourceCostEntry.billLineItemId = ++lineNumber + resourceCostEntry.platformDetails.billRecordId = data[awsBillIndexes.recordId] - if (('bgId' in resource) && (resource.bgId != null)) { - resourceCostEntry.businessGroupId = resource['bgId'] - } + resourceCostEntry.platformDetails.serviceId + = (data[awsBillIndexes.prod] in awsServices)?awsServices[data[awsBillIndexes.prod]] + :resourceCostEntry.platformDetails.serviceId = 'Other' - if (('projectId' in resource) && (resource.projectId != null)) { - resourceCostEntry.projectId = resource['projectId'] - } + resourceCostEntry.platformDetails.zone = (data[awsBillIndexes.zone] == null) + ? 'Global' : data[awsBillIndexes.zone] - if (('environmentId' in resource) && (resource.environmentId != null)) { - resourceCostEntry.environmentId = resource['environmentId'] - } + resourceCostEntry.platformDetails.region = (data[awsBillIndexes.zone] in awsZones) + ? awsZones[data[awsBillIndexes.zone]] : 'Global' - if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { - console.log("BG: " + resource['bgId']) - resourceCostEntry.businessGroupId = resource['bgId'] - } + if (data[awsBillIndexes.instanceId] != null) { + resourceCostEntry.platformDetails.instanceId = data[awsBillIndexes.instanceId] + } - if (('masterDetails.projectId' in resource) - && (resource.masterDetails.projectId != null)) { - resourceCostEntry.projectId = resource['projectId'] - } + if(data[awsBillIndexes.usageType] != null) { + resourceCostEntry.platformDetails.usageType = data[awsBillIndexes.usageType] + } - if (('masterDetails.environmentId' in resource) - && (resource.masterDetails.environmentId != null)) { - resourceCostEntry.environmentId = resource['environmentId'] - } - } + if (data[awsBillIndexes.instanceId] in resources) { + var resource = resources[data[awsBillIndexes.instanceId]] - resourceCostEntry.businessGroupId - = ('businessGroupId' in resourceCostEntry)?resourceCostEntry.businessGroupId:'Unassigned' - resourceCostEntry.projectId - = ('projectId' in resourceCostEntry)?resourceCostEntry.projectId:'Unassigned' - resourceCostEntry.environmentId - = ('environmentId' in resourceCostEntry)?resourceCostEntry.environmentId:'Unassigned' + resourceCostEntry.resourceId = resource._id - resourceCost.upsertResourceCost(resourceCostEntry, function (err, costEntry) { - if (err) { - logger.error(err) - return callback(new Error('Database Error')) + if (('bgId' in resource) && (resource.bgId != null)) { + resourceCostEntry.businessGroupId = resource['bgId'] + } + + if (('projectId' in resource) && (resource.projectId != null)) { + resourceCostEntry.projectId = resource['projectId'] + } + + if (('environmentId' in resource) && (resource.environmentId != null)) { + resourceCostEntry.environmentId = resource['environmentId'] + } + + if (('masterDetails.bgId' in resource) && (resource.masterDetails.bgId != null)) { + console.log("BG: " + resource['bgId']) + resourceCostEntry.businessGroupId = resource['bgId'] + } + + if (('masterDetails.projectId' in resource) + && (resource.masterDetails.projectId != null)) { + resourceCostEntry.projectId = resource['projectId'] + } + + if (('masterDetails.environmentId' in resource) + && (resource.masterDetails.environmentId != null)) { + resourceCostEntry.environmentId = resource['environmentId'] + } + } + + resourceCostEntry.businessGroupId + = ('businessGroupId' in resourceCostEntry)?resourceCostEntry.businessGroupId:'Unassigned' + resourceCostEntry.projectId + = ('projectId' in resourceCostEntry)?resourceCostEntry.projectId:'Unassigned' + resourceCostEntry.environmentId + = ('environmentId' in resourceCostEntry)?resourceCostEntry.environmentId:'Unassigned' + + resourceCost.save(resourceCostEntry, function (err, costEntry) { + if (err) { + logger.error(err) + return next(new Error('Database Error')) + } + }) } + }).on('end', function() { + next() }) } - }).on('end', function() { - callback(null) + ], function(err) { + if(err) { + callback(err) + } else { + callback() + } }) + } function getCostForServices_deprecated(provider,callback) { From 52f566b7744f561a04bcf2c0469d799da1e709f7 Mon Sep 17 00:00:00 2001 From: Karthik K N Date: Mon, 5 Dec 2016 12:32:44 +0530 Subject: [PATCH 5/5] cost aggregation fixes --- server/app/model/resource-costs/resource-costs.js | 2 +- server/app/services/resourceService.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/app/model/resource-costs/resource-costs.js b/server/app/model/resource-costs/resource-costs.js index 28a47e042..9a0761fd4 100644 --- a/server/app/model/resource-costs/resource-costs.js +++ b/server/app/model/resource-costs/resource-costs.js @@ -24,7 +24,7 @@ var Schema = mongoose.Schema // @TODO Date field types to be revised var ResourceCostsSchema = new Schema({ cost: { - type: SchemaTypes.Double, + type: Number, required: true }, currency: { diff --git a/server/app/services/resourceService.js b/server/app/services/resourceService.js index deb4f322d..20a38dafe 100644 --- a/server/app/services/resourceService.js +++ b/server/app/services/resourceService.js @@ -92,7 +92,7 @@ function updateAWSResourceCostsFromCSV(provider, resources, downlaodedCSVPath, u var lineNumber = 0 var date = new Date() - var billIntervalId = date.getFullYear() + '-' + date.getMonth() + var billIntervalId = date.getFullYear() + '-' + (date.getMonth() + 1) async.waterfall([ function(next) {