Skip to content

Commit

Permalink
Merge pull request #13419 from isimluk/euwe-scvmm-chargebacks
Browse files Browse the repository at this point in the history 
[EUWE] Chargebacks for SCVMM (rollup-less)
  • Loading branch information
@simaishi
simaishi committed Jan 18, 2017
2 parents 201c88f + 44aa15b commit 970ed6b
Show file tree
Hide file tree
Showing 16 changed files with 662 additions and 483 deletions.
280 changes: 47 additions & 233 deletions app/models/chargeback.rb
View file
@@ -1,262 +1,74 @@
class Chargeback < ActsAsArModel
HOURS_IN_DAY = 24
HOURS_IN_WEEK = 168

VIRTUAL_COL_USES = {
"v_derived_cpu_total_cores_used" => "cpu_usage_rate_average"
}

def self.build_results_for_report_chargeback(options)
_log.info("Calculating chargeback costs...")
@options = options = ReportOptions.new_from_h(options)

tz = Metric::Helper.get_time_zone(options[:ext_options])
# TODO: Support time profiles via options[:ext_options][:time_profile]

interval = options[:interval] || "daily"
cb = new

options[:ext_options] ||= {}

if @options[:groupby_tag]
@tag_hash = Classification.hash_all_by_type_and_name[@options[:groupby_tag]][:entry]
end

base_rollup = MetricRollup.includes(
:resource => [:hardware, :tenant, :tags, :vim_performance_states, :custom_attributes, {:container_image => :custom_attributes}],
:parent_host => :tags,
:parent_ems_cluster => :tags,
:parent_storage => :tags,
:parent_ems => :tags)
.select(*Metric::BASE_COLS).order("resource_id, timestamp")
perf_cols = MetricRollup.attribute_names
rate_cols = ChargebackRate.where(:default => true).flat_map do |rate|
rate.chargeback_rate_details.map(&:metric).select { |metric| perf_cols.include?(metric.to_s) }
end

rate_cols.map! { |x| VIRTUAL_COL_USES.include?(x) ? VIRTUAL_COL_USES[x] : x }.flatten!
base_rollup = base_rollup.select(*rate_cols)

timerange = get_report_time_range(options, interval, tz)
data = {}
rates = RatesCache.new
ConsumptionHistory.for_report(self, options) do |consumption|
rates_to_apply = rates.get(consumption)

interval_duration = interval_to_duration(interval)

timerange.step_value(interval_duration).each_cons(2) do |query_start_time, query_end_time|
records = base_rollup.where(:timestamp => query_start_time...query_end_time, :capture_interval_name => "hourly")
records = where_clause(records, options)
records = Metric::Helper.remove_duplicate_timestamps(records)
next if records.empty?
_log.info("Found #{records.length} records for time range #{[query_start_time, query_end_time].inspect}")

hours_in_interval = hours_in_interval(query_start_time, query_end_time, interval)

# we are building hash with grouped calculated values
# values are grouped by resource_id and timestamp (query_start_time...query_end_time)
records.group_by(&:resource_id).each do |_, metric_rollup_records|
metric_rollup_records = metric_rollup_records.select { |x| x.resource.present? }
next if metric_rollup_records.empty?

# we need to select ChargebackRates for groups of MetricRollups records
# and rates are selected by first MetricRollup record
metric_rollup_record = metric_rollup_records.first
rates_to_apply = cb.get_rates(metric_rollup_record)

# key contains resource_id and timestamp (query_start_time...query_end_time)
# extra_fields there some extra field like resource name and
# some of them are related to specific chargeback (ChargebackVm, ChargebackContainer,...)
key, extra_fields = key_and_fields(metric_rollup_record, interval, tz)
data[key] ||= extra_fields

chargeback_rates = data[key]["chargeback_rates"].split(', ') + rates_to_apply.collect(&:description)
data[key]["chargeback_rates"] = chargeback_rates.uniq.join(', ')
key = report_row_key(consumption)
data[key] ||= new(options, consumption)

# we are getting hash with metrics and costs for metrics defined for chargeback
metrics_and_costs = calculate_costs(metric_rollup_records, rates_to_apply, hours_in_interval)
chargeback_rates = data[key]["chargeback_rates"].split(', ') + rates_to_apply.collect(&:description)
data[key]["chargeback_rates"] = chargeback_rates.uniq.join(', ')

data[key].merge!(metrics_and_costs)
end
# we are getting hash with metrics and costs for metrics defined for chargeback
data[key].calculate_costs(consumption, rates_to_apply)
end

_log.info("Calculating chargeback costs...Complete")

[data.map { |r| new(r.last) }]
end

def self.hours_in_interval(query_start_time, query_end_time, interval)
return HOURS_IN_DAY if interval == "daily"
return HOURS_IN_WEEK if interval == "weekly"

(query_end_time - query_start_time) / 1.hour
[data.values]
end

def self.interval_to_duration(interval)
case interval
when "daily"
1.day
when "weekly"
1.week
when "monthly"
1.month
def self.report_row_key(consumption)
ts_key = @options.start_of_report_step(consumption.timestamp)
if @options[:groupby_tag].present?
classification = classification_for(consumption)
classification_id = classification.present? ? classification.id : 'none'
"#{classification_id}_#{ts_key}"
else
default_key(consumption, ts_key)
end
end

def self.key_and_fields(metric_rollup_record, interval, tz)
ts_key = get_group_key_ts(metric_rollup_record, interval, tz)

key, extra_fields = if @options[:groupby_tag].present?
get_tag_keys_and_fields(metric_rollup_record, ts_key)
else
get_keys_and_extra_fields(metric_rollup_record, ts_key)
end

[key, date_fields(metric_rollup_record, interval, tz).merge(extra_fields)]
def self.default_key(consumption, ts_key)
"#{consumption.resource_id}_#{ts_key}"
end

def self.date_fields(metric_rollup_record, interval, tz)
start_ts, end_ts, display_range = get_time_range(metric_rollup_record, interval, tz)

{
'start_date' => start_ts,
'end_date' => end_ts,
'display_range' => display_range,
'interval_name' => interval,
'chargeback_rates' => '',
'entity' => metric_rollup_record.resource
}
end

def self.get_tag_keys_and_fields(perf, ts_key)
tag = perf.tag_names.split("|").select { |x| x.starts_with?(@options[:groupby_tag]) }.first # 'department/*'
def self.classification_for(consumption)
tag = consumption.tag_names.find { |x| x.starts_with?(@options[:groupby_tag]) } # 'department/*'
tag = tag.split('/').second unless tag.blank? # 'department/finance' -> 'finance'
classification = @tag_hash[tag]
classification_id = classification.present? ? classification.id : 'none'
key = "#{classification_id}_#{ts_key}"
extra_fields = { "tag_name" => classification.present? ? classification.description : _('<Empty>') }
[key, extra_fields]
@options.tag_hash[tag]
end

def get_rates(perf)
@rates ||= {}
@rates[perf.hash_features_affecting_rate] ||=
begin
prefix = Chargeback.report_cb_model(self.class.name).underscore
ChargebackRate.get_assigned_for_target(perf.resource,
:tag_list => perf.tag_list_reconstruct.map! { |t| prefix + t },
:parents => get_rate_parents(perf))
end
if perf.resource_type == Container.name && @rates[perf.hash_features_affecting_rate].empty?
@rates[perf.hash_features_affecting_rate] = [ChargebackRate.find_by(:description => "Default Container Image Rate", :rate_type => "Compute")]
def initialize(options, consumption)
@options = options
super()
if @options[:groupby_tag].present?
classification = self.class.classification_for(consumption)
self.tag_name = classification.present? ? classification.description : _('<Empty>')
else
init_extra_fields(consumption)
end
@rates[perf.hash_features_affecting_rate]
self.start_date, self.end_date, self.display_range = options.report_step_range(consumption.timestamp)
self.interval_name = options.interval
self.chargeback_rates = ''
self.entity = consumption.resource
end

def self.calculate_costs(metric_rollup_records, rates, hours_in_interval)
calculated_costs = {}

chargeback_fields_present = metric_rollup_records.count(&:chargeback_fields_present?)
calculated_costs['fixed_compute_metric'] = chargeback_fields_present if chargeback_fields_present
def calculate_costs(consumption, rates)
self.fixed_compute_metric = consumption.chargeback_fields_present if consumption.chargeback_fields_present

rates.each do |rate|
rate.chargeback_rate_details.each do |r|
if !chargeback_fields_present && r.fixed?
cost = 0
else
r.hours_in_interval = hours_in_interval
metric_value = r.metric_value_by(metric_rollup_records)
cost = r.cost(metric_value) * hours_in_interval
end

# add values to hash and sum
reportable_metric_and_cost_fields(r.rate_name, r.group, metric_value, cost).each do |k, val|
next unless attribute_names.include?(k)
calculated_costs[k] ||= 0
calculated_costs[k] += val
r.charge(relevant_fields, consumption).each do |field, value|
next unless self.class.attribute_names.include?(field)
self[field] = (self[field] || 0) + value
end
end
end

calculated_costs
end

def self.reportable_metric_and_cost_fields(rate_name, rate_group, metric, cost)
cost_key = "#{rate_name}_cost" # metric cost value (e.g. Storage [Used|Allocated|Fixed] Cost)
metric_key = "#{rate_name}_metric" # metric value (e.g. Storage [Used|Allocated|Fixed])
cost_group_key = "#{rate_group}_cost" # for total of metric's costs (e.g. Storage Total Cost)
metric_group_key = "#{rate_group}_metric" # for total of metrics (e.g. Storage Total)

col_hash = {}

defined_column_for_report = (report_col_options.keys & [metric_key, cost_key]).present?

if defined_column_for_report
[metric_key, metric_group_key].each { |col| col_hash[col] = metric }
[cost_key, cost_group_key, 'total_cost'].each { |col| col_hash[col] = cost }
end

col_hash
end

def self.get_group_key_ts(perf, interval, tz)
ts = perf.timestamp.in_time_zone(tz)
case interval
when "daily"
ts = ts.beginning_of_day
when "weekly"
ts = ts.beginning_of_week
when "monthly"
ts = ts.beginning_of_month
else
raise _("interval '%{interval}' is not supported") % {:interval => interval}
end

ts
end

def self.get_time_range(perf, interval, tz)
ts = perf.timestamp.in_time_zone(tz)
case interval
when "daily"
[ts.beginning_of_day, ts.end_of_day, ts.strftime("%m/%d/%Y")]
when "weekly"
s_ts = ts.beginning_of_week
e_ts = ts.end_of_week
[s_ts, e_ts, "Week of #{s_ts.strftime("%m/%d/%Y")}"]
when "monthly"
s_ts = ts.beginning_of_month
e_ts = ts.end_of_month
[s_ts, e_ts, s_ts.strftime("%b %Y")]
else
raise _("interval '%{interval}' is not supported") % {:interval => interval}
end
end

# @option options :start_time [DateTime] used with :end_time to create time range
# @option options :end_time [DateTime]
# @option options :interval_size [Fixednum] Used with :end_interval_offset to generate time range
# @option options :end_interval_offset
def self.get_report_time_range(options, interval, tz)
return options[:start_time]..options[:end_time] if options[:start_time]
raise _("Option 'interval_size' is required") if options[:interval_size].nil?

end_interval_offset = options[:end_interval_offset] || 0
start_interval_offset = (end_interval_offset + options[:interval_size] - 1)

ts = Time.now.in_time_zone(tz)
case interval
when "daily"
start_time = (ts - start_interval_offset.days).beginning_of_day.utc
end_time = (ts - end_interval_offset.days).end_of_day.utc
when "weekly"
start_time = (ts - start_interval_offset.weeks).beginning_of_week.utc
end_time = (ts - end_interval_offset.weeks).end_of_week.utc
when "monthly"
start_time = (ts - start_interval_offset.months).beginning_of_month.utc
end_time = (ts - end_interval_offset.months).end_of_month.utc
else
raise _("interval '%{interval}' is not supported") % {:interval => interval}
end

start_time..end_time
end

def self.report_cb_model(model)
Expand All @@ -271,10 +83,6 @@ def self.report_tag_field
"tag_name"
end

def self.get_rate_parents
raise "Chargeback: get_rate_parents must be implemented in child class."
end

def self.set_chargeback_report_options(rpt, edit)
rpt.cols = %w(start_date display_range)

Expand Down Expand Up @@ -337,4 +145,10 @@ def self.load_custom_attribute(custom_attribute)
entity.send(custom_attribute)
end
end

private

def relevant_fields
@relevant_fields ||= self.class.report_col_options.keys.to_set
end
end # class Chargeback
11 changes: 11 additions & 0 deletions app/models/chargeback/consumption.rb
View file
@@ -0,0 +1,11 @@
class Chargeback
class Consumption
def initialize(start_time, end_time)
@start_time, @end_time = start_time, end_time
end

def hours_in_interval
@hours_in_interval ||= (@end_time - @start_time).round / 1.hour
end
end
end
54 changes: 54 additions & 0 deletions app/models/chargeback/consumption_history.rb
View file
@@ -0,0 +1,54 @@
class Chargeback
class ConsumptionHistory
VIRTUAL_COL_USES = {
'v_derived_cpu_total_cores_used' => 'cpu_usage_rate_average'
}.freeze

def self.for_report(cb_class, options)
base_rollup = base_rollup_scope
timerange = options.report_time_range
interval_duration = options.duration_of_report_step

extra_resources = cb_class.try(:extra_resources_without_rollups) || []
timerange.step_value(interval_duration).each_cons(2) do |query_start_time, query_end_time|
extra_resources.each do |resource|
yield ConsumptionWithoutRollups.new(resource, query_start_time, query_end_time)
end

records = base_rollup.where(:timestamp => query_start_time...query_end_time, :capture_interval_name => 'hourly')
records = cb_class.where_clause(records, options)
records = Metric::Helper.remove_duplicate_timestamps(records)
next if records.empty?
_log.info("Found #{records.length} records for time range #{[query_start_time, query_end_time].inspect}")

# we are building hash with grouped calculated values
# values are grouped by resource_id and timestamp (query_start_time...query_end_time)
records.group_by(&:resource_id).each do |_, metric_rollup_records|
metric_rollup_records = metric_rollup_records.select { |x| x.resource.present? }
consumption = ConsumptionWithRollups.new(metric_rollup_records, query_start_time, query_end_time)
next if metric_rollup_records.empty?
yield(consumption)
end
end
end

def self.base_rollup_scope
base_rollup = MetricRollup.includes(
:resource => [:hardware, :tenant, :tags, :vim_performance_states, :custom_attributes,
{:container_image => :custom_attributes}],
:parent_host => :tags,
:parent_ems_cluster => :tags,
:parent_storage => :tags,
:parent_ems => :tags)
.select(*Metric::BASE_COLS).order('resource_id, timestamp')

perf_cols = MetricRollup.attribute_names
rate_cols = ChargebackRate.where(:default => true).flat_map do |rate|
rate.chargeback_rate_details.map(&:metric).select { |metric| perf_cols.include?(metric.to_s) }
end
rate_cols.map! { |x| VIRTUAL_COL_USES[x] || x }.flatten!
base_rollup.select(*rate_cols)
end
private_class_method :base_rollup_scope
end
end

0 comments on commit 970ed6b

Please sign in to comment.