Skip to content
This repository has been archived by the owner on Aug 29, 2018. It is now read-only.

Commit

Permalink
Bug 1234603: spreading gears for an app evenly across zones
Browse files Browse the repository at this point in the history
  • Loading branch information
Abhishek Gupta committed Sep 1, 2015
1 parent 2314648 commit 632c425
Show file tree
Hide file tree
Showing 3 changed files with 68 additions and 19 deletions.
6 changes: 3 additions & 3 deletions broker/test/functional/node_selection_test.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ def self.select_best_fit_node_impl(node_list, app_props, current_gears, comp_lis
test "least preferred node selection" do
OpenShift::ApplicationContainerProxy.node_selector_plugin = nil
begin
p = OpenShift::ApplicationContainerProxy.find_available(:node_profile => "small", :least_preferred_servers => [@@server_id])
p = OpenShift::ApplicationContainerProxy.find_available(:node_profile => "small", :existing_gears_hosting => {@@server_id => 1})
assert p.id == @@server_id, "The expected node was not returned"
rescue OpenShift::NodeUnavailableException
assert false, "The least preferred node was not selected and NodeUnavailableException was raised"
Expand Down Expand Up @@ -136,15 +136,15 @@ def self.select_best_fit_node_impl(node_list, app_props, current_gears, comp_lis
# test when multiple web proxies are allowed on the same node
begin
assert Rails.configuration.openshift[:allow_multiple_haproxy_on_node] == true, "Broker configuration for allowing multiple web proxies on the same node is not as expected"
OpenShift::ApplicationContainerProxy.find_available(:node_profile => "small", :least_preferred_servers => new_gear.non_ha_server_identities, :restricted_servers => new_gear.restricted_server_identities, :gear => new_gear)
OpenShift::ApplicationContainerProxy.find_available(:node_profile => "small", :existing_gears_hosting => new_gear.server_identities_gears_map, :restricted_servers => new_gear.restricted_server_identities, :gear => new_gear)
rescue OpenShift::NodeUnavailableException
assert false, "NodeUnavailableException was raised even though multiple web proxies are allowed on the same node"
end

# test when multiple web proxies are not allowed on the same node
# this is being simulated by setting the restricted server identities directly instead of relying on the gear
begin
OpenShift::ApplicationContainerProxy.find_available(:node_profile => "small", :least_preferred_servers => new_gear.non_ha_server_identities, :restricted_servers => [@@server_id], :gear => new_gear)
OpenShift::ApplicationContainerProxy.find_available(:node_profile => "small", :existing_gears_hosting => new_gear.server_identities_gears_map, :restricted_servers => [@@server_id], :gear => new_gear)
rescue OpenShift::NodeUnavailableException
#this is expected
else
Expand Down
18 changes: 17 additions & 1 deletion controller/app/models/gear.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,8 @@ def self.valid_gear_size?(gear_size)
def reserve_uid(gear_size = nil, region_id = nil)
gear_size = group_instance.gear_size unless gear_size
opts = { :node_profile => gear_size, :least_preferred_servers => non_ha_server_identities,
:restricted_servers => restricted_server_identities, :gear => self, :platform => group_instance.platform, :region_id => region_id}
:restricted_servers => restricted_server_identities, :existing_gears_hosting => server_identities_gears_map,
:gear => self, :platform => group_instance.platform, :region_id => region_id}
@container = OpenShift::ApplicationContainerProxy.find_available(opts)
reserved_uid = @container.reserve_uid
Application.where({"_id" => application._id, "gears.uuid" => self.uuid}).update({"$set" => {"gears.$.server_identity" => @container.id, "gears.$.uid" => reserved_uid}})
Expand Down Expand Up @@ -315,6 +316,21 @@ def non_ha_server_identities
group_instance.server_identities.uniq
end

# Gets the map of server identities to the number of gears (for this particular app) hosted on them
# == Returns:
# @return [Array] List of server identities where gears from this gear's group instance are hosted.
# @return Hash of server identities => number of gears for this application that are hosted on them
def server_identities_gears_map
si_map = {}
group_instance.gears.each do |gear|
if gear.server_identity.present?
si_map[gear.server_identity] = 0 unless si_map[gear.server_identity]
si_map[gear.server_identity] += 1
end
end
si_map
end

# Gets the list of server identities where this gear cannot be hosted
# == Returns:
# @return [Array] List of server identities where this gear cannot be hosted
Expand Down
63 changes: 48 additions & 15 deletions plugins/msg-broker/mcollective/lib/openshift/mcollective_application_container_proxy.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ def initialize(id, district=nil)
# * node_profile: string identifier for a set of node characteristics
# * district_uuid: identifier for the district
# * least_preferred_servers: list of server identities that are least preferred. These could be the ones that won't allow the gear group to be highly available
# * existing_gears_hosting: map of server identities to the number of gears (of the application that the gear being scheduled belongs to) hosted on them
# * gear_exists_in_district: true if the gear belongs to a node in the same district
# * required_uid: the uid that is required to be available in the destination district
#
Expand Down Expand Up @@ -2227,7 +2228,6 @@ def resolve_destination(gear, destination_container, destination_district_uuid,
required_uid = nil
end

least_preferred_servers = [source_container.id]
opts = { :node_profile => destination_gear_size, :district_uuid => destination_district_uuid,
:gear => gear, :gear_exists_in_district => gear_exists_in_district, :required_uid => required_uid }

Expand Down Expand Up @@ -3060,6 +3060,7 @@ def self.known_server_identities(force_rediscovery=false, rpc_opts=nil)
# * node_profile: String identifier for a set of node characteristics
# * district_uuid: String identifier for the district
# * least_preferred_servers: list of server identities that are least preferred. These could be the ones that won't allow the gear group to be highly available
# * existing_gears_hosting: map of server identities to the number of gears (of the application that the gear being scheduled belongs to) hosted on them
# * force_rediscovery: Boolean
# * gear_exists_in_district: Boolean - true if the gear belongs to a node in the same district
# * required_uid: String - the uid that is required to be available in the destination district
Expand All @@ -3085,6 +3086,7 @@ def self.rpc_find_all_available(opts=nil)
district_uuid = opts[:district_uuid]
least_preferred_servers = opts[:least_preferred_servers]
restricted_servers = opts[:restricted_servers]
existing_gears_hosting = opts[:existing_gears_hosting]
gear = opts[:gear]
force_rediscovery = opts[:force_rediscovery] if opts[:force_rediscovery]
gear_exists_in_district = opts[:gear_exists_in_district] if opts[:gear_exists_in_district]
Expand Down Expand Up @@ -3221,11 +3223,15 @@ def self.rpc_find_all_available(opts=nil)

# Check if we have min zones for app gear group
zones_consumed_capacity = {}
zone_app_gears_map = {}
server_infos.each do |server_info|
zone_id = server_info.zone_id
if zone_id
zones_consumed_capacity[zone_id] = 0 unless zones_consumed_capacity[zone_id]
zones_consumed_capacity[zone_id] += server_info.node_consumed_capacity
# initialize zone_app_gears_map
# it holds the number of existing gears for this particular app in a given zone
zone_app_gears_map[zone_id] = 0 unless zone_app_gears_map[server.zone_id]
end
end
available_zones_count = zones_consumed_capacity.keys.length
Expand All @@ -3252,32 +3258,59 @@ def self.rpc_find_all_available(opts=nil)
end
end

# Find least preferred zones
least_preferred_zone_ids = []
least_preferred_servers.each do |server_identity|
# Find the zones that have existing gears
existing_gears_zone_ids = []
existing_gears_hosting.each do |server_identity, gear_count|
next unless server_identity
server = District.find_server(server_identity, districts)
least_preferred_zone_ids << server.zone_id if server.zone_id
end if least_preferred_servers.present?
least_preferred_zone_ids = least_preferred_zone_ids.uniq
if server.zone_id
existing_gears_zone_ids << server.zone_id
# set the number of existing gears for this particular app for a given zone
zone_app_gears_map[server.zone_id] = 0 unless zone_app_gears_map[server.zone_id]
zone_app_gears_map[server.zone_id] += gear_count
end
end if existing_gears_hosting.present?
existing_gears_zone_ids = existing_gears_zone_ids.uniq

if least_preferred_zone_ids.present?
if existing_gears_zone_ids.present?
available_zone_ids = zones_consumed_capacity.keys
# Consider least preferred zones only when we have no available zone that's not in least preferred zones.
unless (available_zone_ids - least_preferred_zone_ids).empty?
# Consider removing zones with existing gears (for this app) only when we have other available zones that are not in this list.
unless (available_zone_ids - existing_gears_zone_ids).empty?
# Remove least preferred zones from the list, ensuring there is at least one server remaining
server_infos.delete_if { |server_info| (server_infos.length > 1) && least_preferred_zone_ids.include?(server_info.zone_id) }
zones_consumed_capacity.delete_if { |zone_id, capacity| least_preferred_zone_ids.include?(zone_id) }
server_infos.delete_if { |server_info| (server_infos.length > 1) && existing_gears_zone_ids.include?(server_info.zone_id) }
zones_consumed_capacity.delete_if { |zone_id, capacity| existing_gears_zone_ids.include?(zone_id) }
end
end

# Distribute zones evenly, pick zones that are least consumed/have max available capacity
# Distribute gears for this app across zones evenly
# identify zones that have the least number of gears for this app
min_zone_gears = zone_app_gears_map.values.min || 0
preferred_zones = zone_app_gears_map.map { |server_identity, gears| server_identity if gears <= min_zone_gears }.compact

# find the selected zones in the zones_consumed_capacity map
zones_consumed_capacity.select! { |zone_id, _| preferred_zones.include? zone_id }

# Distribute all gears (across all apps) across zones evenly
# identify the least consumed zones (have max available capacity)
min_consumed_capacity = zones_consumed_capacity.values.min
preferred_zones = zones_consumed_capacity.select { |zone_id, capacity| capacity <= min_consumed_capacity }.keys
preferred_zones = zones_consumed_capacity.map { |zone_id, capacity| zone_id if capacity <= min_consumed_capacity }.compact

# Remove the servers from the list that does not belong to preferred zones
# Remove the servers from the list that do not belong to preferred zones
server_infos.delete_if { |server_info| (server_infos.length > 1) && !preferred_zones.include?(server_info.zone_id) }
end

if existing_gears_hosting.present?
# find out the minimum number of gears for this app that are present on any node
# filter out the nodes that have more gears on them than this minimum number
if server_infos.all? {|server| existing_gears_hosting.keys.include?(server.name) }
min_app_gears_on_nodes = existing_gears_hosting.values.min || 0
existing_gears_hosting.select! {|_, gear_count| gear_count > min_app_gears_on_nodes}
server_infos.delete_if { |server_info| (server_infos.length > 1) && existing_gears_hosting.keys.include?(server_info.name) }
server_infos
else
server_infos.delete_if { |server_info| (server_infos.length > 1) && existing_gears_hosting.keys.include?(server_info.name) }
end
end
end

# Remove the least preferred servers from the list, ensuring there is at least one server remaining
Expand Down

0 comments on commit 632c425

Please sign in to comment.