From 2ed3fa6db4e7598b61ea3bccf7699839525455a4 Mon Sep 17 00:00:00 2001 From: Sally O'Malley Date: Thu, 28 Jul 2016 07:27:22 -0400 Subject: [PATCH] make node buffer configurable --- node/lib/openshift-origin-node/model/node.rb | 3 +++ ...origin-msg-broker-mcollective.conf.example | 4 ++++ ...openshift-origin-msg-broker-mcollective.rb | 3 ++- ...mcollective_application_container_proxy.rb | 22 +++++++++++++++++-- .../mcollective/facts/openshift_facts.rb | 1 + 5 files changed, 30 insertions(+), 3 deletions(-) diff --git a/node/lib/openshift-origin-node/model/node.rb b/node/lib/openshift-origin-node/model/node.rb index ff63336267c..6cb17af4ddc 100644 --- a/node/lib/openshift-origin-node/model/node.rb +++ b/node/lib/openshift-origin-node/model/node.rb @@ -44,6 +44,7 @@ class Node < Model DEFAULT_QUOTA_FILES = '80000' DEFAULT_NO_OVERCOMMIT_ACTIVE = false DEFAULT_MAX_ACTIVE_GEARS = 0 + DEFAULT_NODE_DF_BUFFER = 0.05 @@resource_limits_cache = nil @@ -336,6 +337,8 @@ def self.node_utilization # use max_{active_,}gears if set in resource limits, or fall back to old "apps" names res['max_active_gears'] = (resource.get('max_active_gears') or resource.get('max_active_apps') or DEFAULT_MAX_ACTIVE_GEARS) + res['node_df_buffer'] = resource.get('node_df_buffer', DEFAULT_NODE_DF_BUFFER) + # # Count number of git repos and gear status counts diff --git a/plugins/msg-broker/mcollective/conf/openshift-origin-msg-broker-mcollective.conf.example b/plugins/msg-broker/mcollective/conf/openshift-origin-msg-broker-mcollective.conf.example index aaa187f759f..3fd9bce206f 100644 --- a/plugins/msg-broker/mcollective/conf/openshift-origin-msg-broker-mcollective.conf.example +++ b/plugins/msg-broker/mcollective/conf/openshift-origin-msg-broker-mcollective.conf.example @@ -39,3 +39,7 @@ NODE_RSYNC_ADDRESS="ip_address" # can be moved if a gear is over quota limits, up to this quota multiplier. QUOTA_BLOCKS_BUFFER=1.2 QUOTA_INODES_BUFFER=1.2 + +# Node disk free space buffer. This will ensure that gears will only be moved to +# nodes that will have at least this amount of free disk space buffer after the move. +NODE_DF_BUFFER=0.05 diff --git a/plugins/msg-broker/mcollective/config/initializers/openshift-origin-msg-broker-mcollective.rb b/plugins/msg-broker/mcollective/config/initializers/openshift-origin-msg-broker-mcollective.rb index 62257a6839e..6b533149fb5 100644 --- a/plugins/msg-broker/mcollective/config/initializers/openshift-origin-msg-broker-mcollective.rb +++ b/plugins/msg-broker/mcollective/config/initializers/openshift-origin-msg-broker-mcollective.rb @@ -39,7 +39,8 @@ :node_profile_enabled => conf.get_bool("NODE_PROFILE_ENABLED", "false"), :node_rsync_address => conf.get("NODE_RSYNC_ADDRESS", "ip_address"), :quota_blocks_buffer => conf.get("QUOTA_BLOCKS_BUFFER", 1.2), - :quota_inodes_buffer => conf.get("QUOTA_INODES_BUFFER", 1.2) + :quota_inodes_buffer => conf.get("QUOTA_INODES_BUFFER", 1.2), + :node_df_buffer => conf.get("NODE_DF_BUFFER", 0.05) } end end diff --git a/plugins/msg-broker/mcollective/lib/openshift/mcollective_application_container_proxy.rb b/plugins/msg-broker/mcollective/lib/openshift/mcollective_application_container_proxy.rb index a484bc30988..5e03e4e22a9 100755 --- a/plugins/msg-broker/mcollective/lib/openshift/mcollective_application_container_proxy.rb +++ b/plugins/msg-broker/mcollective/lib/openshift/mcollective_application_container_proxy.rb @@ -39,6 +39,8 @@ def initialize(id, district=nil, blocks_multiplier=1, inodes_multiplier=1) inodes_multiplier=Rails.configuration.msg_broker[:quota_inodes_buffer] @blocks_multiplier = blocks_multiplier @inodes_multiplier = inodes_multiplier + node_df_buffer=Rails.configuration.msg_broker[:node_df_buffer] + @node_df_buffer = node_df_buffer @id = id @district = district @disable_print_debug = false @@ -816,6 +818,20 @@ def get_node_total_size rpc_get_fact_direct('node_total_size').to_i end + # <> + # Get the disk free buffer of a Node + # + # RETURNS: + # * Float: the disk free buffer of a Node + # + # NOTES: + # * method on Node + # * calls rpc_get_fact_direct + # + def get_node_df_buffer + rpc_get_fact_direct('node_df_buffer').to_f + end + # # Add a component to an existing gear on the node # @@ -2413,11 +2429,13 @@ def rsync_destination_container(gear, destination_container, destination_distric log_debug "DEBUG: Gear platform is '#{platform}'" destination_avail_space = destination_container.get_node_disk_free destination_total_space = destination_container.get_node_total_size + destination_df_buffer = destination_container.get_node_df_buffer # check here to make sure addition of gear to destination_container # will not result in > 95% full destination_container - if (destination_avail_space - source_used_blocks.to_f)/destination_total_space < 0.05 - raise OpenShift::NodeUnavailableException.new("Gear '#{gear.uuid}' cannot be moved to '#{destination_container.id}'. Not enough disk space, node would be > 95% full after move.", 140) + if (destination_avail_space - source_used_blocks.to_f)/destination_total_space < destination_df_buffer + max_value = sprintf('%.0f', (1.0 - destination_df_buffer)*100) + raise OpenShift::NodeUnavailableException.new("Gear '#{gear.uuid}' cannot be moved to '#{destination_container.id}'. Not enough disk space, node would be using > #{max_value}% of it's disk space after the move.", 140) end log_debug "DEBUG: Creating new account for gear '#{gear.uuid}' on #{destination_container.id}" diff --git a/plugins/msg-node/mcollective/facts/openshift_facts.rb b/plugins/msg-node/mcollective/facts/openshift_facts.rb index a56da5594ec..1381b822ab8 100644 --- a/plugins/msg-node/mcollective/facts/openshift_facts.rb +++ b/plugins/msg-node/mcollective/facts/openshift_facts.rb @@ -53,6 +53,7 @@ def get_node_config_value(key, default) Facter.add(:node_disk_free) { setcode { results['node_disk_free'] } } Facter.add(:node_total_size) { setcode { results['node_total_size'] } } Facter.add(:node_profile) { setcode { results['node_profile'] } } +Facter.add(:node_df_buffer) { setcode { results['node_df_buffer'] } } Facter.add(:max_active_gears) { setcode { results['max_active_gears'] || '0' } } Facter.add(:no_overcommit_active) { setcode { results['no_overcommit_active'] || false } } Facter.add(:quota_blocks) { setcode { results['quota_blocks'] } }