Skip to content
This repository has been archived by the owner on Jun 26, 2020. It is now read-only.

Commit

Permalink
Set memcached server to local one for non-keyston services
Browse files Browse the repository at this point in the history 
We misconfigured local cache for services with change
https://review.openstack.org/#/q/Id1034e22d79c3ea6b25575d9bcf8e8750a02365d
Thus, it becomes extremely slow when a controller is down.

With this commit we revert things back to normal with local memcached
for all openstack services leaving keystone memcached shared for tokens
(this was thoroughly tested previously)

Change-Id: I8f6bbf77d27f3d8976985241deb8a948984862f5
Closes-bug: #1657727
  • Loading branch information
Vladimir Kuklin committed Jan 19, 2017
1 parent 9795ac8 commit a529033
Show file tree
Hide file tree
Showing 28 changed files with 85 additions and 59 deletions.
3 changes: 2 additions & 1 deletion deployment/puppet/openstack_tasks/manifests/aodh/aodh.pp
View file
Expand Up @@ -77,6 +77,7 @@
$public_cert = get_ssl_property($ssl_hash, $public_ssl_hash, 'keystone', 'public', 'path', [''])

$memcached_servers = hiera('memcached_servers')
$local_memcached_server = hiera('local_memcached_server')

$internal_auth_protocol = get_ssl_property($ssl_hash, {}, 'keystone', 'internal', 'protocol', 'http')
$internal_auth_address = get_ssl_property($ssl_hash, {}, 'keystone', 'internal', 'hostname', [$management_vip])
Expand Down Expand Up @@ -147,7 +148,7 @@
project_name => $tenant,
auth_uri => $keystone_auth_uri,
auth_url => $keystone_auth_url,
memcached_servers => $memcached_servers,
memcached_servers => $local_memcached_server,
}

class { '::aodh::api':
Expand Down
3 changes: 2 additions & 1 deletion deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp
View file
Expand Up @@ -45,6 +45,7 @@
$keystone_auth_uri = "${internal_auth_protocol}://${internal_auth_endpoint}:5000/"

$memcached_servers = hiera('memcached_servers')
$local_memcached_server = hiera('local_memcached_server')

#as $ssl default value in ceilometer::wsgi::apache is true and
#we use SSL at HAproxy, but not the API host we should set 'false'
Expand Down Expand Up @@ -181,7 +182,7 @@
project_name => $ceilometer_hash['tenant'],
auth_url => $keystone_auth_url,
auth_uri => $keystone_auth_uri,
memcached_servers => $memcached_servers,
memcached_servers => $local_memcached_server,
}

# Install the ceilometer-api service
Expand Down
11 changes: 6 additions & 5 deletions deployment/puppet/openstack_tasks/manifests/glance/glance.pp
View file
Expand Up @@ -28,6 +28,7 @@
$primary_controller = hiera('primary_controller')
$kombu_compression = hiera('kombu_compression', $::os_service_default)
$memcached_servers = hiera('memcached_servers')
$local_memcached_server = hiera('local_memcached_server')

$rabbit_heartbeat_timeout_threshold = pick($glance_hash['rabbit_heartbeat_timeout_threshold'], $rabbit_hash['heartbeat_timeout_threshold'], 60)
$rabbit_heartbeat_rate = pick($glance_hash['rabbit_heartbeat_rate'], $rabbit_hash['rabbit_heartbeat_rate'], 2)
Expand Down Expand Up @@ -129,8 +130,8 @@
project_name => $glance_tenant,
auth_url => $auth_url,
auth_uri => $auth_uri,
token_cache_time => '-1',
memcached_servers => $memcached_servers,
token_cache_time => '300',
memcached_servers => $local_memcached_server,
}

# Install and configure glance-api
Expand Down Expand Up @@ -182,8 +183,8 @@
project_name => $glance_glare_tenant,
auth_url => $auth_url,
auth_uri => $auth_uri,
token_cache_time => '-1',
memcached_servers => $memcached_servers,
token_cache_time => '300',
memcached_servers => $local_memcached_server,
}

class { '::glance::glare':
Expand Down Expand Up @@ -211,7 +212,7 @@
project_name => $glance_tenant,
auth_url => $auth_url,
auth_uri => $auth_uri,
memcached_servers => $memcached_servers,
memcached_servers => $local_memcached_server,
}

# Install and configure glance-registry
Expand Down
5 changes: 3 additions & 2 deletions deployment/puppet/openstack_tasks/manifests/heat/heat.pp
View file
Expand Up @@ -52,6 +52,7 @@
$deployment_mode = hiera('deployment_mode')
$bind_host = get_network_role_property('heat/api', 'ipaddr')
$memcached_servers = hiera('memcached_servers')
$local_memcached_server = hiera('local_memcached_server')
$keystone_user = pick($heat_hash['user'], 'heat')
$keystone_tenant = pick($heat_hash['tenant'], 'services')
$region = hiera('region', 'RegionOne')
Expand Down Expand Up @@ -119,7 +120,7 @@
heat_config {
'cache/enabled': value => true;
'cache/backend': value => 'oslo_cache.memcache_pool';
'cache/memcache_servers': value => join(any2array($memcached_servers), ',')
'cache/memcache_servers': value => $local_memcached_server;
}

# TODO(aschultz): ubuntu does not have a heat docker package
Expand Down Expand Up @@ -187,7 +188,7 @@
project_name => $keystone_tenant,
auth_uri => $auth_uri,
auth_url => $auth_url,
memcached_servers => $memcached_servers,
memcached_servers => $local_memcached_server,
}

# Common configuration, logging and RPC
Expand Down
3 changes: 2 additions & 1 deletion deployment/puppet/openstack_tasks/manifests/ironic/ironic.pp
View file
Expand Up @@ -26,6 +26,7 @@
$kombu_compression = hiera('kombu_compression', $::os_service_default)

$memcached_servers = hiera('memcached_servers')
$local_memcached_server = hiera('local_memcached_server')

$db_type = pick($ironic_hash['db_type'], 'mysql+pymysql')
$db_host = pick($ironic_hash['db_host'], $database_vip)
Expand Down Expand Up @@ -98,7 +99,7 @@
project_name => $ironic_tenant,
auth_url => $admin_identity_uri,
auth_uri => $internal_auth_url,
memcached_servers => $memcached_servers,
memcached_servers => $local_memcached_server,
}

class { '::ironic::api':
Expand Down
3 changes: 2 additions & 1 deletion deployment/puppet/openstack_tasks/manifests/murano/murano.pp
View file
Expand Up @@ -28,6 +28,7 @@
$primary_controller = hiera('primary_controller')
$kombu_compression = hiera('kombu_compression', $::os_service_default)
$memcached_servers = hiera('memcached_servers')
$local_memcached_server = hiera('local_memcached_server')

$public_auth_protocol = get_ssl_property($ssl_hash, $public_ssl_hash, 'keystone', 'public', 'protocol', 'http')
$public_auth_address = get_ssl_property($ssl_hash, $public_ssl_hash, 'keystone', 'public', 'hostname', [$public_ip])
Expand Down Expand Up @@ -150,7 +151,7 @@
external_network => $external_network,
use_trusts => true,
kombu_compression => $kombu_compression,
memcached_servers => $memcached_servers,
memcached_servers => $local_memcached_server,
}

class { '::murano::api':
Expand Down
3 changes: 2 additions & 1 deletion deployment/puppet/openstack_tasks/manifests/openstack_cinder/openstack_cinder.pp
View file
Expand Up @@ -24,6 +24,7 @@
$proxy_port = hiera('proxy_port', '8080')
$kombu_compression = hiera('kombu_compression', $::os_service_default)
$memcached_servers = hiera('memcached_servers')
$local_memcached_server = hiera('local_memcached_server')
$default_volume_type = pick($cinder_hash['default_volume_type'], $::os_service_default)
$db_type = pick($cinder_hash['db_type'], 'mysql+pymysql')
$db_host = pick($cinder_hash['db_host'], hiera('database_vip'))
Expand Down Expand Up @@ -163,7 +164,7 @@
username => $keystone_user,
project_name => $keystone_tenant,
password => $cinder_user_password,
memcached_servers => $memcached_servers,
memcached_servers => $local_memcached_server,
auth_version => $keystone_api,
}

Expand Down
12 changes: 3 additions & 9 deletions deployment/puppet/openstack_tasks/manifests/openstack_controller/openstack_controller.pp
View file
Expand Up @@ -153,13 +153,7 @@
}

$memcached_servers = hiera('memcached_servers')

# LP1621541 In order to increase nova performance after failover,
# we need to point nova to local memcached instance for keystone tokens,
# in future we can consider moving memcached under HAproxy
$memcached_port = hiera('memcache_server_port', '11211')
$memcached_address = get_network_role_property('mgmt/memcache', 'ipaddr')
$memcached_authtoken_server = "${memcached_address}:${memcached_port}"
$local_memcached_server = hiera('local_memcached_server')

$debug = pick($openstack_controller_hash['debug'], hiera('debug', true))

Expand Down Expand Up @@ -211,7 +205,7 @@
class { '::nova::cache':
enabled => true,
backend => 'oslo_cache.memcache_pool',
memcache_servers => $memcached_servers,
memcache_servers => $local_memcached_server,
}
} else {
ensure_packages($pymemcache_package_name)
Expand Down Expand Up @@ -264,7 +258,7 @@
auth_url => $keystone_auth_url,
auth_uri => $keystone_auth_uri,
auth_version => pick($nova_hash['auth_version'], $::os_service_default),
memcached_servers => $memcached_authtoken_server,
memcached_servers => $local_memcached_server,
}

# Configure nova-api
Expand Down
3 changes: 2 additions & 1 deletion deployment/puppet/openstack_tasks/manifests/openstack_network/server_config.pp
View file
Expand Up @@ -50,6 +50,7 @@
$region_name = hiera('region', 'RegionOne')
$auth_endpoint_type = 'internalURL'
$memcached_servers = hiera('memcached_servers')
$local_memcached_server = hiera('local_memcached_server')

$ssl_hash = hiera_hash('use_ssl', {})

Expand Down Expand Up @@ -188,7 +189,7 @@
region_name => $region_name,
auth_url => $auth_url,
auth_uri => $auth_uri,
memcached_servers => $memcached_servers,
memcached_servers => $local_memcached_server,
}

class { '::neutron::server':
Expand Down
3 changes: 2 additions & 1 deletion deployment/puppet/openstack_tasks/manifests/roles/compute.pp
View file
Expand Up @@ -60,6 +60,7 @@
##CALCULATED PARAMETERS

$memcached_servers = hiera('memcached_servers')
$local_memcached_server = hiera('local_memcached_server')

# TODO(xarses): We need to validate this is needed
if ($storage_hash['volumes_lvm']) {
Expand Down Expand Up @@ -265,7 +266,7 @@
class { '::nova::cache':
enabled => $nova_cache,
backend => 'oslo_cache.memcache_pool',
memcache_servers => $memcached_servers,
memcache_servers => $local_memcached_server,
}

class { '::nova::availability_zone':
Expand Down
3 changes: 2 additions & 1 deletion deployment/puppet/openstack_tasks/manifests/roles/ironic_conductor.pp
View file
Expand Up @@ -31,6 +31,7 @@
$ironic_user_password = pick($ironic_hash['user_password'],'ironic')
$ironic_swift_tempurl_key = pick($ironic_hash['swift_tempurl_key'],'ironic')
$memcached_servers = hiera('memcached_servers')
$local_memcached_server = hiera('local_memcached_server')

$ssl_hash = hiera('use_ssl', {})
$neutron_endpoint_default = hiera('neutron_endpoint', $management_vip)
Expand Down Expand Up @@ -116,7 +117,7 @@
'keystone_authtoken/admin_tenant_name': value => $ironic_tenant;
'keystone_authtoken/admin_user': value => $ironic_user;
'keystone_authtoken/admin_password': value => $ironic_user_password, secret => true;
'keystone_authtoken/memcached_servers': value => join(any2array($memcached_servers), ',');
'keystone_authtoken/memcached_servers': value => $local_memcached_server;
'glance/swift_endpoint_url': value => "http://${baremetal_vip}:8080";
'glance/temp_url_endpoint_type': value => $temp_url_endpoint_type;
}
Expand Down
3 changes: 2 additions & 1 deletion deployment/puppet/openstack_tasks/manifests/sahara/sahara.pp
View file
Expand Up @@ -32,6 +32,7 @@
$admin_identity_uri = "${admin_identity_protocol}://${admin_identity_address}:35357"
$kombu_compression = hiera('kombu_compression', $::os_service_default)
$memcached_servers = hiera('memcached_servers')
$local_memcached_server = hiera('local_memcached_server')

#################################################################

Expand Down Expand Up @@ -111,7 +112,7 @@
rabbit_port => $amqp_port,
rabbit_hosts => split($amqp_hosts, ','),
kombu_compression => $kombu_compression,
memcached_servers => $memcached_servers,
memcached_servers => $local_memcached_server,
}

if $public_ssl_hash['services'] {
Expand Down
3 changes: 2 additions & 1 deletion deployment/puppet/openstack_tasks/manifests/swift/proxy_storage.pp
View file
Expand Up @@ -13,6 +13,7 @@
$swift_host_ip_map = get_node_to_ipaddr_map_by_network_role(hiera_hash('swift_proxies', {}), 'swift/api')
$swift_proxies_addr_list = sorted_hosts($swift_host_ip_map, 'ip', 'ip')
$memcached_servers = hiera('memcached_servers')
$local_memcached_server = hiera('local_memcached_server')
$is_primary_swift_proxy = hiera('is_primary_swift_proxy', false)
$proxy_port = hiera('proxy_port', '8080')
$storage_hash = hiera_hash('storage')
Expand Down Expand Up @@ -79,7 +80,7 @@
class { 'openstack_tasks::swift::parts::proxy':
swift_user_password => $swift_hash['user_password'],
swift_operator_roles => $swift_operator_roles,
memcached_servers => $memcached_servers,
memcached_servers => $local_memcached_server,
ring_part_power => $ring_part_power,
ring_replicas => $ring_replicas,
primary_proxy => $is_primary_swift_proxy,
Expand Down
6 changes: 6 additions & 0 deletions deployment/puppet/osnailyfacter/manifests/globals/globals.pp
View file
Expand Up @@ -519,6 +519,12 @@
$memcached_port = hiera('memcache_server_port', '11211')
$memcached_servers = suffix($memcached_addresses, ":${memcached_port}")

# LP1621541 In order to increase nova performance after failover,
# we need to point nova to local memcached instance for keystone tokens,
# in future we can consider moving memcached under HAproxy
$memcached_bind_address = get_network_role_property('mgmt/memcache', 'ipaddr')
$local_memcached_server = "${memcached_bind_address}:${memcached_port}"

$cinder_backends = {
'volumes_ceph' => $storage_hash['volumes_ceph'] ? { true => 'RBD-backend', default => false },
'volumes_lvm' => $storage_hash['volumes_lvm'] ? { true => 'LVM-backend', default => false },
Expand Down
1 change: 1 addition & 0 deletions deployment/puppet/osnailyfacter/templates/globals_yaml.erb
View file
Expand Up @@ -122,6 +122,7 @@
<% globals.store "memcached_addresses", @memcached_addresses -%>
<% globals.store "memcached_port", @memcached_port -%>
<% globals.store "memcached_servers", @memcached_servers -%>
<% globals.store "local_memcached_server", @local_memcached_server -%>
<% globals.store "swift_master_role", @swift_master_role -%>
<% globals.store "swift_nodes", @swift_nodes -%>
<% globals.store "swift_proxies", @swift_proxies -%>
Expand Down
5 changes: 3 additions & 2 deletions tests/noop/spec/hosts/aodh/aodh_spec.rb
View file
Expand Up @@ -16,6 +16,7 @@
end

let(:memcached_servers) { Noop.hiera 'memcached_servers' }
let(:local_memcached_server) { Noop.hiera 'local_memcached_server' }

let(:aodh_api_bind_host) do
aodh_api_bind_host = Noop.puppet_function 'get_network_role_property', 'aodh/api', 'ipaddr'
Expand Down Expand Up @@ -72,7 +73,7 @@
'project_name' => tenant,
'auth_uri' => keystone_auth_uri,
'auth_url' => keystone_auth_url,
'memcached_servers' => memcached_servers,
'memcached_servers' => local_memcached_server,
)
end

Expand Down Expand Up @@ -108,7 +109,7 @@


it 'should configure "keystone_authtoken/" section' do
should contain_aodh_config('keystone_authtoken/memcached_servers').with(:value => memcached_servers.join(','))
should contain_aodh_config('keystone_authtoken/memcached_servers').with(:value => local_memcached_server)
should contain_aodh_config('keystone_authtoken/auth_url').with(:value => keystone_auth_url)
should contain_aodh_config('keystone_authtoken/auth_uri').with(:value => keystone_auth_uri)
should contain_aodh_config('keystone_authtoken/project_name').with(:value => tenant)
Expand Down
5 changes: 3 additions & 2 deletions tests/noop/spec/hosts/ceilometer/controller_spec.rb
View file
Expand Up @@ -58,6 +58,7 @@
ssl = 'false'

let(:memcached_servers) { Noop.hiera 'memcached_servers' }
let(:local_memcached_server) { Noop.hiera 'local_memcached_server' }
let (:api_bind_address) do
api_bind_address = Noop.puppet_function('get_network_role_property', 'ceilometer/api', 'ipaddr')
end
Expand Down Expand Up @@ -101,7 +102,7 @@
'project_name' => ceilometer_tenant,
'auth_url' => keystone_auth_url,
'auth_uri' => keystone_auth_uri,
'memcached_servers' => memcached_servers,
'memcached_servers' => local_memcached_server
)
end

Expand All @@ -118,7 +119,7 @@
should contain_ceilometer_config('keystone_authtoken/project_name').with(:value => ceilometer_tenant)
should contain_ceilometer_config('keystone_authtoken/auth_uri').with(:value => keystone_auth_uri)
should contain_ceilometer_config('keystone_authtoken/auth_url').with(:value => keystone_auth_url)
should contain_ceilometer_config('keystone_authtoken/memcached_servers').with(:value => memcached_servers.join(','))
should contain_ceilometer_config('keystone_authtoken/memcached_servers').with(:value => local_memcached_server)
end

it 'should configure interface (ex. OS ENDPOINT TYPE) for ceilometer' do
Expand Down

0 comments on commit a529033

Please sign in to comment.