BZ #1180158: Ceilometer coordination resilience with redis-sentinel

https://bugzilla.redhat.com/show_bug.cgi?id=1180158 We use redis-sentinel to monitor the redis instances running on controller nodes, and to detect when mastership needs to be failed over to a slave. The sentinel cluster has a default quorum of 2, so can survive the loss of one controller on a minimally sized HA controller plane. The ceilometer backend_url contains a list of fallback sentinels, similar in style to the mongodb connect URL, except that the additional addresses are specified as query parameters.
redhat-openstack · Jan 13, 2015 · 195822a · 195822a
1 parent f77577e
commit 195822a
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 11 deletions.
diff --git a/puppet/modules/quickstack/manifests/db/redis.pp b/puppet/modules/quickstack/manifests/db/redis.pp
@@ -11,11 +11,23 @@
 # [*port*]
 #   (optional) Configure which port to listen on.
 #   Defaults to '6379'
+#
+# [*monitoring_port*]
+#   (optional) Configure which port for sentinel to listen on.
+#   Defaults to '26379'
+#
+# [*monitoring_group*]
+#   (optional) Configure which group name for sentinel to use.
+#   Defaults to '26379'
+
 
 
 class quickstack::db::redis(
   $bind_host             = '127.0.0.1',
   $port                  = '6379',
+  $master_host           = '127.0.0.1',
+  $monitoring_port       = '26379',
+  $monitoring_group      = 'sentinel-group',
   $slaveof               = undef,
 ) {
 
@@ -26,4 +38,11 @@
     daemonize  => false,
     slaveof    => $slaveof,
   }
+
+  class { '::redis::sentinel':
+    master_name   => $monitoring_group,
+    redis_host    => $master_host,
+    redis_port    => $port,
+    sentinel_port => $monitoring_port,
+  }
 }
diff --git a/puppet/modules/quickstack/manifests/pacemaker/ceilometer.pp b/puppet/modules/quickstack/manifests/pacemaker/ceilometer.pp
@@ -1,10 +1,12 @@
 class quickstack::pacemaker::ceilometer (
   $ceilometer_metering_secret,
-  $memcached_port            = '11211',
-  $db_port                   = '27017',
-  $verbose                   = 'false',
-  $coordination_backend      = 'redis',
-  $coordination_backend_port = '6379',
+  $memcached_port                = '11211',
+  $db_port                       = '27017',
+  $verbose                       = 'false',
+  $coordination_backend          = 'redis',
+  $coordination_backend_port     = '6379',
+  $coordination_monitoring_port  = '26379',
+  $coordination_monitoring_group = 'ceilometer-sentinel',
 ) {
 
   include quickstack::pacemaker::common
@@ -30,19 +32,24 @@
     }
 
     if $coordination_backend == 'redis' {
-      $initial_master = $backend_ips[0]
-      $coordination_url = "redis://${initial_master}:${coordination_backend_port}"
+      $_initial_master = $backend_ips[0]
+      $_fallback_sentinels = split(inline_template('<%= @backend_ips.map {
+          |x| "&sentinel_fallback="+x+":"+@coordination_monitoring_port }.join(",")%>'),",")
+      $coordination_url = "redis://${_initial_master}:${coordination_monitoring_port}?sentinel=${coordination_monitoring_group}${_fallback_sentinels}"
 
-      if has_interface_with("ipaddress", $initial_master) {
+      if has_interface_with("ipaddress", $_initial_master) {
         $_slaveof = undef
       } else {
-        $_slaveof = "${initial_master} ${coordination_backend_port}"
+        $_slaveof = "${_initial_master} ${coordination_backend_port}"
       }
 
       class { '::quickstack::pacemaker::redis':
         bind_host => map_params("local_bind_addr"),
         port      => $coordination_backend_port,
-        slaveof   => $_slaveof,
+        master_host => $_initial_master,
+        monitoring_port => $coordination_monitoring_port,
+        monitoring_group => $coordination_monitoring_group,
+        slaveof  => $_slaveof,
       }
     } else {
       $coordination_url = undef

diff --git a/puppet/modules/quickstack/manifests/pacemaker/redis.pp b/puppet/modules/quickstack/manifests/pacemaker/redis.pp
@@ -1,16 +1,22 @@
 class quickstack::pacemaker::redis(
   $bind_host             = '127.0.0.1',
   $port                  = '6379',
+  $master_host           = '127.0.0.1',
+  $monitoring_port       = '26379',
+  $monitoring_group      = 'sentinel-group',
   $slaveof               = undef,
 ) {
 
   class {'::quickstack::firewall::redis':
-    ports => [$port],
+    ports => [$port, $monitoring_port],
   }
 
   class {'::quickstack::db::redis':
     bind_host => $bind_host,
     port      => $port,
+    master_host => $master_host,
+    monitoring_port => $monitoring_port,
+    monitoring_group => $monitoring_group,
     slaveof   => $slaveof,
   }
 }