Adds keepalived based VRRPIPManager

This adds a new IP manager driver for configuring addresses
and routes via keepalived instead of directly.  It used when
the logical resource is configured to be highly-available,
according to configuration pushed by the orchestrator.

We rely on a 'ha_resource' flag attached to the main config
dict to enable it, and use specific HA config about peers and
cluster priority contained in the 'ha_config' section of the
main config.

The resulting keepalived cluster contains a VRRP instance for
each interface, with the exception of the management interface.

Partially-implements: blueprint appliance-ha

Change-Id: I5ababa41d65642b00f6b808197af9b2a59ebc67a

ansible/tasks/base.yml

@@ -11,6 +11,8 @@
     - ntp
     - tcpdump
     - vim
+    - keepalived
+    - conntrackd
 
 - name: latest bash (CVE-2014-6271)
   apt: name=bash state=latest install_recommends=no

astara_router/api/server.py

@@ -44,5 +44,6 @@ def main():
     # app.config.from_object('astara_router.config.Default')
     # manager.state_path = app.config['STATE_PATH']
 
-    app.run(host=manager.management_address(ensure_configuration=True),
+    addr = str(manager.ip_mgr.get_interfaces()[0].addresses[0])
+    app.run(host=addr,
             port=5000)

astara_router/drivers/arp.py

@@ -151,4 +151,9 @@ def _delete_from_arp_cache(self, ip):
         :type ip: str
         :param ip: IP address to search for in the ARP table.
         """
-        self.sudo('-d', ip)
+        try:
+            self.sudo('-d', ip)
+        except:
+            # We may be attempting to delete from ARP for interfaces which
+            # are managed by keepalived and do not yet have addresses
+            pass

astara_router/drivers/ip.py

@@ -22,7 +22,7 @@
 import netaddr
 
 from astara_router import models
-from astara_router.drivers import base
+from astara_router.drivers import base, keepalived
 from astara_router import utils
 
 LOG = logging.getLogger(__name__)
@@ -555,3 +555,54 @@ def _parse_lladdr(line):
     """
     tokens = line.split()
     return tokens[1]
+
+
+class VRRPIPManager(IPManager):
+    def __init__(self, root_helper='sudo astara-rootwrap /etc/rootwrap.conf'):
+        super(VRRPIPManager, self).__init__(root_helper)
+        self.keepalived = keepalived.KeepalivedManager(root_helper)
+        self.ensure_mapping()
+
+    def set_peers(self, peers):
+        self.keepalived.peers = peers
+
+    def set_priority(self, priority):
+        self.keepalived.set_priority(priority)
+
+    def update_interfaces(self, interfaces):
+        for interface in interfaces:
+            if interface.management:
+                # the mgt interface is not managed as a vip, but
+                # it used for keepalived mcast cluster comms
+                self.update_interface(interface)
+                self.keepalived.set_management_address(
+                    address=interface.first_v4 or interface.first_v6)
+            else:
+                self.up(interface)
+                self.keepalived.add_vrrp_instance(
+                    interface=self.generic_to_host(interface.ifname),
+                    addresses=interface.all_addresses)
+
+    def _set_default_gateway(self, gateway_ip, ifname):
+        """
+        Sets the default gateway.
+
+        :param gateway_ip: the IP address to set as the default gateway_ip
+        :type gateway_ip: netaddr.IPAddress
+        :param ifname: the interface name (in our case, of the external
+                       network)
+        :type ifname: str
+        """
+        version = 4
+        if gateway_ip.version == 6:
+            version = 6
+        self.keepalived.set_default_gateway(
+            ip_version=version, gateway_ip=gateway_ip,
+            interface=self.generic_to_host(ifname))
+
+    def update_host_routes(self, config, cache):
+        # XXX TODO
+        return
+
+    def reload(self):
+        self.keepalived.reload()

astara_router/drivers/keepalived.conf.template

@@ -0,0 +1,35 @@
+{%- for instance in vrrp_instances %}
+vrrp_instance {{ instance.name }} {
+    native_ipv6
+    state {{ instance.state }}
+    interface {{ instance.interface }}
+    virtual_router_id {{ instance.vrrp_id }}
+    priority {{ priority }}
+    garp_master_delay {{ instance.garp_master_delay }}
+    unicast_src_ip {{ instance.unicast_src_ip }}
+    unicast_peer {
+        {%- for peer in peers %}
+        {{ peer }}
+        {%- endfor %}
+    }
+    {%- if instance.vips %}
+    virtual_ipaddress {
+        {{ instance.vips[0].address }} dev {{ instance.vips[0].interface }}
+    }
+    virtual_ipaddress_excluded {
+        {%- for vip in instance.vips[1:] %}
+        {{ vip.address }} dev {{ vip.interface }}
+        {%- endfor %}
+    }
+    {%- endif %}
+
+    {%- if instance.routes %}
+    virtual_routes {
+        {%- for route in instance.routes %}
+        {{ route.destination }} via {{ route.gateway }} dev {{ instance.interface }}
+        {%- endfor %}
+    }
+    {%- endif %}
+}
+
+{%- endfor %}

astara_router/drivers/keepalived.py

@@ -0,0 +1,156 @@
+# Copyright (c) 2016 Akanda, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+from astara_router.drivers import base
+from astara_router import utils
+
+
+class KeepalivedVipAddress(object):
+    """A virtual address entry of a keepalived configuration."""
+
+    def __init__(self, address, interface):
+        self.address = address
+        self.interface = interface
+
+    def __eq__(self, other):
+        return (isinstance(other, KeepalivedVipAddress) and
+                self.address.ip == other.address.ip)
+
+
+class KeepalivedRoute(object):
+    """A virtual route entry in keepalived instance configuration"""
+    def __init__(self, destination, gateway):
+        self.destination = destination
+        self.gateway = gateway
+
+    def __eq__(self, other):
+        return (
+            isinstance(other, KeepalivedRoute) and
+            (self.destination, self.gateway) ==
+            (other.destination, other.gateway)
+        )
+
+
+class KeepalivedInstance(object):
+    def __init__(self, interface, unicast_src_ip, vrrp_id, state='BACKUP',
+                 garp_master_delay=60):
+        self.interface = interface
+        self.vrrp_id = vrrp_id
+        self.unicast_src_ip = unicast_src_ip
+        self.name = 'astara_vrrp_' + interface
+        self.state = state
+        self.garp_master_delay = 60
+        self.vips = []
+        self.routes = []
+
+    def add_vip(self, address):
+        vip = KeepalivedVipAddress(address, self.interface)
+        if vip not in self.vips:
+            self.vips.append(vip)
+
+    def add_route(self, destination, gateway):
+        route = KeepalivedRoute(destination, gateway)
+        if route not in self.routes:
+            self.routes.append(route)
+
+
+class KeepalivedManager(base.Manager):
+    CONFIG_FILE_TEMPLATE = os.path.join(
+        os.path.dirname(__file__), 'keepalived.conf.template')
+
+    # Debian defaults
+    CONFIG_FILE = '/etc/keepalived/keepalived.conf'
+    PID_FILE = '/var/run/keepalived.pid'
+
+    EXECUTABLE = 'service'
+
+    def __init__(self, root_helper='sudo astara-rootwrap /etc/rootwrap.conf'):
+        super(KeepalivedManager, self).__init__(root_helper)
+        self.instances = {}
+        self.unicast_src_ip = None
+        self.config_tmpl = utils.load_template(self.CONFIG_FILE_TEMPLATE)
+        self.peers = []
+        self.priority = 0
+        self._last_config_hash = None
+
+    def set_management_address(self, address):
+        """Specify the address used for keepalived cluster communication"""
+        self.unicast_src_ip = address
+        for instance in self.instances.values():
+            instance.unicast_src_ip = address
+
+    def _get_instance(self, interface):
+            if interface in self.instances:
+                return self.instances[interface]
+
+            vrrp_id = len(self.instances) + 1
+            self.instances[interface] = KeepalivedInstance(
+                interface, self.unicast_src_ip, vrrp_id=vrrp_id)
+            return self.instances[interface]
+
+    def _is_running(self):
+        if not os.path.isfile(self.PID_FILE):
+            return False
+
+        pid = open(self.PID_FILE).read().strip()
+        proc_cmd = os.path.join('/proc', pid, 'cmdline')
+        if not os.path.isfile(proc_cmd):
+            return False
+
+        if 'keepalived' not in open(proc_cmd).read():
+            return False
+
+        return True
+
+    def add_vrrp_instance(self, interface, addresses):
+        instance = self._get_instance(interface)
+        [instance.add_vip(addr) for addr in addresses]
+
+    def config(self):
+        return self.config_tmpl.render(
+            priority=self.priority,
+            peers=self.peers,
+            vrrp_instances=self.instances.values())
+
+    def reload(self):
+        try:
+            last_config_hash = utils.hash_file(self.CONFIG_FILE)
+        except IOError:
+            last_config_hash = None
+
+        utils.replace_file('/tmp/keepalived.conf', self.config())
+        utils.execute(
+            ['mv', '/tmp/keepalived.conf', '/etc/keepalived/keepalived.conf'],
+            self.root_helper)
+
+        if utils.hash_file(self.CONFIG_FILE) == last_config_hash:
+            return
+
+        if self._is_running():
+            self.sudo('keepalived', 'reload')
+        else:
+            self.sudo('keepalived', 'restart')
+
+    def set_default_gateway(self, ip_version, gateway_ip, interface):
+        instance = self._get_instance(interface)
+        if ip_version == 6:
+            default = 'default6'
+        else:
+            default = 'default'
+        instance.add_route(default, gateway_ip)
+
+    def set_priority(self, priority):
+        self.priority = priority

astara_router/drivers/loadbalancer/nginx.py

@@ -12,18 +12,11 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import os
 
-import jinja2
+import os
 
 from astara_router.drivers import base
-from astara_router.utils import execute
-
-
-class NginxTemplateNotFound(Exception):
-    # TODO(adam_g): These should return 50x errors and not logged
-    # exceptions.
-    pass
+from astara_router.utils import execute, load_template
 
 
 class NginxLB(base.Manager):
@@ -35,25 +28,15 @@ class NginxLB(base.Manager):
 
     def __init__(self, root_helper='sudo astara-rootwrap /etc/rootwrap.conf'):
         """
-        Initializes DHCPManager class.
+        Initializes NginxLB class.
 
         :type root_helper: str
         :param root_helper: System utility used to gain escalate privileges.
         """
         super(NginxLB, self).__init__(root_helper)
-        self._load_template()
-
-    def _load_template(self):
-        if not os.path.exists(self.CONFIG_FILE_TEMPLATE):
-            raise NginxTemplateNotFound(
-                'NGINX Config template not found @ %s' %
-                self.CONFIG_FILE_TEMPLATE
-            )
-        self.config_tmpl = jinja2.Template(
-            open(self.CONFIG_FILE_TEMPLATE).read())
+        self.config_tmpl = load_template(self.CONFIG_FILE_TEMPLATE)
 
     def _render_config_template(self, path, config):
-        self._load_template()
         with open(path, 'w') as out:
             out.write(
                 self.config_tmpl.render(loadbalancer=config)