From 08d60702f9995f9e756a16c733f6a26b9d0f5019 Mon Sep 17 00:00:00 2001 From: Dan Wendlandt Date: Fri, 22 Jul 2011 16:56:00 -0700 Subject: [PATCH 1/8] merge --- nova/network/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/network/manager.py b/nova/network/manager.py index 005fa73e78d..4706da6ea10 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -879,7 +879,6 @@ def create_networks(self, context, **kwargs): def _setup_network(self, context, network_ref): """Sets up network on this host.""" - network_ref['dhcp_server'] = self._get_dhcp_ip(context, network_ref) if not network_ref['vpn_public_address']: net = {} address = FLAGS.vpn_ip @@ -887,6 +886,7 @@ def _setup_network(self, context, network_ref): network_ref = db.network_update(context, network_ref['id'], net) else: address = network_ref['vpn_public_address'] + network_ref['dhcp_server'] = self._get_dhcp_ip(context, network_ref) self.driver.ensure_vlan_bridge(network_ref['vlan'], network_ref['bridge'], network_ref['bridge_interface'], From 873aad92944f8840e772d65eda4b3320d65a9ce7 Mon Sep 17 00:00:00 2001 From: Dan Wendlandt Date: Mon, 1 Aug 2011 18:11:15 -0700 Subject: [PATCH 2/8] initial commit of vif-plugging for network-service interfaces --- bin/nova-dhcpbridge | 24 ++--- nova/network/linux_net.py | 205 ++++++++++++++++++++++++++------------ nova/network/manager.py | 28 ++---- nova/utils.py | 9 ++ nova/virt/libvirt/vif.py | 2 +- 5 files changed, 174 insertions(+), 94 deletions(-) diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 325642d52ee..1727ebf9b15 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -48,12 +48,11 @@ flags.DECLARE('auth_driver', 'nova.auth.manager') flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('num_networks', 'nova.network.manager') flags.DECLARE('update_dhcp_on_disassociate', 'nova.network.manager') -flags.DEFINE_string('dnsmasq_interface', 'br0', 'Default Dnsmasq interface') LOG = logging.getLogger('nova.dhcpbridge') -def add_lease(mac, ip_address, _hostname, _interface): +def add_lease(mac, ip_address, _hostname): """Set the IP that was assigned by the DHCP server.""" if FLAGS.fake_rabbit: LOG.debug(_("leasing ip")) @@ -67,13 +66,13 @@ def add_lease(mac, ip_address, _hostname, _interface): "args": {"address": ip_address}}) -def old_lease(mac, ip_address, hostname, interface): +def old_lease(mac, ip_address, hostname): """Update just as add lease.""" LOG.debug(_("Adopted old lease or got a change of mac/hostname")) - add_lease(mac, ip_address, hostname, interface) + add_lease(mac, ip_address, hostname) -def del_lease(mac, ip_address, _hostname, _interface): +def del_lease(mac, ip_address, _hostname): """Called when a lease expires.""" if FLAGS.fake_rabbit: LOG.debug(_("releasing ip")) @@ -87,10 +86,10 @@ def del_lease(mac, ip_address, _hostname, _interface): "args": {"address": ip_address}}) -def init_leases(interface): - """Get the list of hosts for an interface.""" +def init_leases(network_id): + """Get the list of hosts for a network.""" ctxt = context.get_admin_context() - network_ref = db.network_get_by_bridge(ctxt, interface) + network_ref = db.network_get(ctxt, network_id) return linux_net.get_dhcp_leases(ctxt, network_ref) @@ -101,7 +100,8 @@ def main(): argv = FLAGS(sys.argv) logging.setup() # check ENV first so we don't break any older deploys - interface = os.environ.get('DNSMASQ_INTERFACE', FLAGS.dnsmasq_interface) + network_id = int(os.environ.get('NETWORK_ID')) + if int(os.environ.get('TESTING', '0')): from nova.tests import fake_flags @@ -117,11 +117,11 @@ def main(): ip = argv[3] hostname = argv[4] msg = _("Called %(action)s for mac %(mac)s with ip %(ip)s and" - " hostname %(hostname)s on interface %(interface)s") % locals() + " hostname %(hostname)s on network %(network_id)s") % locals() LOG.debug(msg) - globals()[action + '_lease'](mac, ip, hostname, interface) + globals()[action + '_lease'](mac, ip, hostname) else: - print init_leases(interface) + print init_leases(network_id) if __name__ == "__main__": main() diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 8ace0788468..ee0ef0b85d5 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -63,6 +63,11 @@ def _bin_file(script): 'dmz range that should be accepted') flags.DEFINE_string('dnsmasq_config_file', "", 'Override the default dnsmasq settings with this file') +flags.DEFINE_string('linuxnet_interface_driver', + 'nova.network.linux_net.LinuxBridgeInterfaceDriver', + 'Driver used to create ethernet devices.') +flags.DEFINE_string('linuxnet_ovs_integration_bridge', + 'br-int', 'Name of Open vSwitch bridge used with linuxnet') binary_name = os.path.basename(inspect.stack()[-1][1]) @@ -413,7 +418,7 @@ def ensure_metadata_ip(): 'scope', 'link', 'dev', 'lo', check_exit_code=False) -def ensure_vlan_forward(public_ip, port, private_ip): +def ensure_vpn_forward(public_ip, port, private_ip): """Sets up forwarding rules for vlan.""" iptables_manager.ipv4['filter'].add_rule('FORWARD', '-d %s -p udp ' @@ -491,32 +496,11 @@ def ensure_bridge(bridge, interface, net_attrs=None): _execute('sudo', 'brctl', 'setfd', bridge, 0) # _execute('sudo brctl setageing %s 10' % bridge) _execute('sudo', 'brctl', 'stp', bridge, 'off') - _execute('sudo', 'ip', 'link', 'set', bridge, 'up') - if net_attrs: - # NOTE(vish): The ip for dnsmasq has to be the first address on the - # bridge for it to respond to reqests properly - suffix = net_attrs['cidr'].rpartition('/')[2] - out, err = _execute('sudo', 'ip', 'addr', 'add', - '%s/%s' % - (net_attrs['dhcp_server'], suffix), - 'brd', - net_attrs['broadcast'], - 'dev', - bridge, - check_exit_code=False) - if err and err != 'RTNETLINK answers: File exists\n': - raise exception.Error('Failed to add ip: %s' % err) - if(FLAGS.use_ipv6): - _execute('sudo', 'ip', '-f', 'inet6', 'addr', - 'change', net_attrs['cidr_v6'], - 'dev', bridge) - # NOTE(vish): If the public interface is the same as the - # bridge, then the bridge has to be in promiscuous - # to forward packets properly. - if(FLAGS.public_interface == bridge): - _execute('sudo', 'ip', 'link', 'set', - 'dev', bridge, 'promisc', 'on') + if interface: + out, err = _execute('sudo', 'brctl', 'addif', bridge, interface, + check_exit_code=False) + # NOTE(vish): This will break if there is already an ip on the # interface, so we move any ips to the bridge gateway = None @@ -526,9 +510,9 @@ def ensure_bridge(bridge, interface, net_attrs=None): if fields and fields[0] == '0.0.0.0' and fields[-1] == interface: gateway = fields[1] _execute('sudo', 'route', 'del', 'default', 'gw', gateway, - 'dev', interface, check_exit_code=False) + 'dev', interface, check_exit_code=False) out, err = _execute('sudo', 'ip', 'addr', 'show', 'dev', interface, - 'scope', 'global') + 'scope', 'global') for line in out.split('\n'): fields = line.split() if fields and fields[0] == 'inet': @@ -537,8 +521,6 @@ def ensure_bridge(bridge, interface, net_attrs=None): _execute(*_ip_bridge_cmd('add', params, bridge)) if gateway: _execute('sudo', 'route', 'add', 'default', 'gw', gateway) - out, err = _execute('sudo', 'brctl', 'addif', bridge, interface, - check_exit_code=False) if (err and err != "device %s is already a member of a bridge; can't " "enslave it to bridge %s.\n" % (interface, bridge)): @@ -552,6 +534,36 @@ def ensure_bridge(bridge, interface, net_attrs=None): bridge) +def initialize_gateway_device(dev, network_ref): + if not network_ref: + return + + # NOTE(vish): The ip for dnsmasq has to be the first address on the + # bridge for it to respond to reqests properly + suffix = network_ref['cidr'].rpartition('/')[2] + out, err = _execute('sudo', 'ip', 'addr', 'add', + '%s/%s' % + (network_ref['dhcp_server'], suffix), + 'brd', + network_ref['broadcast'], + 'dev', + dev, + check_exit_code=False) + if err and err != 'RTNETLINK answers: File exists\n': + raise exception.Error('Failed to add ip: %s' % err) + if(FLAGS.use_ipv6): + _execute('sudo', 'ip', '-f', 'inet6', 'addr', + 'change', network_ref['cidr_v6'], + 'dev', dev) + # NOTE(vish): If the public interface is the same as the + # bridge, then the bridge has to be in promiscuous + # to forward packets properly. + if(FLAGS.public_interface == dev): + _execute('sudo', 'ip', 'link', 'set', + 'dev', dev, 'promisc', 'on') + _execute('sudo', 'ip', 'link', 'set', dev, 'up') + + def get_dhcp_leases(context, network_ref): """Return a network's hosts config in dnsmasq leasefile format.""" hosts = [] @@ -580,21 +592,21 @@ def get_dhcp_hosts(context, network_ref): # configuration options (like dchp-range, vlan, ...) # aren't reloaded. @utils.synchronized('dnsmasq_start') -def update_dhcp(context, network_ref): +def update_dhcp(context, dev, network_ref): """(Re)starts a dnsmasq server for a given network. If a dnsmasq instance is already running then send a HUP signal causing it to reload, otherwise spawn a new instance. """ - conffile = _dhcp_file(network_ref['bridge'], 'conf') + conffile = _dhcp_file(dev, 'conf') with open(conffile, 'w') as f: f.write(get_dhcp_hosts(context, network_ref)) # Make sure dnsmasq can actually read it (it setuid()s to "nobody") os.chmod(conffile, 0644) - pid = _dnsmasq_pid_for(network_ref['bridge']) + pid = _dnsmasq_pid_for(dev) # if dnsmasq is already running, then tell it to reload if pid: @@ -609,16 +621,16 @@ def update_dhcp(context, network_ref): else: LOG.debug(_('Pid %d is stale, relaunching dnsmasq'), pid) - # FLAGFILE and DNSMASQ_INTERFACE in env + # FLAGFILE and NETWORK_ID in env env = {'FLAGFILE': FLAGS.dhcpbridge_flagfile, - 'DNSMASQ_INTERFACE': network_ref['bridge']} - command = _dnsmasq_cmd(network_ref) + 'NETWORK_ID': str(network_ref['id'])} + command = _dnsmasq_cmd(dev, network_ref) _execute(*command, addl_env=env) @utils.synchronized('radvd_start') -def update_ra(context, network_ref): - conffile = _ra_file(network_ref['bridge'], 'conf') +def update_ra(context, dev, network_ref): + conffile = _ra_file(dev, 'conf') with open(conffile, 'w') as f: conf_str = """ interface %s @@ -632,13 +644,13 @@ def update_ra(context, network_ref): AdvAutonomous on; }; }; -""" % (network_ref['bridge'], network_ref['cidr_v6']) +""" % (dev, network_ref['cidr_v6']) f.write(conf_str) # Make sure radvd can actually read it (it setuid()s to "nobody") os.chmod(conffile, 0644) - pid = _ra_pid_for(network_ref['bridge']) + pid = _ra_pid_for(dev) # if radvd is already running, then tell it to reload if pid: @@ -651,7 +663,7 @@ def update_ra(context, network_ref): LOG.debug(_('killing radvd threw %s'), exc) else: LOG.debug(_('Pid %d is stale, relaunching radvd'), pid) - command = _ra_cmd(network_ref) + command = _ra_cmd(dev) _execute(*command) @@ -696,20 +708,20 @@ def _device_exists(device): return not err -def _dnsmasq_cmd(net): +def _dnsmasq_cmd(dev, net): """Builds dnsmasq command.""" cmd = ['sudo', '-E', 'dnsmasq', '--strict-order', '--bind-interfaces', - '--interface=%s' % net['bridge'], + '--interface=%s' % dev, '--conf-file=%s' % FLAGS.dnsmasq_config_file, '--domain=%s' % FLAGS.dhcp_domain, - '--pid-file=%s' % _dhcp_file(net['bridge'], 'pid'), + '--pid-file=%s' % _dhcp_file(dev, 'pid'), '--listen-address=%s' % net['dhcp_server'], '--except-interface=lo', '--dhcp-range=%s,static,120s' % net['dhcp_start'], '--dhcp-lease-max=%s' % len(netaddr.IPNetwork(net['cidr'])), - '--dhcp-hostsfile=%s' % _dhcp_file(net['bridge'], 'conf'), + '--dhcp-hostsfile=%s' % _dhcp_file(dev, 'conf'), '--dhcp-script=%s' % FLAGS.dhcpbridge, '--leasefile-ro'] if FLAGS.dns_server: @@ -717,18 +729,18 @@ def _dnsmasq_cmd(net): return cmd -def _ra_cmd(net): +def _ra_cmd(dev): """Builds radvd command.""" cmd = ['sudo', '-E', 'radvd', # '-u', 'nobody', - '-C', '%s' % _ra_file(net['bridge'], 'conf'), - '-p', '%s' % _ra_file(net['bridge'], 'pid')] + '-C', '%s' % _ra_file(dev, 'conf'), + '-p', '%s' % _ra_file(dev, 'pid')] return cmd -def _stop_dnsmasq(network): +def _stop_dnsmasq(dev): """Stops the dnsmasq instance for a given network.""" - pid = _dnsmasq_pid_for(network) + pid = _dnsmasq_pid_for(dev) if pid: try: @@ -737,49 +749,49 @@ def _stop_dnsmasq(network): LOG.debug(_('Killing dnsmasq threw %s'), exc) -def _dhcp_file(bridge, kind): - """Return path to a pid, leases or conf file for a bridge.""" +def _dhcp_file(dev, kind): + """Return path to a pid, leases or conf file for a bridge/device.""" if not os.path.exists(FLAGS.networks_path): os.makedirs(FLAGS.networks_path) return os.path.abspath('%s/nova-%s.%s' % (FLAGS.networks_path, - bridge, + dev, kind)) -def _ra_file(bridge, kind): - """Return path to a pid or conf file for a bridge.""" +def _ra_file(dev, kind): + """Return path to a pid or conf file for a bridge/device.""" if not os.path.exists(FLAGS.networks_path): os.makedirs(FLAGS.networks_path) return os.path.abspath('%s/nova-ra-%s.%s' % (FLAGS.networks_path, - bridge, + dev, kind)) -def _dnsmasq_pid_for(bridge): - """Returns the pid for prior dnsmasq instance for a bridge. +def _dnsmasq_pid_for(dev): + """Returns the pid for prior dnsmasq instance for a bridge/device. Returns None if no pid file exists. If machine has rebooted pid might be incorrect (caller should check). """ - pid_file = _dhcp_file(bridge, 'pid') + pid_file = _dhcp_file(dev, 'pid') if os.path.exists(pid_file): with open(pid_file, 'r') as f: return int(f.read()) -def _ra_pid_for(bridge): - """Returns the pid for prior radvd instance for a bridge. +def _ra_pid_for(dev): + """Returns the pid for prior radvd instance for a bridge/device. Returns None if no pid file exists. If machine has rebooted pid might be incorrect (caller should check). """ - pid_file = _ra_file(bridge, 'pid') + pid_file = _ra_file(dev, 'pid') if os.path.exists(pid_file): with open(pid_file, 'r') as f: @@ -795,3 +807,72 @@ def _ip_bridge_cmd(action, params, device): iptables_manager = IptablesManager() + +# Similar to compute virt layers, the Linux network node +# code uses a flexible driver model to support different ways +# of creating ethernet interfaces and attaching them to the network. +# In the case of a network host, these interfaces +# act as gateway/dhcp/vpn/etc. endpoints not VM interfaces. + + +def plug(network): + return interface_driver.plug(network) + + +def unplug(network): + return interface_driver.unplug(network) + + +class LinuxNetInterfaceDriver(object): + """Abstract class that defines generic network host API""" + """ for for all Linux interface drivers.""" + + def plug(self, network): + """Create Linux device, return device name""" + raise NotImplementedError() + + def unplug(self, network): + """Destory Linux device, return device name""" + raise NotImplementedError() + + +# plugs interfaces using Linux Bridge +class LinuxBridgeInterfaceDriver(LinuxNetInterfaceDriver): + + def plug(self, network): + self.driver.ensure_vlan_bridge(network['vlan'], + network['bridge'], + network['bridge_interface'], + network) + return network['bridge'] + + def unplug(self, network): + return network['bridge'] + + +# plugs interfaces using Open vSwitch +class LinuxOVSInterfaceDriver(LinuxNetInterfaceDriver): + + def plug(self, network): + dev = "gw-" + str(network['id']) + if not _device_exists(dev): + bridge = FLAGS.linuxnet_ovs_integration_bridge + mac_addr = utils.generate_mac_address() + _execute('sudo', 'ovs-vsctl', + '--', '--may-exist', 'add-port', bridge, dev, + '--', 'set', 'Interface', dev, "type=internal", + '--', 'set', 'Interface', dev, + "external-ids:iface-id=nova-%s" % dev, + '--', 'set', 'Interface', dev, + "external-ids:iface-status=active", + '--', 'set', 'Interface', dev, + "external-ids:attached-mac=%s" % mac_addr) + _execute('sudo', 'ip', 'link', 'set', + dev, "address", mac_addr) + return dev + + def unplug(self, network): + dev = "gw-" + str(network['id']) + return dev + +interface_driver = utils.import_object(FLAGS.linuxnet_interface_driver) diff --git a/nova/network/manager.py b/nova/network/manager.py index 4706da6ea10..0fc55f44110 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -504,7 +504,7 @@ def ip6_dict(): def _allocate_mac_addresses(self, context, instance_id, networks): """Generates mac addresses and creates vif rows in db for them.""" for network in networks: - vif = {'address': self.generate_mac_address(), + vif = {'address': utils.generate_mac_address(), 'instance_id': instance_id, 'network_id': network['id']} # try FLAG times to create a vif record with a unique mac_address @@ -513,20 +513,12 @@ def _allocate_mac_addresses(self, context, instance_id, networks): self.db.virtual_interface_create(context, vif) break except exception.VirtualInterfaceCreateException: - vif['address'] = self.generate_mac_address() + vif['address'] = utils.generate_mac_address() else: self.db.virtual_interface_delete_by_instance(context, instance_id) raise exception.VirtualInterfaceMacAddressException() - def generate_mac_address(self): - """Generate a mac address for a vif on an instance.""" - mac = [0x02, 0x16, 0x3e, - random.randint(0x00, 0x7f), - random.randint(0x00, 0xff), - random.randint(0x00, 0xff)] - return ':'.join(map(lambda x: "%02x" % x, mac)) - def add_fixed_ip_to_instance(self, context, instance_id, host, network_id): """Adds a fixed ip to an instance from specified network.""" networks = [self.db.network_get(context, network_id)] @@ -887,23 +879,21 @@ def _setup_network(self, context, network_ref): else: address = network_ref['vpn_public_address'] network_ref['dhcp_server'] = self._get_dhcp_ip(context, network_ref) - self.driver.ensure_vlan_bridge(network_ref['vlan'], - network_ref['bridge'], - network_ref['bridge_interface'], - network_ref) + dev = self.driver.plug(network_ref) + self.driver.initialize_gateway_device(dev, network_ref) # NOTE(vish): only ensure this forward if the address hasn't been set # manually. if address == FLAGS.vpn_ip and hasattr(self.driver, - "ensure_vlan_forward"): - self.driver.ensure_vlan_forward(FLAGS.vpn_ip, + "ensure_vpn_forward"): + self.driver.ensure_vpn_forward(FLAGS.vpn_ip, network_ref['vpn_public_port'], network_ref['vpn_private_address']) if not FLAGS.fake_network: - self.driver.update_dhcp(context, network_ref) + self.driver.update_dhcp(context, dev, network_ref) if(FLAGS.use_ipv6): - self.driver.update_ra(context, network_ref) - gateway = utils.get_my_linklocal(network_ref['bridge']) + self.driver.update_ra(context, dev, network_ref) + gateway = utils.get_my_linklocal(dev) self.db.network_update(context, network_ref['id'], {'gateway_v6': gateway}) diff --git a/nova/utils.py b/nova/utils.py index 8784a227dd8..d56fa614cc1 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -320,6 +320,15 @@ def get_my_linklocal(interface): " :%(ex)s") % locals()) +def generate_mac_address(): + """Generate an Ethernet MAC address.""" + mac = [0x02, 0x16, 0x3e, + random.randint(0x00, 0x7f), + random.randint(0x00, 0xff), + random.randint(0x00, 0xff)] + return ':'.join(map(lambda x: "%02x" % x, mac)) + + def utcnow(): """Overridable version of utils.utcnow.""" if utcnow.override_time: diff --git a/nova/virt/libvirt/vif.py b/nova/virt/libvirt/vif.py index aa566cc7262..432a585ad06 100644 --- a/nova/virt/libvirt/vif.py +++ b/nova/virt/libvirt/vif.py @@ -99,8 +99,8 @@ class LibvirtOpenVswitchDriver(VIFDriver): def plug(self, instance, network, mapping): vif_id = str(instance['id']) + "-" + str(network['id']) dev = "tap-%s" % vif_id + iface_id = "nova-" + vif_id if not linux_net._device_exists(dev): - iface_id = "nova-" + vif_id utils.execute('sudo', 'ip', 'tuntap', 'add', dev, 'mode', 'tap') utils.execute('sudo', 'ip', 'link', 'set', dev, 'up') utils.execute('sudo', 'ovs-vsctl', '--', '--may-exist', 'add-port', From 857f4453efaca98ced3e07d55ee6f0188713e60e Mon Sep 17 00:00:00 2001 From: Dan Wendlandt Date: Mon, 1 Aug 2011 18:30:59 -0700 Subject: [PATCH 3/8] fix LinuxBridgeInterfaceDriver --- nova/network/linux_net.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index ee0ef0b85d5..ce1f53ab9e2 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -840,10 +840,10 @@ def unplug(self, network): class LinuxBridgeInterfaceDriver(LinuxNetInterfaceDriver): def plug(self, network): - self.driver.ensure_vlan_bridge(network['vlan'], - network['bridge'], - network['bridge_interface'], - network) + ensure_vlan_bridge(network['vlan'], + network['bridge'], + network['bridge_interface'], + network) return network['bridge'] def unplug(self, network): From dfc9c9c2b5e92e599bdeae4c03d3761215a0deca Mon Sep 17 00:00:00 2001 From: Dan Wendlandt Date: Thu, 4 Aug 2011 12:36:11 -0700 Subject: [PATCH 4/8] modify _setup_network for flatDHCP as well --- nova/network/linux_net.py | 12 ++++++++---- nova/network/manager.py | 13 +++++++------ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index ce1f53ab9e2..7012cceebea 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -805,9 +805,6 @@ def _ip_bridge_cmd(action, params, device): cmd.extend(['dev', device]) return cmd - -iptables_manager = IptablesManager() - # Similar to compute virt layers, the Linux network node # code uses a flexible driver model to support different ways # of creating ethernet interfaces and attaching them to the network. @@ -840,10 +837,16 @@ def unplug(self, network): class LinuxBridgeInterfaceDriver(LinuxNetInterfaceDriver): def plug(self, network): - ensure_vlan_bridge(network['vlan'], + if network.get('vlan', None) is not None: + ensure_vlan_bridge(network['vlan'], network['bridge'], network['bridge_interface'], network) + else: + ensure_bridge(network['bridge'], + network['bridge_interface'], + network) + return network['bridge'] def unplug(self, network): @@ -875,4 +878,5 @@ def unplug(self, network): dev = "gw-" + str(network['id']) return dev +iptables_manager = IptablesManager() interface_driver = utils.import_object(FLAGS.linuxnet_interface_driver) diff --git a/nova/network/manager.py b/nova/network/manager.py index bedbe2c2134..e7c84d47862 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -785,14 +785,15 @@ def init_host(self): def _setup_network(self, context, network_ref): """Sets up network on this host.""" network_ref['dhcp_server'] = self._get_dhcp_ip(context, network_ref) - self.driver.ensure_bridge(network_ref['bridge'], - network_ref['bridge_interface'], - network_ref) + + dev = self.driver.plug(network_ref) + self.driver.initialize_gateway_device(dev, network_ref) + if not FLAGS.fake_network: - self.driver.update_dhcp(context, network_ref) + self.driver.update_dhcp(context, dev, network_ref) if(FLAGS.use_ipv6): - self.driver.update_ra(context, network_ref) - gateway = utils.get_my_linklocal(network_ref['bridge']) + self.driver.update_ra(context, dev, network_ref) + gateway = utils.get_my_linklocal(dev) self.db.network_update(context, network_ref['id'], {'gateway_v6': gateway}) From 18f09f165b5dca5f11253b143045b2ff7327532d Mon Sep 17 00:00:00 2001 From: Dan Wendlandt Date: Thu, 4 Aug 2011 16:20:38 -0700 Subject: [PATCH 5/8] move ensure_vlan_bridge,ensure_bridge,ensure_vlan to the bridge/vlan specific vif-plugging driver --- nova/network/linux_net.py | 166 ++++++++++++++++++++------------------ nova/virt/libvirt/vif.py | 6 +- 2 files changed, 90 insertions(+), 82 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 7012cceebea..17b63a849b8 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -456,84 +456,6 @@ def floating_forward_rules(floating_ip, fixed_ip): '-s %s -j SNAT --to %s' % (fixed_ip, floating_ip))] -def ensure_vlan_bridge(vlan_num, bridge, bridge_interface, net_attrs=None): - """Create a vlan and bridge unless they already exist.""" - interface = ensure_vlan(vlan_num, bridge_interface) - ensure_bridge(bridge, interface, net_attrs) - return interface - - -@utils.synchronized('ensure_vlan', external=True) -def ensure_vlan(vlan_num, bridge_interface): - """Create a vlan unless it already exists.""" - interface = 'vlan%s' % vlan_num - if not _device_exists(interface): - LOG.debug(_('Starting VLAN inteface %s'), interface) - _execute('sudo', 'vconfig', 'set_name_type', 'VLAN_PLUS_VID_NO_PAD') - _execute('sudo', 'vconfig', 'add', bridge_interface, vlan_num) - _execute('sudo', 'ip', 'link', 'set', interface, 'up') - return interface - - -@utils.synchronized('ensure_bridge', external=True) -def ensure_bridge(bridge, interface, net_attrs=None): - """Create a bridge unless it already exists. - - :param interface: the interface to create the bridge on. - :param net_attrs: dictionary with attributes used to create the bridge. - - If net_attrs is set, it will add the net_attrs['gateway'] to the bridge - using net_attrs['broadcast'] and net_attrs['cidr']. It will also add - the ip_v6 address specified in net_attrs['cidr_v6'] if use_ipv6 is set. - - The code will attempt to move any ips that already exist on the interface - onto the bridge and reset the default gateway if necessary. - - """ - if not _device_exists(bridge): - LOG.debug(_('Starting Bridge interface for %s'), interface) - _execute('sudo', 'brctl', 'addbr', bridge) - _execute('sudo', 'brctl', 'setfd', bridge, 0) - # _execute('sudo brctl setageing %s 10' % bridge) - _execute('sudo', 'brctl', 'stp', bridge, 'off') - - if interface: - out, err = _execute('sudo', 'brctl', 'addif', bridge, interface, - check_exit_code=False) - - # NOTE(vish): This will break if there is already an ip on the - # interface, so we move any ips to the bridge - gateway = None - out, err = _execute('sudo', 'route', '-n') - for line in out.split('\n'): - fields = line.split() - if fields and fields[0] == '0.0.0.0' and fields[-1] == interface: - gateway = fields[1] - _execute('sudo', 'route', 'del', 'default', 'gw', gateway, - 'dev', interface, check_exit_code=False) - out, err = _execute('sudo', 'ip', 'addr', 'show', 'dev', interface, - 'scope', 'global') - for line in out.split('\n'): - fields = line.split() - if fields and fields[0] == 'inet': - params = fields[1:-1] - _execute(*_ip_bridge_cmd('del', params, fields[-1])) - _execute(*_ip_bridge_cmd('add', params, bridge)) - if gateway: - _execute('sudo', 'route', 'add', 'default', 'gw', gateway) - - if (err and err != "device %s is already a member of a bridge; can't " - "enslave it to bridge %s.\n" % (interface, bridge)): - raise exception.Error('Failed to add interface: %s' % err) - - iptables_manager.ipv4['filter'].add_rule('FORWARD', - '--in-interface %s -j ACCEPT' % \ - bridge) - iptables_manager.ipv4['filter'].add_rule('FORWARD', - '--out-interface %s -j ACCEPT' % \ - bridge) - - def initialize_gateway_device(dev, network_ref): if not network_ref: return @@ -838,12 +760,14 @@ class LinuxBridgeInterfaceDriver(LinuxNetInterfaceDriver): def plug(self, network): if network.get('vlan', None) is not None: - ensure_vlan_bridge(network['vlan'], + LinuxBridgeInterfaceDriver.ensure_vlan_bridge( + network['vlan'], network['bridge'], network['bridge_interface'], network) else: - ensure_bridge(network['bridge'], + LinuxBridgeInterfaceDriver.ensure_bridge( + network['bridge'], network['bridge_interface'], network) @@ -852,6 +776,88 @@ def plug(self, network): def unplug(self, network): return network['bridge'] + @classmethod + def ensure_vlan_bridge(_self, vlan_num, bridge, bridge_interface, + net_attrs=None): + """Create a vlan and bridge unless they already exist.""" + interface = LinuxBridgeInterfaceDriver.ensure_vlan(vlan_num, + bridge_interface) + LinuxBridgeInterfaceDriver.ensure_bridge(bridge, interface, net_attrs) + return interface + + @classmethod + @utils.synchronized('ensure_vlan', external=True) + def ensure_vlan(_self, vlan_num, bridge_interface): + """Create a vlan unless it already exists.""" + interface = 'vlan%s' % vlan_num + if not _device_exists(interface): + LOG.debug(_('Starting VLAN inteface %s'), interface) + _execute('sudo', 'vconfig', 'set_name_type', + 'VLAN_PLUS_VID_NO_PAD') + _execute('sudo', 'vconfig', 'add', bridge_interface, vlan_num) + _execute('sudo', 'ip', 'link', 'set', interface, 'up') + return interface + + @classmethod + @utils.synchronized('ensure_bridge', external=True) + def ensure_bridge(_self, bridge, interface, net_attrs=None): + """Create a bridge unless it already exists. + + :param interface: the interface to create the bridge on. + :param net_attrs: dictionary with attributes used to create bridge. + + If net_attrs is set, it will add the net_attrs['gateway'] to the bridge + using net_attrs['broadcast'] and net_attrs['cidr']. It will also add + the ip_v6 address specified in net_attrs['cidr_v6'] if use_ipv6 is set. + + The code will attempt to move any ips that already exist on the + interface onto the bridge and reset the default gateway if necessary. + + """ + if not _device_exists(bridge): + LOG.debug(_('Starting Bridge interface for %s'), interface) + _execute('sudo', 'brctl', 'addbr', bridge) + _execute('sudo', 'brctl', 'setfd', bridge, 0) + # _execute('sudo brctl setageing %s 10' % bridge) + _execute('sudo', 'brctl', 'stp', bridge, 'off') + + if interface: + out, err = _execute('sudo', 'brctl', 'addif', bridge, interface, + check_exit_code=False) + + # NOTE(vish): This will break if there is already an ip on the + # interface, so we move any ips to the bridge + gateway = None + out, err = _execute('sudo', 'route', '-n') + for line in out.split('\n'): + fields = line.split() + if fields and fields[0] == '0.0.0.0' and \ + fields[-1] == interface: + gateway = fields[1] + _execute('sudo', 'route', 'del', 'default', 'gw', gateway, + 'dev', interface, check_exit_code=False) + out, err = _execute('sudo', 'ip', 'addr', 'show', 'dev', interface, + 'scope', 'global') + for line in out.split('\n'): + fields = line.split() + if fields and fields[0] == 'inet': + params = fields[1:-1] + _execute(*_ip_bridge_cmd('del', params, fields[-1])) + _execute(*_ip_bridge_cmd('add', params, bridge)) + if gateway: + _execute('sudo', 'route', 'add', 'default', 'gw', gateway) + + if (err and err != "device %s is already a member of a bridge;" + "can't enslave it to bridge %s.\n" % (interface, bridge)): + raise exception.Error('Failed to add interface: %s' % err) + + iptables_manager.ipv4['filter'].add_rule('FORWARD', + '--in-interface %s -j ACCEPT' % \ + bridge) + iptables_manager.ipv4['filter'].add_rule('FORWARD', + '--out-interface %s -j ACCEPT' % \ + bridge) + # plugs interfaces using Open vSwitch class LinuxOVSInterfaceDriver(LinuxNetInterfaceDriver): diff --git a/nova/virt/libvirt/vif.py b/nova/virt/libvirt/vif.py index eef582fac6d..f42eeec98e5 100644 --- a/nova/virt/libvirt/vif.py +++ b/nova/virt/libvirt/vif.py @@ -79,12 +79,14 @@ def plug(self, instance, network, mapping): LOG.debug(_('Ensuring vlan %(vlan)s and bridge %(bridge)s'), {'vlan': network['vlan'], 'bridge': network['bridge']}) - linux_net.ensure_vlan_bridge(network['vlan'], + linux_net.LinuxBridgeInterfaceDriver.ensure_vlan_bridge( + network['vlan'], network['bridge'], network['bridge_interface']) else: LOG.debug(_("Ensuring bridge %s"), network['bridge']) - linux_net.ensure_bridge(network['bridge'], + linux_net.LinuxBridgeInterfaceDriver.ensure_bridge( + network['bridge'], network['bridge_interface']) return self._get_configurations(network, mapping) From 1f3eb69ec547737447e91116881a8cb85157d65c Mon Sep 17 00:00:00 2001 From: Dan Wendlandt Date: Fri, 12 Aug 2011 18:16:07 -0700 Subject: [PATCH 6/8] fix issue introduced in merge --- nova/network/linux_net.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index be426939266..904014716fa 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -463,7 +463,7 @@ def initialize_gateway_device(dev, network_ref): # NOTE(vish): The ip for dnsmasq has to be the first address on the # bridge for it to respond to reqests properly - suffix = net_attrs['cidr'].rpartition('/')[2] + suffix = network_ref['cidr'].rpartition('/')[2] out, err = _execute('ip', 'addr', 'add', '%s/%s' % (network_ref['dhcp_server'], suffix), @@ -477,7 +477,7 @@ def initialize_gateway_device(dev, network_ref): raise exception.Error('Failed to add ip: %s' % err) if(FLAGS.use_ipv6): _execute('ip', '-f', 'inet6', 'addr', - 'change', net_attrs['cidr_v6'], + 'change', network_ref['cidr_v6'], 'dev', dev, run_as_root=True) # NOTE(vish): If the public interface is the same as the # bridge, then the bridge has to be in promiscuous From e0e49dd7340bbb26c82f18a94a6582a5684925fa Mon Sep 17 00:00:00 2001 From: Dan Wendlandt Date: Sat, 13 Aug 2011 18:58:29 -0700 Subject: [PATCH 7/8] have NetworkManager generate MAC address and pass it to the driver for plugging. Sets the stage for being able to do duplicate checks on those MACs as well. --- nova/network/linux_net.py | 68 +++++++++++++++++++++++---------------- nova/network/manager.py | 18 ++++++++--- nova/utils.py | 9 ------ 3 files changed, 54 insertions(+), 41 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 904014716fa..248f1ce5acd 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -458,13 +458,13 @@ def floating_forward_rules(floating_ip, fixed_ip): def initialize_gateway_device(dev, network_ref): - if not network_ref: - return + if not network_ref: + return - # NOTE(vish): The ip for dnsmasq has to be the first address on the - # bridge for it to respond to reqests properly - suffix = network_ref['cidr'].rpartition('/')[2] - out, err = _execute('ip', 'addr', 'add', + # NOTE(vish): The ip for dnsmasq has to be the first address on the + # bridge for it to respond to reqests properly + suffix = network_ref['cidr'].rpartition('/')[2] + out, err = _execute('ip', 'addr', 'add', '%s/%s' % (network_ref['dhcp_server'], suffix), 'brd', @@ -473,19 +473,18 @@ def initialize_gateway_device(dev, network_ref): dev, run_as_root=True, check_exit_code=False) - if err and err != 'RTNETLINK answers: File exists\n': - raise exception.Error('Failed to add ip: %s' % err) - if(FLAGS.use_ipv6): - _execute('ip', '-f', 'inet6', 'addr', + if err and err != 'RTNETLINK answers: File exists\n': + raise exception.Error('Failed to add ip: %s' % err) + if(FLAGS.use_ipv6): + _execute('ip', '-f', 'inet6', 'addr', 'change', network_ref['cidr_v6'], 'dev', dev, run_as_root=True) - # NOTE(vish): If the public interface is the same as the - # bridge, then the bridge has to be in promiscuous - # to forward packets properly. - if(FLAGS.public_interface == dev): - _execute('ip', 'link', 'set', + # NOTE(vish): If the public interface is the same as the + # bridge, then the bridge has to be in promiscuous + # to forward packets properly. + if(FLAGS.public_interface == dev): + _execute('ip', 'link', 'set', 'dev', dev, 'promisc', 'on', run_as_root=True) - _execute('ip', 'link', 'set', dev, 'up', run_as_root=True) def get_dhcp_leases(context, network_ref): @@ -718,6 +717,7 @@ def _ip_bridge_cmd(action, params, device): cmd.extend(['dev', device]) return cmd + # Similar to compute virt layers, the Linux network node # code uses a flexible driver model to support different ways # of creating ethernet interfaces and attaching them to the network. @@ -725,8 +725,8 @@ def _ip_bridge_cmd(action, params, device): # act as gateway/dhcp/vpn/etc. endpoints not VM interfaces. -def plug(network): - return interface_driver.plug(network) +def plug(network, mac_address): + return interface_driver.plug(network, mac_address) def unplug(network): @@ -737,7 +737,7 @@ class LinuxNetInterfaceDriver(object): """Abstract class that defines generic network host API""" """ for for all Linux interface drivers.""" - def plug(self, network): + def plug(self, network, mac_address): """Create Linux device, return device name""" raise NotImplementedError() @@ -749,13 +749,14 @@ def unplug(self, network): # plugs interfaces using Linux Bridge class LinuxBridgeInterfaceDriver(LinuxNetInterfaceDriver): - def plug(self, network): + def plug(self, network, mac_address): if network.get('vlan', None) is not None: LinuxBridgeInterfaceDriver.ensure_vlan_bridge( network['vlan'], network['bridge'], network['bridge_interface'], - network) + network, + mac_address) else: LinuxBridgeInterfaceDriver.ensure_bridge( network['bridge'], @@ -769,16 +770,16 @@ def unplug(self, network): @classmethod def ensure_vlan_bridge(_self, vlan_num, bridge, bridge_interface, - net_attrs=None): + net_attrs=None, mac_address=None): """Create a vlan and bridge unless they already exist.""" interface = LinuxBridgeInterfaceDriver.ensure_vlan(vlan_num, - bridge_interface) + bridge_interface, mac_address) LinuxBridgeInterfaceDriver.ensure_bridge(bridge, interface, net_attrs) return interface @classmethod @utils.synchronized('ensure_vlan', external=True) - def ensure_vlan(_self, vlan_num, bridge_interface): + def ensure_vlan(_self, vlan_num, bridge_interface, mac_address=None): """Create a vlan unless it already exists.""" interface = 'vlan%s' % vlan_num if not _device_exists(interface): @@ -787,6 +788,11 @@ def ensure_vlan(_self, vlan_num, bridge_interface): 'VLAN_PLUS_VID_NO_PAD', run_as_root=True) _execute('vconfig', 'add', bridge_interface, vlan_num, run_as_root=True) + # (danwent) the bridge will inherit this address, so we want to + # make sure it is the value set from the NetworkManager + if mac_address: + _execute('ip', 'link', 'set', interface, "address", + mac_address, run_as_root=True) _execute('ip', 'link', 'set', interface, 'up', run_as_root=True) return interface @@ -812,6 +818,11 @@ def ensure_bridge(_self, bridge, interface, net_attrs=None): _execute('brctl', 'setfd', bridge, 0, run_as_root=True) # _execute('brctl setageing %s 10' % bridge, run_as_root=True) _execute('brctl', 'stp', bridge, 'off', run_as_root=True) + # (danwent) bridge device MAC address can't be set directly. + # instead it inherits the MAC address of the first device on the + # bridge, which will either be the vlan interface, or a + # physical NIC. + _execute('ip', 'link', 'set', bridge, 'up', run_as_root=True) if interface: out, err = _execute('brctl', 'addif', bridge, interface, @@ -856,11 +867,10 @@ def ensure_bridge(_self, bridge, interface, net_attrs=None): # plugs interfaces using Open vSwitch class LinuxOVSInterfaceDriver(LinuxNetInterfaceDriver): - def plug(self, network): + def plug(self, network, mac_address): dev = "gw-" + str(network['id']) if not _device_exists(dev): bridge = FLAGS.linuxnet_ovs_integration_bridge - mac_addr = utils.generate_mac_address() _execute('ovs-vsctl', '--', '--may-exist', 'add-port', bridge, dev, '--', 'set', 'Interface', dev, "type=internal", @@ -869,10 +879,12 @@ def plug(self, network): '--', 'set', 'Interface', dev, "external-ids:iface-status=active", '--', 'set', 'Interface', dev, - "external-ids:attached-mac=%s" % mac_addr, + "external-ids:attached-mac=%s" % mac_address, run_as_root=True) - _execute('ip', 'link', 'set', dev, "address", mac_addr, + _execute('ip', 'link', 'set', dev, "address", mac_address, run_as_root=True) + _execute('ip', 'link', 'set', dev, 'up', run_as_root=True) + return dev def unplug(self, network): diff --git a/nova/network/manager.py b/nova/network/manager.py index 653d116fb02..1ad48fbc1e9 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -510,7 +510,7 @@ def ip6_dict(): def _allocate_mac_addresses(self, context, instance_id, networks): """Generates mac addresses and creates vif rows in db for them.""" for network in networks: - vif = {'address': utils.generate_mac_address(), + vif = {'address': self.generate_mac_address(), 'instance_id': instance_id, 'network_id': network['id']} # try FLAG times to create a vif record with a unique mac_address @@ -519,12 +519,20 @@ def _allocate_mac_addresses(self, context, instance_id, networks): self.db.virtual_interface_create(context, vif) break except exception.VirtualInterfaceCreateException: - vif['address'] = utils.generate_mac_address() + vif['address'] = self.generate_mac_address() else: self.db.virtual_interface_delete_by_instance(context, instance_id) raise exception.VirtualInterfaceMacAddressException() + def generate_mac_address(self): + """Generate an Ethernet MAC address.""" + mac = [0x02, 0x16, 0x3e, + random.randint(0x00, 0x7f), + random.randint(0x00, 0xff), + random.randint(0x00, 0xff)] + return ':'.join(map(lambda x: "%02x" % x, mac)) + def add_fixed_ip_to_instance(self, context, instance_id, host, network_id): """Adds a fixed ip to an instance from specified network.""" networks = [self.db.network_get(context, network_id)] @@ -796,7 +804,8 @@ def _setup_network(self, context, network_ref): """Sets up network on this host.""" network_ref['dhcp_server'] = self._get_dhcp_ip(context, network_ref) - dev = self.driver.plug(network_ref) + mac_address = self.generate_mac_address() + dev = self.driver.plug(network_ref, mac_address) self.driver.initialize_gateway_device(dev, network_ref) if not FLAGS.fake_network: @@ -897,7 +906,8 @@ def _setup_network(self, context, network_ref): address = network_ref['vpn_public_address'] network_ref['dhcp_server'] = self._get_dhcp_ip(context, network_ref) - dev = self.driver.plug(network_ref) + mac_address = self.generate_mac_address() + dev = self.driver.plug(network_ref, mac_address) self.driver.initialize_gateway_device(dev, network_ref) # NOTE(vish): only ensure this forward if the address hasn't been set diff --git a/nova/utils.py b/nova/utils.py index 46d60c73595..7276b6bd50c 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -338,15 +338,6 @@ def get_my_linklocal(interface): " :%(ex)s") % locals()) -def generate_mac_address(): - """Generate an Ethernet MAC address.""" - mac = [0x02, 0x16, 0x3e, - random.randint(0x00, 0x7f), - random.randint(0x00, 0xff), - random.randint(0x00, 0xff)] - return ':'.join(map(lambda x: "%02x" % x, mac)) - - def utcnow(): """Overridable version of utils.utcnow.""" if utcnow.override_time: From 53ca062830639de242d2cadda4f6bf473d4b6b62 Mon Sep 17 00:00:00 2001 From: Dan Wendlandt Date: Sun, 14 Aug 2011 20:05:18 -0700 Subject: [PATCH 8/8] fix missing 'run_as_root' from bad merge --- nova/network/linux_net.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 248f1ce5acd..57c1d0c283e 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -846,8 +846,10 @@ def ensure_bridge(_self, bridge, interface, net_attrs=None): fields = line.split() if fields and fields[0] == 'inet': params = fields[1:-1] - _execute(*_ip_bridge_cmd('del', params, fields[-1])) - _execute(*_ip_bridge_cmd('add', params, bridge)) + _execute(*_ip_bridge_cmd('del', params, fields[-1]), + run_as_root=True) + _execute(*_ip_bridge_cmd('add', params, bridge), + run_as_root=True) if gateway: _execute('route', 'add', 'default', 'gw', gateway, run_as_root=True)