From 0ceb3d53b27bcd81ef577eb00d849c5dfe7dae6d Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Tue, 24 Nov 2015 10:19:01 -0800 Subject: [PATCH] Add 'public' attribute to 'networks' entry Update the 'networks' entry to expect a different format. Instead of a list of dictionaries containing either 'net-id' or 'net-label, it now expects a list of dictionaries containing at least 'name' and optionally 'public' -- a boolean defaulting to false that indicates that the specified network should be considered by nodepool to be the public network. Both 'net-id' and 'net-label' are deprecated. This is a straightforward transition from usage of 'net-label' to this form. There is no equivalent of 'net-id' in this form. This is in service of adding Internap to the OpenStack configuration. Change-Id: I3eb8d5183bd4c082f56853cf71842f1353977248 --- doc/source/configuration.rst | 13 +++-- nodepool/cmd/config_validator.py | 15 ++++-- nodepool/fakeprovider.py | 14 ++++++ nodepool/nodepool.py | 25 +++++++++- nodepool/provider_manager.py | 47 +++++++++++------- .../tests/fixtures/config_validate/good.yaml | 17 +++++++ nodepool/tests/fixtures/node_net_name.yaml | 49 +++++++++++++++++++ nodepool/tests/test_nodepool.py | 15 ++++++ 8 files changed, 164 insertions(+), 31 deletions(-) create mode 100644 nodepool/tests/fixtures/node_net_name.yaml diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index 947c9c5b3..397ede3e8 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -288,8 +288,8 @@ provider, the Nodepool image types are also defined (see image-type: qcow2 ipv6-preferred: False networks: - - net-id: 'some-uuid' - - net-label: 'some-network-name' + - name: 'some-network-name' + public: True images: - name: trusty base-image: 'Trusty' @@ -393,11 +393,10 @@ provider, the Nodepool image types are also defined (see Default None ``networks`` (dict) - Specify custom Neutron networks that get attached to each node. You can - specify Neutron networks using either the ``net-id`` or ``net-label``. If - only the ``net-label`` is specified the network UUID is automatically - queried via the Nova os-tenant-networks API extension (this requires that - the cloud provider has deployed this extension). + Specify custom Neutron networks that get attached to each + node. Specify the ``name`` of the network (a string) and if the + network routes to the Internet, set the boolean ``public`` to + true. ``ipv6_preferred`` If it is set to True, nodepool will try to find ipv6 in public net first diff --git a/nodepool/cmd/config_validator.py b/nodepool/cmd/config_validator.py index e1a434d14..dd4880e19 100644 --- a/nodepool/cmd/config_validator.py +++ b/nodepool/cmd/config_validator.py @@ -43,6 +43,16 @@ def validate(self): 'config-drive': bool, } + old_network = { + 'net-id': str, + 'net-label': str, + } + + network = { + 'name': v.Required(str), + 'public': bool, + } + providers = { 'name': str, 'region-name': str, @@ -59,10 +69,7 @@ def validate(self): 'max-servers': int, 'pool': str, 'image-type': str, - 'networks': [{ - 'net-id': str, - 'net-label': str, - }], + 'networks': [v.Any(old_network, network)], 'boot-timeout': int, 'api-timeout': int, 'rate': float, diff --git a/nodepool/fakeprovider.py b/nodepool/fakeprovider.py index a3f9f2d15..8702042e6 100644 --- a/nodepool/fakeprovider.py +++ b/nodepool/fakeprovider.py @@ -190,6 +190,19 @@ def __init__(self, images, **kwargs): self.images = images +class FakeNeutronClient(object): + def __init__(self, networks=None): + if networks is None: + networks = [dict(id='fake-public-network-uuid', + name='fake-public-network-name'), + dict(id='fake-private-network-uuid', + name='fake-private-network-name')] + self.networks = networks + + def list_networks(self): + return dict(networks=self.networks) + + class FakeOpenStackCloud(object): def __init__(self, images=None): if images is None: @@ -200,6 +213,7 @@ def __init__(self, images=None): metadata={})]) self.nova_client = FakeClient(images) self._glance_client = FakeGlanceClient(images) + self.neutron_client = FakeNeutronClient() def create_image(self, **kwargs): image = self._glance_client.images.create(**kwargs) diff --git a/nodepool/nodepool.py b/nodepool/nodepool.py index 2ded927d7..316977430 100644 --- a/nodepool/nodepool.py +++ b/nodepool/nodepool.py @@ -1226,7 +1226,11 @@ def bootstrapServer(self, server, key, use_password=False): class ConfigValue(object): - pass + def __eq__(self, other): + if isinstance(other, ConfigValue): + if other.__dict__ == self.__dict__: + return True + return False class Config(ConfigValue): @@ -1269,6 +1273,10 @@ class DiskImage(ConfigValue): pass +class Network(ConfigValue): + pass + + class NodePool(threading.Thread): log = logging.getLogger("nodepool.NodePool") @@ -1371,7 +1379,20 @@ def loadConfig(self): p.boot_timeout = provider.get('boot-timeout', 60) p.launch_timeout = provider.get('launch-timeout', 3600) p.use_neutron = bool(provider.get('networks', ())) - p.networks = provider.get('networks') + p.networks = [] + for network in provider.get('networks', []): + n = Network() + p.networks.append(n) + if 'net-id' in network: + n.id = network['net-id'] + n.name = None + elif 'net-label' in network: + n.name = network['net-label'] + n.id = None + else: + n.name = network.get('name') + n.id = None + n.public = network.get('public', False) p.ipv6_preferred = provider.get('ipv6-preferred') p.azs = provider.get('availability-zones') p.template_hostname = provider.get( diff --git a/nodepool/provider_manager.py b/nodepool/provider_manager.py index 621f1056b..dc60e20b3 100644 --- a/nodepool/provider_manager.py +++ b/nodepool/provider_manager.py @@ -36,7 +36,7 @@ IPS_LIST_AGE = 5 # How long to keep a cached copy of the ip list -def get_public_ip(server, version=4): +def get_public_ip(server, provider, version=4): for addr in server.addresses.get('public', []): if type(addr) == type(u''): # Rackspace/openstack 1.0 return addr @@ -56,6 +56,12 @@ def get_public_ip(server, version=4): for addr in server.addresses.get('Ext-Net', []): if addr['version'] == version: # OVH return addr['addr'] + if provider.use_neutron: # Internap + for network in provider.networks: + if network.public and network.name: + for addr in server.addresses.get(network.name, []): + if addr['version'] == version: + return addr['addr'] return None @@ -83,7 +89,7 @@ def get_private_ip(server): return ret[0] -def make_server_dict(server): +def make_server_dict(server, provider): d = dict(id=str(server.id), name=server.name, status=server.status, @@ -96,9 +102,9 @@ def make_server_dict(server): d['progress'] = server.progress if hasattr(server, 'metadata'): d['metadata'] = server.metadata - d['public_v4'] = get_public_ip(server) + d['public_v4'] = get_public_ip(server, provider) d['private_v4'] = get_private_ip(server) - d['public_v6'] = get_public_ip(server, version=6) + d['public_v6'] = get_public_ip(server, provider, version=6) return d @@ -122,11 +128,12 @@ def main(self, client): class GetServerTask(Task): def main(self, client): + provider = self.args.pop('_nodepool_provider') try: server = client.nova_client.servers.get(self.args['server_id']) except novaclient.exceptions.NotFound: raise NotFound() - return make_server_dict(server) + return make_server_dict(server, provider) class DeleteServerTask(Task): @@ -136,8 +143,10 @@ def main(self, client): class ListServersTask(Task): def main(self, client): + provider = self.args.pop('_nodepool_provider') servers = client.nova_client.servers.list() - return [make_server_dict(server) for server in servers] + return [make_server_dict(server, provider) + for server in servers] class AddKeypairTask(Task): @@ -247,7 +256,7 @@ def main(self, client): class FindNetworkTask(Task): def main(self, client): for network in client.neutron_client.list_networks()['networks']: - if self.args['label'] == network['name']: + if self.args['name'] == network['name']: return dict(id=str(network['id'])) @@ -340,11 +349,11 @@ def findImage(self, name): self._images[name] = image return image - def findNetwork(self, label): - if label in self._networks: - return self._networks[label] - network = self.submitTask(FindNetworkTask(label=label)) - self._networks[label] = network + def findNetwork(self, name): + if name in self._networks: + return self._networks[name] + network = self.submitTask(FindNetworkTask(name=name)) + self._networks[name] = network return network def deleteImage(self, name): @@ -381,10 +390,10 @@ def createServer(self, name, min_ram, image_id=None, image_name=None, if self.provider.use_neutron: nics = [] for network in self.provider.networks: - if 'net-id' in network: - nics.append({'net-id': network['net-id']}) - elif 'net-label' in network: - net_id = self.findNetwork(network['net-label'])['id'] + if network.id: + nics.append({'net-id': network.id}) + elif network.name: + net_id = self.findNetwork(network.name)['id'] nics.append({'net-id': net_id}) else: raise Exception("Invalid 'networks' configuration.") @@ -412,7 +421,8 @@ def createServer(self, name, min_ram, image_id=None, image_name=None, return self.submitTask(CreateServerTask(**create_args)) def getServer(self, server_id): - return self.submitTask(GetServerTask(server_id=server_id)) + return self.submitTask(GetServerTask(server_id=server_id, + _nodepool_provider=self.provider)) def getFloatingIP(self, ip_id): return self.submitTask(GetFloatingIPTask(ip_id=ip_id)) @@ -571,7 +581,8 @@ def listServers(self, cache=True): # data until it succeeds. if self._servers_lock.acquire(False): try: - self._servers = self.submitTask(ListServersTask()) + self._servers = self.submitTask(ListServersTask( + _nodepool_provider=self.provider)) self._servers_time = time.time() finally: self._servers_lock.release() diff --git a/nodepool/tests/fixtures/config_validate/good.yaml b/nodepool/tests/fixtures/config_validate/good.yaml index 6bfbf0447..fea8f3464 100644 --- a/nodepool/tests/fixtures/config_validate/good.yaml +++ b/nodepool/tests/fixtures/config_validate/good.yaml @@ -883,6 +883,23 @@ providers: diskimage: devstack-fedora21-dib username: jenkins private-key: /home/nodepool/.ssh/id_rsa + - name: internap-nyj01 + region-name: 'nyj01' + cloud: internap + api-timeout: 60 + boot-timeout: 120 + max-servers: 0 + rate: 0.001 + networks: + - name: inap-17304-WAN2342 + public: True + images: + - name: centos-6 + min-ram: 8000 + name-filter: 'A1.8' + diskimage: centos-6 + username: jenkins + private-key: /home/nodepool/.ssh/id_rsa - name: tripleo-test-cloud-hp1 service-type: 'compute' service-name: 'nova' diff --git a/nodepool/tests/fixtures/node_net_name.yaml b/nodepool/tests/fixtures/node_net_name.yaml new file mode 100644 index 000000000..c7c935e0e --- /dev/null +++ b/nodepool/tests/fixtures/node_net_name.yaml @@ -0,0 +1,49 @@ +script-dir: . +images-dir: '{images_dir}' + +cron: + check: '*/15 * * * *' + cleanup: '*/1 * * * *' + image-update: '14 2 * * *' + +zmq-publishers: + - tcp://localhost:8881 + +gearman-servers: + - host: localhost + port: {gearman_port} + +labels: + - name: fake-label + image: fake-image + min-ready: 1 + providers: + - name: fake-provider + +providers: + - name: fake-provider + region-name: fake-region + keypair: 'if-present-use-this-keypair' + username: 'fake' + password: 'fake' + auth-url: 'fake' + project-id: 'fake' + max-servers: 96 + pool: 'fake' + networks: + - name: 'fake-public-network-name' + public: true + - name: 'fake-private-network-name' + rate: 0.0001 + images: + - name: fake-image + base-image: 'Fake Precise' + min-ram: 8192 + name-filter: 'Fake' + meta: + key: value + key2: value + setup: prepare_node_devstack.sh + +targets: + - name: fake-target diff --git a/nodepool/tests/test_nodepool.py b/nodepool/tests/test_nodepool.py index d5d0f14e9..dfa324532 100644 --- a/nodepool/tests/test_nodepool.py +++ b/nodepool/tests/test_nodepool.py @@ -49,6 +49,21 @@ def test_node(self): state=nodedb.READY) self.assertEqual(len(nodes), 1) + def test_node_net_name(self): + """Test that a node is created with a net name""" + configfile = self.setup_config('node_net_name.yaml') + pool = self.useNodepool(configfile, watermark_sleep=1) + pool.start() + self.waitForImage(pool, 'fake-provider', 'fake-image') + self.waitForNodes(pool) + + with pool.getDB().getSession() as session: + nodes = session.getNodes(provider_name='fake-provider', + label_name='fake-label', + target_name='fake-target', + state=nodedb.READY) + self.assertEqual(len(nodes), 1) + def test_dib_node(self): """Test that a dib image and node are created""" configfile = self.setup_config('node_dib.yaml')