Add 'public' attribute to 'networks' entry

Update the 'networks' entry to expect a different format.  Instead
of a list of dictionaries containing either 'net-id' or 'net-label,
it now expects a list of dictionaries containing at least 'name' and
optionally 'public' -- a boolean defaulting to false that indicates
that the specified network should be considered by nodepool to be the
public network.

Both 'net-id' and 'net-label' are deprecated.  This is a straightforward
transition from usage of 'net-label' to this form.  There is no
equivalent of 'net-id' in this form.

This is in service of adding Internap to the OpenStack configuration.

Change-Id: I3eb8d5183bd4c082f56853cf71842f1353977248

doc/source/configuration.rst

@@ -288,8 +288,8 @@ provider, the Nodepool image types are also defined (see
       image-type: qcow2
       ipv6-preferred: False
       networks:
-        - net-id: 'some-uuid'
-        - net-label: 'some-network-name'
+        - name: 'some-network-name'
+          public: True
       images:
         - name: trusty
           base-image: 'Trusty'
@@ -393,11 +393,10 @@ provider, the Nodepool image types are also defined (see
     Default None
 
   ``networks`` (dict)
-    Specify custom Neutron networks that get attached to each node. You can
-    specify Neutron networks using either the ``net-id`` or ``net-label``. If
-    only the ``net-label`` is specified the network UUID is automatically
-    queried via the Nova os-tenant-networks API extension (this requires that
-    the cloud provider has deployed this extension).
+    Specify custom Neutron networks that get attached to each
+    node. Specify the ``name`` of the network (a string) and if the
+    network routes to the Internet, set the boolean ``public`` to
+    true.
 
   ``ipv6_preferred``
     If it is set to True, nodepool will try to find ipv6 in public net first

nodepool/cmd/config_validator.py

@@ -43,6 +43,16 @@ def validate(self):
             'config-drive': bool,
         }
 
+        old_network = {
+            'net-id': str,
+            'net-label': str,
+        }
+
+        network = {
+            'name': v.Required(str),
+            'public': bool,
+        }
+
         providers = {
             'name': str,
             'region-name': str,
@@ -59,10 +69,7 @@ def validate(self):
             'max-servers': int,
             'pool': str,
             'image-type': str,
-            'networks': [{
-                'net-id': str,
-                'net-label': str,
-            }],
+            'networks': [v.Any(old_network, network)],
             'boot-timeout': int,
             'api-timeout': int,
             'rate': float,

nodepool/fakeprovider.py

@@ -190,6 +190,19 @@ def __init__(self, images, **kwargs):
         self.images = images
 
 
+class FakeNeutronClient(object):
+    def __init__(self, networks=None):
+        if networks is None:
+            networks = [dict(id='fake-public-network-uuid',
+                             name='fake-public-network-name'),
+                        dict(id='fake-private-network-uuid',
+                             name='fake-private-network-name')]
+        self.networks = networks
+
+    def list_networks(self):
+        return dict(networks=self.networks)
+
+
 class FakeOpenStackCloud(object):
     def __init__(self, images=None):
         if images is None:
@@ -200,6 +213,7 @@ def __init__(self, images=None):
                                      metadata={})])
         self.nova_client = FakeClient(images)
         self._glance_client = FakeGlanceClient(images)
+        self.neutron_client = FakeNeutronClient()
 
     def create_image(self, **kwargs):
         image = self._glance_client.images.create(**kwargs)

nodepool/nodepool.py

@@ -1226,7 +1226,11 @@ def bootstrapServer(self, server, key, use_password=False):
 
 
 class ConfigValue(object):
-    pass
+    def __eq__(self, other):
+        if isinstance(other, ConfigValue):
+            if other.__dict__ == self.__dict__:
+                return True
+        return False
 
 
 class Config(ConfigValue):
@@ -1269,6 +1273,10 @@ class DiskImage(ConfigValue):
     pass
 
 
+class Network(ConfigValue):
+    pass
+
+
 class NodePool(threading.Thread):
     log = logging.getLogger("nodepool.NodePool")
 
@@ -1371,7 +1379,20 @@ def loadConfig(self):
             p.boot_timeout = provider.get('boot-timeout', 60)
             p.launch_timeout = provider.get('launch-timeout', 3600)
             p.use_neutron = bool(provider.get('networks', ()))
-            p.networks = provider.get('networks')
+            p.networks = []
+            for network in provider.get('networks', []):
+                n = Network()
+                p.networks.append(n)
+                if 'net-id' in network:
+                    n.id = network['net-id']
+                    n.name = None
+                elif 'net-label' in network:
+                    n.name = network['net-label']
+                    n.id = None
+                else:
+                    n.name = network.get('name')
+                    n.id = None
+                n.public = network.get('public', False)
             p.ipv6_preferred = provider.get('ipv6-preferred')
             p.azs = provider.get('availability-zones')
             p.template_hostname = provider.get(

nodepool/provider_manager.py

@@ -36,7 +36,7 @@
 IPS_LIST_AGE = 5      # How long to keep a cached copy of the ip list
 
 
-def get_public_ip(server, version=4):
+def get_public_ip(server, provider, version=4):
     for addr in server.addresses.get('public', []):
         if type(addr) == type(u''):  # Rackspace/openstack 1.0
             return addr
@@ -56,6 +56,12 @@ def get_public_ip(server, version=4):
     for addr in server.addresses.get('Ext-Net', []):
         if addr['version'] == version:  # OVH
             return addr['addr']
+    if provider.use_neutron:  # Internap
+        for network in provider.networks:
+            if network.public and network.name:
+                for addr in server.addresses.get(network.name, []):
+                    if addr['version'] == version:
+                        return addr['addr']
     return None
 
 
@@ -83,7 +89,7 @@ def get_private_ip(server):
     return ret[0]
 
 
-def make_server_dict(server):
+def make_server_dict(server, provider):
     d = dict(id=str(server.id),
              name=server.name,
              status=server.status,
@@ -96,9 +102,9 @@ def make_server_dict(server):
         d['progress'] = server.progress
     if hasattr(server, 'metadata'):
         d['metadata'] = server.metadata
-    d['public_v4'] = get_public_ip(server)
+    d['public_v4'] = get_public_ip(server, provider)
     d['private_v4'] = get_private_ip(server)
-    d['public_v6'] = get_public_ip(server, version=6)
+    d['public_v6'] = get_public_ip(server, provider, version=6)
     return d
 
 
@@ -122,11 +128,12 @@ def main(self, client):
 
 class GetServerTask(Task):
     def main(self, client):
+        provider = self.args.pop('_nodepool_provider')
         try:
             server = client.nova_client.servers.get(self.args['server_id'])
         except novaclient.exceptions.NotFound:
             raise NotFound()
-        return make_server_dict(server)
+        return make_server_dict(server, provider)
 
 
 class DeleteServerTask(Task):
@@ -136,8 +143,10 @@ def main(self, client):
 
 class ListServersTask(Task):
     def main(self, client):
+        provider = self.args.pop('_nodepool_provider')
         servers = client.nova_client.servers.list()
-        return [make_server_dict(server) for server in servers]
+        return [make_server_dict(server, provider)
+                for server in servers]
 
 
 class AddKeypairTask(Task):
@@ -247,7 +256,7 @@ def main(self, client):
 class FindNetworkTask(Task):
     def main(self, client):
         for network in client.neutron_client.list_networks()['networks']:
-            if self.args['label'] == network['name']:
+            if self.args['name'] == network['name']:
                 return dict(id=str(network['id']))
 
 
@@ -340,11 +349,11 @@ def findImage(self, name):
         self._images[name] = image
         return image
 
-    def findNetwork(self, label):
-        if label in self._networks:
-            return self._networks[label]
-        network = self.submitTask(FindNetworkTask(label=label))
-        self._networks[label] = network
+    def findNetwork(self, name):
+        if name in self._networks:
+            return self._networks[name]
+        network = self.submitTask(FindNetworkTask(name=name))
+        self._networks[name] = network
         return network
 
     def deleteImage(self, name):
@@ -381,10 +390,10 @@ def createServer(self, name, min_ram, image_id=None, image_name=None,
         if self.provider.use_neutron:
             nics = []
             for network in self.provider.networks:
-                if 'net-id' in network:
-                    nics.append({'net-id': network['net-id']})
-                elif 'net-label' in network:
-                    net_id = self.findNetwork(network['net-label'])['id']
+                if network.id:
+                    nics.append({'net-id': network.id})
+                elif network.name:
+                    net_id = self.findNetwork(network.name)['id']
                     nics.append({'net-id': net_id})
                 else:
                     raise Exception("Invalid 'networks' configuration.")
@@ -412,7 +421,8 @@ def createServer(self, name, min_ram, image_id=None, image_name=None,
         return self.submitTask(CreateServerTask(**create_args))
 
     def getServer(self, server_id):
-        return self.submitTask(GetServerTask(server_id=server_id))
+        return self.submitTask(GetServerTask(server_id=server_id,
+                                             _nodepool_provider=self.provider))
 
     def getFloatingIP(self, ip_id):
         return self.submitTask(GetFloatingIPTask(ip_id=ip_id))
@@ -571,7 +581,8 @@ def listServers(self, cache=True):
             # data until it succeeds.
             if self._servers_lock.acquire(False):
                 try:
-                    self._servers = self.submitTask(ListServersTask())
+                    self._servers = self.submitTask(ListServersTask(
+                        _nodepool_provider=self.provider))
                     self._servers_time = time.time()
                 finally:
                     self._servers_lock.release()

nodepool/tests/fixtures/config_validate/good.yaml

@@ -883,6 +883,23 @@ providers:
         diskimage: devstack-fedora21-dib
         username: jenkins
         private-key: /home/nodepool/.ssh/id_rsa
+  - name: internap-nyj01
+    region-name: 'nyj01'
+    cloud: internap
+    api-timeout: 60
+    boot-timeout: 120
+    max-servers: 0
+    rate: 0.001
+    networks:
+      - name: inap-17304-WAN2342
+        public: True
+    images:
+      - name: centos-6
+        min-ram: 8000
+        name-filter: 'A1.8'
+        diskimage: centos-6
+        username: jenkins
+        private-key: /home/nodepool/.ssh/id_rsa
   - name: tripleo-test-cloud-hp1
     service-type: 'compute'
     service-name: 'nova'

nodepool/tests/fixtures/node_net_name.yaml

@@ -0,0 +1,49 @@
+script-dir: .
+images-dir: '{images_dir}'
+
+cron:
+  check: '*/15 * * * *'
+  cleanup: '*/1 * * * *'
+  image-update: '14 2 * * *'
+
+zmq-publishers:
+  - tcp://localhost:8881
+
+gearman-servers:
+  - host: localhost
+    port: {gearman_port}
+
+labels:
+  - name: fake-label
+    image: fake-image
+    min-ready: 1
+    providers:
+      - name: fake-provider
+
+providers:
+  - name: fake-provider
+    region-name: fake-region
+    keypair: 'if-present-use-this-keypair'
+    username: 'fake'
+    password: 'fake'
+    auth-url: 'fake'
+    project-id: 'fake'
+    max-servers: 96
+    pool: 'fake'
+    networks:
+      - name: 'fake-public-network-name'
+        public: true
+      - name: 'fake-private-network-name'
+    rate: 0.0001
+    images:
+      - name: fake-image
+        base-image: 'Fake Precise'
+        min-ram: 8192
+        name-filter: 'Fake'
+        meta:
+          key: value
+          key2: value
+        setup: prepare_node_devstack.sh
+
+targets:
+  - name: fake-target

nodepool/tests/test_nodepool.py

@@ -49,6 +49,21 @@ def test_node(self):
                                      state=nodedb.READY)
             self.assertEqual(len(nodes), 1)
 
+    def test_node_net_name(self):
+        """Test that a node is created with a net name"""
+        configfile = self.setup_config('node_net_name.yaml')
+        pool = self.useNodepool(configfile, watermark_sleep=1)
+        pool.start()
+        self.waitForImage(pool, 'fake-provider', 'fake-image')
+        self.waitForNodes(pool)
+
+        with pool.getDB().getSession() as session:
+            nodes = session.getNodes(provider_name='fake-provider',
+                                     label_name='fake-label',
+                                     target_name='fake-target',
+                                     state=nodedb.READY)
+            self.assertEqual(len(nodes), 1)
+
     def test_dib_node(self):
         """Test that a dib image and node are created"""
         configfile = self.setup_config('node_dib.yaml')