Initial support for per-instance metadata, though the OpenStack API. …

…Key/value pairs can be specified at instance creation time and are returned in the details view. Support limits based on quota system.

nova/api/ec2/cloud.py

@@ -783,6 +783,9 @@ def disassociate_address(self, context, public_ip, **kwargs):
 
     def run_instances(self, context, **kwargs):
         max_count = int(kwargs.get('max_count', 1))
+        # NOTE(justinsb): the EC2 API doesn't support metadata here, but this
+        #  is needed for the unit tests.  Maybe the unit tests shouldn't be
+        #  calling the EC2 code
         instances = self.compute_api.create(context,
             instance_type=instance_types.get_by_type(
                 kwargs.get('instance_type', None)),
@@ -797,7 +800,8 @@ def run_instances(self, context, **kwargs):
             user_data=kwargs.get('user_data'),
             security_group=kwargs.get('security_group'),
             availability_zone=kwargs.get('placement', {}).get(
-                                  'AvailabilityZone'))
+                                  'AvailabilityZone'),
+            metadata=kwargs.get('metadata', []))
         return self._format_run_instances(context,
                                           instances[0]['reservation_id'])
 

nova/api/openstack/servers.py

@@ -78,9 +78,14 @@ def _translate_detail_keys(inst):
     except KeyError:
         LOG.debug(_("Failed to read public ip(s)"))
 
-    inst_dict['metadata'] = {}
     inst_dict['hostId'] = ''
 
+    # Return the metadata as a dictionary
+    metadata = {}
+    for item in inst['metadata']:
+        metadata[item['key']] = item['value']
+    inst_dict['metadata'] = metadata
+
     return dict(server=inst_dict)
 
 
@@ -162,22 +167,35 @@ def create(self, req):
         if not env:
             return faults.Fault(exc.HTTPUnprocessableEntity())
 
-        key_pair = auth_manager.AuthManager.get_key_pairs(
-            req.environ['nova.context'])[0]
+        context = req.environ['nova.context']
+
+        key_pair = auth_manager.AuthManager.get_key_pairs(context)[0]
         image_id = common.get_image_id_from_image_hash(self._image_service,
-            req.environ['nova.context'], env['server']['imageId'])
+            context, env['server']['imageId'])
         kernel_id, ramdisk_id = self._get_kernel_ramdisk_from_image(
             req, image_id)
+
+        # Metadata is a list, not a Dictionary, because we allow duplicate keys
+        # (even though JSON can't encode this)
+        # In future, we may not allow duplicate keys.
+        # However, the CloudServers API is not definitive on this front,
+        #  and we want to be compatible.
+        metadata = []
+        if env['server']['metadata']:
+            for k, v in env['server']['metadata'].items():
+                metadata.append({'key': k, 'value': v})
+
         instances = self.compute_api.create(
-            req.environ['nova.context'],
+            context,
             instance_types.get_by_flavor_id(env['server']['flavorId']),
             image_id,
             kernel_id=kernel_id,
             ramdisk_id=ramdisk_id,
             display_name=env['server']['name'],
             display_description=env['server']['name'],
             key_name=key_pair['name'],
-            key_data=key_pair['public_key'])
+            key_data=key_pair['public_key'],
+            metadata=metadata)
         return _translate_keys(instances[0])
 
     def update(self, req, id):

nova/compute/api.py

@@ -85,7 +85,7 @@ def create(self, context, instance_type,
                min_count=1, max_count=1,
                display_name='', display_description='',
                key_name=None, key_data=None, security_group='default',
-               availability_zone=None, user_data=None):
+               availability_zone=None, user_data=None, metadata=[]):
         """Create the number of instances requested if quota and
         other arguments check out ok."""
 
@@ -99,6 +99,30 @@ def create(self, context, instance_type,
                                      "run %s more instances of this type.") %
                                    num_instances, "InstanceLimitExceeded")
 
+        num_metadata = len(metadata)
+        quota_metadata = quota.allowed_metadata_items(context, num_metadata)
+        if quota_metadata < num_metadata:
+            pid = context.project_id
+            msg = (_("Quota exceeeded for %(pid)s,"
+                     " tried to set %(num_metadata)s metadata properties")
+                   % locals())
+            LOG.warn(msg)
+            raise quota.QuotaError(msg, "MetadataLimitExceeded")
+
+        # Because metadata is stored in the DB, we hard-code the size limits
+        # In future, we may support more variable length strings, so we act
+        #  as if this is quota-controlled for forwards compatibility
+        for metadata_item in metadata:
+            k = metadata_item['key']
+            v = metadata_item['value']
+            if len(k) > 255 or len(v) > 255:
+                pid = context.project_id
+                msg = (_("Quota exceeeded for %(pid)s,"
+                         " metadata property key or value too long")
+                       % locals())
+                LOG.warn(msg)
+                raise quota.QuotaError(msg, "MetadataLimitExceeded")
+
         is_vpn = image_id == FLAGS.vpn_image_id
         if not is_vpn:
             image = self.image_service.show(context, image_id)
@@ -155,7 +179,8 @@ def create(self, context, instance_type,
             'key_name': key_name,
             'key_data': key_data,
             'locked': False,
-            'availability_zone': availability_zone}
+            'availability_zone': availability_zone,
+            'metadata': metadata}
 
         elevated = context.elevated()
         instances = []

nova/db/sqlalchemy/api.py

@@ -715,6 +715,7 @@ def instance_get(context, instance_id, session=None):
                          options(joinedload_all('security_groups.rules')).\
                          options(joinedload('volumes')).\
                          options(joinedload_all('fixed_ip.network')).\
+                         options(joinedload('metadata')).\
                          filter_by(id=instance_id).\
                          filter_by(deleted=can_read_deleted(context)).\
                          first()
@@ -723,6 +724,7 @@ def instance_get(context, instance_id, session=None):
                          options(joinedload_all('fixed_ip.floating_ips')).\
                          options(joinedload_all('security_groups.rules')).\
                          options(joinedload('volumes')).\
+                         options(joinedload('metadata')).\
                          filter_by(project_id=context.project_id).\
                          filter_by(id=instance_id).\
                          filter_by(deleted=False).\

nova/db/sqlalchemy/migrate_repo/versions/004_add_instance_metadata.py

@@ -0,0 +1,78 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 Justin Santa Barbara
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from sqlalchemy import *
+from migrate import *
+
+from nova import log as logging
+
+
+meta = MetaData()
+
+
+# Just for the ForeignKey and column creation to succeed, these are not the
+# actual definitions of instances or services.
+instances = Table('instances', meta,
+        Column('id', Integer(), primary_key=True, nullable=False),
+        )
+
+quotas = Table('quotas', meta,
+        Column('id', Integer(),  primary_key=True, nullable=False),
+        )
+
+
+#
+# New Tables
+#
+
+instance_metadata_table = Table('instance_metadata', meta,
+        Column('created_at', DateTime(timezone=False)),
+        Column('updated_at', DateTime(timezone=False)),
+        Column('deleted_at', DateTime(timezone=False)),
+        Column('deleted', Boolean(create_constraint=True, name=None)),
+        Column('id', Integer(), primary_key=True, nullable=False),
+        Column('instance_id',
+               Integer(),
+               ForeignKey('instances.id'),
+               nullable=False),
+        Column('key',
+               String(length=255, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)),
+        Column('value',
+               String(length=255, convert_unicode=False, assert_unicode=None,
+                      unicode_error=None, _warn_on_bytestring=False)))
+
+
+#
+# New columns
+#
+quota_metadata_items = Column('metadata_items', Integer())
+
+
+def upgrade(migrate_engine):
+    # Upgrade operations go here. Don't create your own engine;
+    # bind migrate_engine to your metadata
+    meta.bind = migrate_engine
+    for table in (instance_metadata_table, ):
+        try:
+            table.create()
+        except Exception:
+            logging.info(repr(table))
+            logging.exception('Exception while creating table')
+            raise
+
+    quotas.create_column(quota_metadata_items)

nova/db/sqlalchemy/models.py

@@ -256,6 +256,7 @@ class Quota(BASE, NovaBase):
     volumes = Column(Integer)
     gigabytes = Column(Integer)
     floating_ips = Column(Integer)
+    metadata_items = Column(Integer)
 
 
 class ExportDevice(BASE, NovaBase):
@@ -536,6 +537,20 @@ class Console(BASE, NovaBase):
     pool = relationship(ConsolePool, backref=backref('consoles'))
 
 
+class InstanceMetadata(BASE, NovaBase):
+    """Represents a metadata key/value pair for an instance"""
+    __tablename__ = 'instance_metadata'
+    id = Column(Integer, primary_key=True)
+    key = Column(String(255))
+    value = Column(String(255))
+    instance_id = Column(Integer, ForeignKey('instances.id'), nullable=False)
+    instance = relationship(Instance, backref="metadata",
+                            foreign_keys=instance_id,
+                            primaryjoin='and_('
+                                'InstanceMetadata.instance_id == Instance.id,'
+                                'InstanceMetadata.deleted == False)')
+
+
 class Zone(BASE, NovaBase):
     """Represents a child zone of this zone."""
     __tablename__ = 'zones'
@@ -557,7 +572,8 @@ def register_models():
               Volume, ExportDevice, IscsiTarget, FixedIp, FloatingIp,
               Network, SecurityGroup, SecurityGroupIngressRule,
               SecurityGroupInstanceAssociation, AuthToken, User,
-              Project, Certificate, ConsolePool, Console, Zone)
+              Project, Certificate, ConsolePool, Console, Zone,
+              InstanceMetadata)
     engine = create_engine(FLAGS.sql_connection, echo=False)
     for model in models:
         model.metadata.create_all(engine)

nova/quota.py

@@ -35,14 +35,17 @@
                      'number of volume gigabytes allowed per project')
 flags.DEFINE_integer('quota_floating_ips', 10,
                      'number of floating ips allowed per project')
+flags.DEFINE_integer('quota_metadata_items', 128,
+                     'number of metadata items allowed per instance')
 
 
 def get_quota(context, project_id):
     rval = {'instances': FLAGS.quota_instances,
             'cores': FLAGS.quota_cores,
             'volumes': FLAGS.quota_volumes,
             'gigabytes': FLAGS.quota_gigabytes,
-            'floating_ips': FLAGS.quota_floating_ips}
+            'floating_ips': FLAGS.quota_floating_ips,
+            'metadata_items': FLAGS.quota_metadata_items}
     try:
         quota = db.quota_get(context, project_id)
         for key in rval.keys():
@@ -94,6 +97,15 @@ def allowed_floating_ips(context, num_floating_ips):
     return min(num_floating_ips, allowed_floating_ips)
 
 
+def allowed_metadata_items(context, num_metadata_items):
+    """Check quota; return min(num_metadata_items,allowed_metadata_items)"""
+    project_id = context.project_id
+    context = context.elevated()
+    quota = get_quota(context, project_id)
+    num_allowed_metadata_items = quota['metadata_items']
+    return min(num_metadata_items, num_allowed_metadata_items)
+
+
 class QuotaError(exception.ApiError):
     """Quota Exceeeded"""
     pass

nova/tests/api/openstack/test_servers.py

@@ -28,6 +28,7 @@
 from nova.api.openstack import servers
 import nova.db.api
 from nova.db.sqlalchemy.models import Instance
+from nova.db.sqlalchemy.models import InstanceMetadata
 import nova.rpc
 from nova.tests.api.openstack import fakes
 
@@ -64,6 +65,9 @@ def instance_address(context, instance_id):
 
 
 def stub_instance(id, user_id=1, private_address=None, public_addresses=None):
+    metadata = []
+    metadata.append(InstanceMetadata(key='seq', value=id))
+
     if public_addresses == None:
         public_addresses = list()
 
@@ -95,7 +99,8 @@ def stub_instance(id, user_id=1, private_address=None, public_addresses=None):
         "availability_zone": "",
         "display_name": "server%s" % id,
         "display_description": "",
-        "locked": False}
+        "locked": False,
+        "metadata": metadata}
 
     instance["fixed_ip"] = {
         "address": private_address,
@@ -214,7 +219,8 @@ def image_id_from_hash(*args, **kwargs):
             "get_image_id_from_image_hash", image_id_from_hash)
 
         body = dict(server=dict(
-            name='server_test', imageId=2, flavorId=2, metadata={},
+            name='server_test', imageId=2, flavorId=2,
+            metadata={'hello': 'world', 'open': 'stack'},
             personality={}))
         req = webob.Request.blank('/v1.0/servers')
         req.method = 'POST'
@@ -291,6 +297,7 @@ def test_get_all_server_details(self):
             self.assertEqual(s['id'], i)
             self.assertEqual(s['name'], 'server%d' % i)
             self.assertEqual(s['imageId'], 10)
+            self.assertEqual(s['metadata']['seq'], i)
             i += 1
 
     def test_server_pause(self):

nova/tests/test_quota.py

@@ -87,6 +87,18 @@ def test_quota_overrides(self):
         num_instances = quota.allowed_instances(self.context, 100,
             instance_types.INSTANCE_TYPES['m1.small'])
         self.assertEqual(num_instances, 10)
+
+        # metadata_items
+        too_many_items = FLAGS.quota_metadata_items + 1000
+        num_metadata_items = quota.allowed_metadata_items(self.context,
+                                                          too_many_items)
+        self.assertEqual(num_metadata_items, FLAGS.quota_metadata_items)
+        db.quota_update(self.context, self.project.id, {'metadata_items': 5})
+        num_metadata_items = quota.allowed_metadata_items(self.context,
+                                                          too_many_items)
+        self.assertEqual(num_metadata_items, 5)
+
+        # Cleanup
         db.quota_destroy(self.context, self.project.id)
 
     def test_too_many_instances(self):
@@ -151,3 +163,15 @@ def test_too_many_addresses(self):
         self.assertRaises(quota.QuotaError, self.cloud.allocate_address,
                           self.context)
         db.floating_ip_destroy(context.get_admin_context(), address)
+
+    def test_too_many_metadata_items(self):
+        metadata = {}
+        for i in range(FLAGS.quota_metadata_items + 1):
+            metadata['key%s' % i] = 'value%s' % i
+        self.assertRaises(quota.QuotaError, self.cloud.run_instances,
+                                            self.context,
+                                            min_count=1,
+                                            max_count=1,
+                                            instance_type='m1.small',
+                                            image_id='fake',
+                                            metadata=metadata)

run_tests.sh

@@ -73,7 +73,9 @@ fi
 
 if [ -z "$noseargs" ];
 then
-  run_tests && pep8 --repeat --show-pep8 --show-source --exclude=vcsversion.py bin/* nova setup.py || exit 1
+  srcfiles=`find bin -type f ! -name "nova.conf*"`
+  srcfiles+=" nova setup.py"
+  run_tests && pep8 --repeat --show-pep8 --show-source --exclude=vcsversion.py ${srcfiles} || exit 1
 else
   run_tests
 fi