Enable Record Data Validation in v2 API

This enables the validation of data against schemas defined in the
individual record objects

* This is a permanent interface, implemented in a temp fashion *

* Current we convert all the Record() objects to objects of the
  right type, and then validate. If validation is successful we restore
  the generic objects and send them to central.

* We override the RecordSet validate() command, and add extra logic
  This allows for the v2 API to function, but recursive validation from
  a Domain object will not work, until another solution is found.

* The way schemas are build up has also changed
** Each schema is built on .validate()
** There is no embedded "obj://<object_name>" references anymore
** The entire schema is added as one blob

Closes-Bug: #1338256
Implements: blueprint validation-cleanup
APIImpact

Change-Id: I8d1d614a9a9c0c1d3faeb0f98778231278f37bc4

designate/objects/__init__.py

@@ -29,17 +29,6 @@
 from designate.objects.pool_attribute import PoolAttribute, PoolAttributeList  # noqa
 from designate.objects.pool_ns_record import PoolNsRecord, PoolNsRecordList  # noqa
 from designate.objects.quota import Quota, QuotaList  # noqa
-from designate.objects.rrdata_a import RRData_A  # noqa
-from designate.objects.rrdata_aaaa import RRData_AAAA  # noqa
-from designate.objects.rrdata_cname import RRData_CNAME  # noqa
-from designate.objects.rrdata_mx import RRData_MX  # noqa
-from designate.objects.rrdata_ns import RRData_NS  # noqa
-from designate.objects.rrdata_ptr import RRData_PTR  # noqa
-from designate.objects.rrdata_soa import RRData_SOA  # noqa
-from designate.objects.rrdata_spf import RRData_SPF  # noqa
-from designate.objects.rrdata_srv import RRData_SRV  # noqa
-from designate.objects.rrdata_sshfp import RRData_SSHFP  # noqa
-from designate.objects.rrdata_txt import RRData_TXT  # noqa
 from designate.objects.record import Record, RecordList  # noqa
 from designate.objects.recordset import RecordSet, RecordSetList  # noqa
 from designate.objects.server import Server, ServerList  # noqa
@@ -50,3 +39,17 @@
 from designate.objects.validation_error import ValidationErrorList  # noqa
 from designate.objects.zone_transfer_request import ZoneTransferRequest, ZoneTransferRequestList  # noqa
 from designate.objects.zone_transfer_accept import ZoneTransferAccept, ZoneTransferAcceptList  # noqa
+
+#  Record Types
+
+from designate.objects.rrdata_a import A, AList  # noqa
+from designate.objects.rrdata_aaaa import AAAA, AAAAList  # noqa
+from designate.objects.rrdata_cname import CNAME, CNAMEList  # noqa
+from designate.objects.rrdata_mx import MX, MXList  # noqa
+from designate.objects.rrdata_ns import NS, NSList  # noqa
+from designate.objects.rrdata_ptr import PTR, PTRList  # noqa
+from designate.objects.rrdata_soa import SOA, SOAList  # noqa
+from designate.objects.rrdata_spf import SPF, SPFList  # noqa
+from designate.objects.rrdata_srv import SRV, SRVList  # noqa
+from designate.objects.rrdata_sshfp import SSHFP, SSHFPList  # noqa
+from designate.objects.rrdata_txt import TXT, TXTList  # noqa

designate/objects/base.py

@@ -84,32 +84,30 @@ def _schema_ref_resolver(uri):
     return obj.obj_get_schema()
 
 
-def make_class_validator(cls):
+def make_class_validator(obj):
 
     schema = {
         '$schema': 'http://json-schema.org/draft-04/hyper-schema',
-        'title': cls.obj_name(),
-        'description': 'Designate %s Object' % cls.obj_name(),
+        'title': obj.obj_name(),
+        'description': 'Designate %s Object' % obj.obj_name(),
     }
 
-    if issubclass(cls, ListObjectMixin):
+    if isinstance(obj, ListObjectMixin):
 
         schema['type'] = 'array',
-        schema['items'] = {
-            '$ref': 'obj://%s#/' % cls.LIST_ITEM_TYPE.obj_name()
-        }
+        schema['items'] = make_class_validator(obj.LIST_ITEM_TYPE)
 
     else:
         schema['type'] = 'object'
         schema['additionalProperties'] = False
         schema['required'] = []
         schema['properties'] = {}
 
-        for name, properties in cls.FIELDS.items():
+        for name, properties in obj.FIELDS.items():
             if properties.get('relation', False):
-                schema['properties'][name] = {
-                    '$ref': 'obj://%s#/' % properties.get('relation_cls')
-                }
+                if obj.obj_attr_is_set(name):
+                    schema['properties'][name] = \
+                        make_class_validator(getattr(obj, name))
             else:
                 schema['properties'][name] = properties.get('schema', {})
 
@@ -119,9 +117,11 @@ def make_class_validator(cls):
     resolver = jsonschema.RefResolver.from_schema(
         schema, handlers={'obj': _schema_ref_resolver})
 
-    cls._obj_validator = validators.Draft4Validator(
+    obj._obj_validator = validators.Draft4Validator(
         schema, resolver=resolver, format_checker=format.draft4_format_checker)
 
+    return schema
+
 
 class DesignateObjectMetaclass(type):
     def __init__(cls, names, bases, dict_):
@@ -132,7 +132,6 @@ def __init__(cls, names, bases, dict_):
             return
 
         make_class_properties(cls)
-        make_class_validator(cls)
 
         # Add a reference to the finished class into the _obj_classes
         # dictionary, allowing us to lookup classes by their name later - this
@@ -192,11 +191,10 @@ def from_dict(cls, _dict):
         for field, value in _dict.items():
             if (field in instance.FIELDS and
                     instance.FIELDS[field].get('relation', False)):
-
+                relation_cls_name = instance.FIELDS[field]['relation_cls']
                 # We're dealing with a relation, we'll want to create the
                 # correct object type and recurse
-                relation_cls = cls.obj_cls_from_name(
-                    instance.FIELDS[field]['relation_cls'])
+                relation_cls = cls.obj_cls_from_name(relation_cls_name)
 
                 if isinstance(value, list):
                     setattr(instance, field, relation_cls.from_list(value))
@@ -282,9 +280,15 @@ def update(self, values):
     @property
     def is_valid(self):
         """Returns True if the Object is valid."""
+
+        make_class_validator(self)
+
         return self._obj_validator.is_valid(self.to_dict())
 
     def validate(self):
+
+        make_class_validator(self)
+
         # NOTE(kiall): We make use of the Object registry here in order to
         #              avoid an impossible circular import.
         ValidationErrorList = self.obj_cls_from_name('ValidationErrorList')

designate/objects/recordset.py

@@ -12,11 +12,17 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+from copy import deepcopy
+
+from designate import exceptions
 from designate.objects import base
+from designate.objects.validation_error import ValidationError
+from designate.objects.validation_error import ValidationErrorList
 
 
 class RecordSet(base.DictObjectMixin, base.PersistentObjectMixin,
                 base.DesignateObject):
+
     @property
     def action(self):
         # Return action as UPDATE if present. CREATE and DELETE are returned
@@ -100,8 +106,84 @@ def status(self):
             'relation': True,
             'relation_cls': 'RecordList'
         },
+        # TODO(graham): implement the polymorphic class relations
+        # 'records': {
+        #     'polymorphic': 'type',
+        #     'relation': True,
+        #     'relation_cls': lambda type_: '%sList' % type_
+        # },
     }
 
+    def validate(self):
+
+        # Get the right classes (e.g. A for Recordsets with type: 'A')
+        record_list_cls = self.obj_cls_from_name('%sList' % self.type)
+        record_cls = self.obj_cls_from_name(self.type)
+
+        errors = ValidationErrorList()
+        error_indexes = []
+        # Copy these for safekeeping
+        old_records = deepcopy(self.records)
+
+        # Blank the records for this object with the right list type
+        self.records = record_list_cls()
+
+        i = 0
+
+        for record in old_records:
+            record_obj = record_cls()
+            try:
+                record_obj._from_string(record.data)
+            # The _from_string() method will throw a ValueError if there is not
+            # enough data blobs
+            except ValueError as e:
+                # Something broke in the _from_string() method
+                # Fake a correct looking ValidationError() object
+                e = ValidationError()
+                e.path = ['records', i]
+                e.validator = 'format'
+                e.validator_value = [self.type]
+                e.message = ("'%(data)s' is not a '%(type)s' Record"
+                             % {'data': record.data, 'type': self.type})
+                # Add it to the list for later
+                errors.append(e)
+                error_indexes.append(i)
+            else:
+                # Seems to have loaded right - add it to be validated by
+                # JSONSchema
+                self.records.append(record_obj)
+            i += 1
+
+        try:
+            # Run the actual validate code
+            super(RecordSet, self).validate()
+
+        except exceptions.InvalidObject as e:
+            # Something is wrong according to JSONSchema - append our errors
+            increment = 0
+            # This code below is to make sure we have the index for the record
+            # list correct. JSONSchema may be missing some of the objects due
+            # to validation above, so this re - inserts them, and makes sure
+            # the index is right
+            for error in e.errors:
+                error.path[1] += increment
+                while error.path[1] in error_indexes:
+                    increment += 1
+                    error.path[1] += 1
+            # Add the list from above
+            e.errors.extend(errors)
+            # Raise the exception
+            raise e
+        else:
+            # If JSONSchema passes, but we found parsing errors,
+            # raise an exception
+            if len(errors) > 0:
+                raise exceptions.InvalidObject(
+                    "Provided object does not match "
+                    "schema", errors=errors, object=self)
+        # Send in the traditional Record objects to central / storage
+        self.records = old_records
+
 
 class RecordSetList(base.ListObjectMixin, base.DesignateObject,
                     base.PagedListObjectMixin):

designate/objects/rrdata_a.py

@@ -13,9 +13,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 from designate.objects.record import Record
+from designate.objects.record import RecordList
 
 
-class RRData_A(Record):
+class A(Record):
     """
     A Resource Record Type
     Defined in: RFC1035
@@ -39,3 +40,8 @@ def _from_string(self, value):
     # The record type is defined in the RFC. This will be used when the record
     # is sent by mini-dns.
     RECORD_TYPE = 1
+
+
+class AList(RecordList):
+
+    LIST_ITEM_TYPE = A

designate/objects/rrdata_aaaa.py

@@ -13,9 +13,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 from designate.objects.record import Record
+from designate.objects.record import RecordList
 
 
-class RRData_AAAA(Record):
+class AAAA(Record):
     """
     AAAA Resource Record Type
     Defined in: RFC3596
@@ -39,3 +40,8 @@ def _from_string(self, value):
     # The record type is defined in the RFC. This will be used when the record
     # is sent by mini-dns.
     RECORD_TYPE = 28
+
+
+class AAAAList(RecordList):
+
+    LIST_ITEM_TYPE = AAAA

designate/objects/rrdata_cname.py

@@ -13,9 +13,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 from designate.objects.record import Record
+from designate.objects.record import RecordList
 
 
-class RRData_CNAME(Record):
+class CNAME(Record):
     """
     CNAME Resource Record Type
     Defined in: RFC1035
@@ -40,3 +41,8 @@ def _from_string(self, value):
     # The record type is defined in the RFC. This will be used when the record
     # is sent by mini-dns.
     RECORD_TYPE = 5
+
+
+class CNAMEList(RecordList):
+
+    LIST_ITEM_TYPE = CNAME

designate/objects/rrdata_mx.py

@@ -13,9 +13,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 from designate.objects.record import Record
+from designate.objects.record import RecordList
 
 
-class RRData_MX(Record):
+class MX(Record):
     """
     MX Resource Record Type
     Defined in: RFC1035
@@ -43,8 +44,16 @@ def _to_string(self):
         return '%(priority)s %(exchange)s' % self
 
     def _from_string(self, value):
-        self.priority, self.exchange = value.split(' ')
+        priority, exchange = value.split(' ')
+
+        self.priority = int(priority)
+        self.exchange = exchange
 
     # The record type is defined in the RFC. This will be used when the record
     # is sent by mini-dns.
     RECORD_TYPE = 15
+
+
+class MXList(RecordList):
+
+    LIST_ITEM_TYPE = MX

designate/objects/rrdata_ns.py

@@ -13,9 +13,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 from designate.objects.record import Record
+from designate.objects.record import RecordList
 
 
-class RRData_NS(Record):
+class NS(Record):
     """
     NS Resource Record Type
     Defined in: RFC1035
@@ -40,3 +41,8 @@ def _from_string(self, value):
     # The record type is defined in the RFC. This will be used when the record
     # is sent by mini-dns.
     RECORD_TYPE = 2
+
+
+class NSList(RecordList):
+
+    LIST_ITEM_TYPE = NS

designate/objects/rrdata_ptr.py

@@ -13,9 +13,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 from designate.objects.record import Record
+from designate.objects.record import RecordList
 
 
-class RRData_PTR(Record):
+class PTR(Record):
     """
     PTR Resource Record Type
     Defined in: RFC1035
@@ -40,3 +41,8 @@ def _from_string(self, value):
     # The record type is defined in the RFC. This will be used when the record
     # is sent by mini-dns.
     RECORD_TYPE = 12
+
+
+class PTRList(RecordList):
+
+    LIST_ITEM_TYPE = PTR