Permalink
Browse files

Fixed #7052 -- Added support for natural keys in serialization.

git-svn-id: http://code.djangoproject.com/svn/django/trunk@11863 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
1 parent 44b9076 commit 35cc439228cd32dfa7a3ec919db01a8a5cd17d33 @freakboy3742 freakboy3742 committed Dec 14, 2009
@@ -47,6 +47,13 @@ def check_password(raw_password, enc_password):
class SiteProfileNotAvailable(Exception):
pass
+class PermissionManager(models.Manager):
+ def get_by_natural_key(self, codename, app_label, model):
+ return self.get(
+ codename=codename,
+ content_type=ContentType.objects.get_by_natural_key(app_label, model)
+ )
+
class Permission(models.Model):
"""The permissions system provides a way to assign permissions to specific users and groups of users.
@@ -63,6 +70,7 @@ class Permission(models.Model):
name = models.CharField(_('name'), max_length=50)
content_type = models.ForeignKey(ContentType)
codename = models.CharField(_('codename'), max_length=100)
+ objects = PermissionManager()
class Meta:
verbose_name = _('permission')
@@ -76,6 +84,10 @@ def __unicode__(self):
unicode(self.content_type),
unicode(self.name))
+ def natural_key(self):
+ return (self.codename,) + self.content_type.natural_key()
+ natural_key.dependencies = ['contenttypes.contenttype']
+
class Group(models.Model):
"""Groups are a generic way of categorizing users to apply permissions, or some other label, to those users. A user can belong to any number of groups.
@@ -8,6 +8,13 @@ class ContentTypeManager(models.Manager):
# This cache is shared by all the get_for_* methods.
_cache = {}
+ def get_by_natural_key(self, app_label, model):
+ try:
+ ct = self.__class__._cache[(app_label, model)]
+ except KeyError:
+ ct = self.get(app_label=app_label, model=model)
+ return ct
+
def get_for_model(self, model):
"""
Returns the ContentType object for a given model, creating the
@@ -93,3 +100,6 @@ def get_object_for_this_type(self, **kwargs):
so code that calls this method should catch it.
"""
return self.model_class()._default_manager.get(**kwargs)
+
+ def natural_key(self):
+ return (self.app_label, self.model)
@@ -13,6 +13,8 @@ class Command(BaseCommand):
help='Specifies the indent level to use when pretty-printing output'),
make_option('-e', '--exclude', dest='exclude',action='append', default=[],
help='App to exclude (use multiple --exclude to exclude multiple apps).'),
+ make_option('-n', '--natural', action='store_true', dest='use_natural_keys', default=False,
+ help='Use natural keys if they are available.'),
)
help = 'Output the contents of the database as a fixture of the given format.'
args = '[appname ...]'
@@ -24,6 +26,7 @@ def handle(self, *app_labels, **options):
indent = options.get('indent',None)
exclude = options.get('exclude',[])
show_traceback = options.get('traceback', False)
+ use_natural_keys = options.get('use_natural_keys', False)
excluded_apps = [get_app(app_label) for app_label in exclude]
@@ -67,18 +70,86 @@ def handle(self, *app_labels, **options):
except KeyError:
raise CommandError("Unknown serialization format: %s" % format)
+ # Now collate the objects to be serialized.
objects = []
- for app, model_list in app_list.items():
- if model_list is None:
- model_list = get_models(app)
-
- for model in model_list:
- if not model._meta.proxy:
- objects.extend(model._default_manager.all())
+ for model in sort_dependencies(app_list.items()):
+ if not model._meta.proxy:
+ objects.extend(model._default_manager.all())
try:
- return serializers.serialize(format, objects, indent=indent)
+ return serializers.serialize(format, objects, indent=indent,
+ use_natural_keys=use_natural_keys)
except Exception, e:
if show_traceback:
raise
raise CommandError("Unable to serialize database: %s" % e)
+
+def sort_dependencies(app_list):
+ """Sort a list of app,modellist pairs into a single list of models.
+
+ The single list of models is sorted so that any model with a natural key
+ is serialized before a normal model, and any model with a natural key
+ dependency has it's dependencies serialized first.
+ """
+ from django.db.models import get_model, get_models
+ # Process the list of models, and get the list of dependencies
+ model_dependencies = []
+ models = set()
+ for app, model_list in app_list:
+ if model_list is None:
+ model_list = get_models(app)
+
+ for model in model_list:
+ models.add(model)
+ # Add any explicitly defined dependencies
+ if hasattr(model, 'natural_key'):
+ deps = getattr(model.natural_key, 'dependencies', [])
+ if deps:
+ deps = [get_model(*d.split('.')) for d in deps]
+ else:
+ deps = []
+
+ # Now add a dependency for any FK or M2M relation with
+ # a model that defines a natural key
+ for field in model._meta.fields:
+ if hasattr(field.rel, 'to'):
+ rel_model = field.rel.to
+ if hasattr(rel_model, 'natural_key'):
+ deps.append(rel_model)
+ for field in model._meta.many_to_many:
+ rel_model = field.rel.to
+ if hasattr(rel_model, 'natural_key'):
+ deps.append(rel_model)
+ model_dependencies.append((model, deps))
+
+ model_dependencies.reverse()
+ # Now sort the models to ensure that dependencies are met. This
+ # is done by repeatedly iterating over the input list of models.
+ # If all the dependencies of a given model are in the final list,
+ # that model is promoted to the end of the final list. This process
+ # continues until the input list is empty, or we do a full iteration
+ # over the input models without promoting a model to the final list.
+ # If we do a full iteration without a promotion, that means there are
+ # circular dependencies in the list.
+ model_list = []
+ while model_dependencies:
+ skipped = []
+ changed = False
+ while model_dependencies:
+ model, deps = model_dependencies.pop()
+ if all((d not in models or d in model_list) for d in deps):
+ # If all of the models in the dependency list are either already
+ # on the final model list, or not on the original serialization list,
+ # then we've found another model with all it's dependencies satisfied.
+ model_list.append(model)
+ changed = True
+ else:
+ skipped.append((model, deps))
+ if not changed:
+ raise CommandError("Can't resolve dependencies for %s in serialized app list." %
+ ', '.join('%s.%s' % (model._meta.app_label, model._meta.object_name)
+ for model, deps in sorted(skipped, key=lambda obj: obj[0].__name__))
+ )
+ model_dependencies = skipped
+
+ return model_list
@@ -33,6 +33,7 @@ def serialize(self, queryset, **options):
self.stream = options.get("stream", StringIO())
self.selected_fields = options.get("fields")
+ self.use_natural_keys = options.get("use_natural_keys", False)
self.start_serialization()
for obj in queryset:
@@ -24,6 +24,7 @@ class Serializer(PythonSerializer):
def end_serialization(self):
self.options.pop('stream', None)
self.options.pop('fields', None)
+ self.options.pop('use_natural_keys', None)
simplejson.dump(self.objects, self.stream, cls=DjangoJSONEncoder, **self.options)
def getvalue(self):
@@ -47,17 +47,24 @@ def handle_field(self, obj, field):
def handle_fk_field(self, obj, field):
related = getattr(obj, field.name)
if related is not None:
- if field.rel.field_name == related._meta.pk.name:
- # Related to remote object via primary key
- related = related._get_pk_val()
+ if self.use_natural_keys and hasattr(related, 'natural_key'):
+ related = related.natural_key()
else:
- # Related to remote object via other field
- related = getattr(related, field.rel.field_name)
- self._current[field.name] = smart_unicode(related, strings_only=True)
+ if field.rel.field_name == related._meta.pk.name:
+ # Related to remote object via primary key
+ related = related._get_pk_val()
+ else:
+ # Related to remote object via other field
+ related = smart_unicode(getattr(related, field.rel.field_name), strings_only=True)
+ self._current[field.name] = related
def handle_m2m_field(self, obj, field):
if field.rel.through._meta.auto_created:
- self._current[field.name] = [smart_unicode(related._get_pk_val(), strings_only=True)
+ if self.use_natural_keys and hasattr(field.rel.to, 'natural_key'):
+ m2m_value = lambda value: value.natural_key()
+ else:
+ m2m_value = lambda value: smart_unicode(value._get_pk_val(), strings_only=True)
+ self._current[field.name] = [m2m_value(related)
for related in getattr(obj, field.name).iterator()]
def getvalue(self):
@@ -86,13 +93,28 @@ def Deserializer(object_list, **options):
# Handle M2M relations
if field.rel and isinstance(field.rel, models.ManyToManyRel):
- m2m_convert = field.rel.to._meta.pk.to_python
- m2m_data[field.name] = [m2m_convert(smart_unicode(pk)) for pk in field_value]
+ if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
+ def m2m_convert(value):
+ if hasattr(value, '__iter__'):
+ return field.rel.to._default_manager.get_by_natural_key(*value).pk
+ else:
+ return smart_unicode(field.rel.to._meta.pk.to_python(value))
+ else:
+ m2m_convert = lambda v: smart_unicode(field.rel.to._meta.pk.to_python(v))
+ m2m_data[field.name] = [m2m_convert(pk) for pk in field_value]
# Handle FK fields
elif field.rel and isinstance(field.rel, models.ManyToOneRel):
if field_value is not None:
- data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
+ if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
+ if hasattr(field_value, '__iter__'):
+ obj = field.rel.to._default_manager.get_by_natural_key(*field_value)
+ value = getattr(obj, field.rel.field_name)
+ else:
+ value = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
+ data[field.attname] = value
+ else:
+ data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
else:
data[field.attname] = None
@@ -26,9 +26,9 @@ class Serializer(PythonSerializer):
"""
Convert a queryset to YAML.
"""
-
+
internal_use_only = False
-
+
def handle_field(self, obj, field):
# A nasty special case: base YAML doesn't support serialization of time
# types (as opposed to dates or datetimes, which it does support). Since
@@ -40,10 +40,11 @@ def handle_field(self, obj, field):
self._current[field.name] = str(getattr(obj, field.name))
else:
super(Serializer, self).handle_field(obj, field)
-
+
def end_serialization(self):
self.options.pop('stream', None)
self.options.pop('fields', None)
+ self.options.pop('use_natural_keys', None)
yaml.dump(self.objects, self.stream, Dumper=DjangoSafeDumper, **self.options)
def getvalue(self):
@@ -81,13 +81,22 @@ def handle_fk_field(self, obj, field):
self._start_relational_field(field)
related = getattr(obj, field.name)
if related is not None:
- if field.rel.field_name == related._meta.pk.name:
- # Related to remote object via primary key
- related = related._get_pk_val()
+ if self.use_natural_keys and hasattr(related, 'natural_key'):
+ # If related object has a natural key, use it
+ related = related.natural_key()
+ # Iterable natural keys are rolled out as subelements
+ for key_value in related:
+ self.xml.startElement("natural", {})
+ self.xml.characters(smart_unicode(key_value))
+ self.xml.endElement("natural")
else:
- # Related to remote object via other field
- related = getattr(related, field.rel.field_name)
- self.xml.characters(smart_unicode(related))
+ if field.rel.field_name == related._meta.pk.name:
+ # Related to remote object via primary key
+ related = related._get_pk_val()
+ else:
+ # Related to remote object via other field
+ related = getattr(related, field.rel.field_name)
+ self.xml.characters(smart_unicode(related))
else:
self.xml.addQuickElement("None")
self.xml.endElement("field")
@@ -100,8 +109,25 @@ def handle_m2m_field(self, obj, field):
"""
if field.rel.through._meta.auto_created:
self._start_relational_field(field)
+ if self.use_natural_keys and hasattr(field.rel.to, 'natural_key'):
+ # If the objects in the m2m have a natural key, use it
+ def handle_m2m(value):
+ natural = value.natural_key()
+ # Iterable natural keys are rolled out as subelements
+ self.xml.startElement("object", {})
+ for key_value in natural:
+ self.xml.startElement("natural", {})
+ self.xml.characters(smart_unicode(key_value))
+ self.xml.endElement("natural")
+ self.xml.endElement("object")
+ else:
+ def handle_m2m(value):
+ self.xml.addQuickElement("object", attrs={
+ 'pk' : smart_unicode(value._get_pk_val())
+ })
for relobj in getattr(obj, field.name).iterator():
- self.xml.addQuickElement("object", attrs={"pk" : smart_unicode(relobj._get_pk_val())})
+ handle_m2m(relobj)
+
self.xml.endElement("field")
def _start_relational_field(self, field):
@@ -187,16 +213,40 @@ def _handle_fk_field_node(self, node, field):
if node.getElementsByTagName('None'):
return None
else:
- return field.rel.to._meta.get_field(field.rel.field_name).to_python(
- getInnerText(node).strip())
+ if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
+ keys = node.getElementsByTagName('natural')
+ if keys:
+ # If there are 'natural' subelements, it must be a natural key
+ field_value = [getInnerText(k).strip() for k in keys]
+ obj = field.rel.to._default_manager.get_by_natural_key(*field_value)
+ obj_pk = getattr(obj, field.rel.field_name)
+ else:
+ # Otherwise, treat like a normal PK
+ field_value = getInnerText(node).strip()
+ obj_pk = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
+ return obj_pk
+ else:
+ field_value = getInnerText(node).strip()
+ return field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
def _handle_m2m_field_node(self, node, field):
"""
Handle a <field> node for a ManyToManyField.
"""
- return [field.rel.to._meta.pk.to_python(
- c.getAttribute("pk"))
- for c in node.getElementsByTagName("object")]
+ if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
+ def m2m_convert(n):
+ keys = n.getElementsByTagName('natural')
+ if keys:
+ # If there are 'natural' subelements, it must be a natural key
+ field_value = [getInnerText(k).strip() for k in keys]
+ obj_pk = field.rel.to._default_manager.get_by_natural_key(*field_value).pk
+ else:
+ # Otherwise, treat like a normal PK value.
+ obj_pk = field.rel.to._meta.pk.to_python(n.getAttribute('pk'))
+ return obj_pk
+ else:
+ m2m_convert = lambda n: field.rel.to._meta.pk.to_python(n.getAttribute('pk'))
+ return [m2m_convert(c) for c in node.getElementsByTagName("object")]
def _get_model_from_node(self, node, attr):
"""
View
@@ -234,6 +234,17 @@ name to ``dumpdata``, the dumped output will be restricted to that model,
rather than the entire application. You can also mix application names and
model names.
+.. django-admin-option:: --natural
+
+.. versionadded:: 1.2
+
+Use :ref:`natural keys <topics-serialization-natural-keys>` to represent
+any foreign key and many-to-many relationship with a model that provides
+a natural key definition. If you are dumping ``contrib.auth`` ``Permission``
+objects or ``contrib.contenttypes`` ``ContentType`` objects, you should
+probably be using this flag.
+
+
flush
-----
@@ -701,7 +712,7 @@ information.
.. versionadded:: 1.2
-Use the ``--failfast`` option to stop running tests and report the failure
+Use the ``--failfast`` option to stop running tests and report the failure
immediately after a test fails.
testserver <fixture fixture ...>
Oops, something went wrong.

0 comments on commit 35cc439

Please sign in to comment.