Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Fixed 19385 -- Added ORM support for multicolumn joins

This patch iproved two major parts in Django. First, the fields.related
was refactored. The main addition there was ForeignObject. Second, the
ORM now handles multicolumn joins in most cases, though there are still
cases that do not work correcly (split_exclude() for example).

In addition there were extesive changes to how GenericRelation works.
Before it was a fake m2m field, now it is a pure virtual fields and is
based on ForeignObject.

There is still much room for improvement. The related fields code is
still somewhat confusing, and how fields are represented in model._meta
should also be revisited.

This patch was written mostly by Jeremy Tillman with some final polish
by the committer.
  • Loading branch information...
commit 266de5f9ae9e9f2fbfaec3b7e4b5fb9941967801 1 parent bc35b95
Anssi Kääriäinen authored March 24, 2013
0  tests/foreign_object/__init__.py
No changes.
152  tests/foreign_object/models.py
... ...
@@ -0,0 +1,152 @@
  1
+import datetime
  2
+
  3
+from django.db import models
  4
+from django.db.models.fields.related import ReverseSingleRelatedObjectDescriptor
  5
+from django.utils.encoding import python_2_unicode_compatible
  6
+from django.utils.translation import get_language
  7
+
  8
+class Country(models.Model):
  9
+    # Table Column Fields
  10
+    name = models.CharField(max_length=50)
  11
+
  12
+    def __unicode__(self):
  13
+        return self.name
  14
+
  15
+
  16
+class Person(models.Model):
  17
+    # Table Column Fields
  18
+    name = models.CharField(max_length=128)
  19
+    person_country_id = models.IntegerField()
  20
+
  21
+    # Relation Fields
  22
+    person_country = models.ForeignObject(
  23
+        Country, from_fields=['person_country_id'], to_fields=['id'])
  24
+    friends = models.ManyToManyField('self', through='Friendship', symmetrical=False)
  25
+
  26
+    class Meta:
  27
+        ordering = ('name',)
  28
+
  29
+    def __unicode__(self):
  30
+        return self.name
  31
+
  32
+class Group(models.Model):
  33
+    # Table Column Fields
  34
+    name = models.CharField(max_length=128)
  35
+    group_country = models.ForeignKey(Country)
  36
+    members = models.ManyToManyField(Person, related_name='groups', through='Membership')
  37
+
  38
+    class Meta:
  39
+        ordering = ('name',)
  40
+
  41
+    def __unicode__(self):
  42
+        return self.name
  43
+
  44
+
  45
+class Membership(models.Model):
  46
+    # Table Column Fields
  47
+    membership_country = models.ForeignKey(Country)
  48
+    date_joined = models.DateTimeField(default=datetime.datetime.now)
  49
+    invite_reason = models.CharField(max_length=64, null=True)
  50
+    person_id = models.IntegerField()
  51
+    group_id = models.IntegerField()
  52
+
  53
+    # Relation Fields
  54
+    person = models.ForeignObject(Person,
  55
+        from_fields=['membership_country', 'person_id'],
  56
+        to_fields=['person_country_id', 'id'])
  57
+    group = models.ForeignObject(Group,
  58
+        from_fields=['membership_country', 'group_id'],
  59
+        to_fields=['group_country', 'id'])
  60
+
  61
+    class Meta:
  62
+        ordering = ('date_joined', 'invite_reason')
  63
+
  64
+    def __unicode__(self):
  65
+        return "%s is a member of %s" % (self.person.name, self.group.name)
  66
+
  67
+
  68
+class Friendship(models.Model):
  69
+    # Table Column Fields
  70
+    from_friend_country = models.ForeignKey(Country, related_name="from_friend_country")
  71
+    from_friend_id = models.IntegerField()
  72
+    to_friend_country_id = models.IntegerField()
  73
+    to_friend_id = models.IntegerField()
  74
+
  75
+    # Relation Fields
  76
+    from_friend = models.ForeignObject(Person,
  77
+        from_fields=['from_friend_country', 'from_friend_id'],
  78
+        to_fields=['person_country_id', 'id'],
  79
+        related_name='from_friend')
  80
+
  81
+    to_friend_country = models.ForeignObject(Country,
  82
+        from_fields=['to_friend_country_id'],
  83
+        to_fields=['id'],
  84
+        related_name='to_friend_country')
  85
+
  86
+    to_friend = models.ForeignObject(Person,
  87
+        from_fields=['to_friend_country_id', 'to_friend_id'],
  88
+        to_fields=['person_country_id', 'id'],
  89
+        related_name='to_friend')
  90
+
  91
+class ArticleTranslationDescriptor(ReverseSingleRelatedObjectDescriptor):
  92
+    """
  93
+    The set of articletranslation should not set any local fields.
  94
+    """
  95
+    def __set__(self, instance, value):
  96
+        if instance is None:
  97
+            raise AttributeError("%s must be accessed via instance" % self.field.name)
  98
+        setattr(instance, self.cache_name, value)
  99
+        if value is not None and not self.field.rel.multiple:
  100
+            setattr(value, self.field.related.get_cache_name(), instance)
  101
+
  102
+class ColConstraint(object):
  103
+    # Antyhing with as_sql() method works in get_extra_restriction().
  104
+    def __init__(self, alias, col, value):
  105
+        self.alias, self.col, self.value = alias, col, value
  106
+
  107
+    def as_sql(self, qn, connection):
  108
+        return '%s.%s = %%s' % (qn(self.alias), qn(self.col)), [self.value]
  109
+
  110
+class ActiveTranslationField(models.ForeignObject):
  111
+    """
  112
+    This field will allow querying and fetching the currently active translation
  113
+    for Article from ArticleTranslation.
  114
+    """
  115
+    requires_unique_target = False
  116
+
  117
+    def get_extra_restriction(self, where_class, alias, related_alias):
  118
+        return ColConstraint(alias, 'lang', get_language())
  119
+
  120
+    def get_extra_descriptor_filter(self):
  121
+        return {'lang': get_language()}
  122
+
  123
+    def contribute_to_class(self, cls, name):
  124
+        super(ActiveTranslationField, self).contribute_to_class(cls, name)
  125
+        setattr(cls, self.name, ArticleTranslationDescriptor(self))
  126
+
  127
+@python_2_unicode_compatible
  128
+class Article(models.Model):
  129
+    active_translation = ActiveTranslationField(
  130
+        'ArticleTranslation',
  131
+        from_fields=['id'],
  132
+        to_fields=['article'],
  133
+        related_name='+',
  134
+        null=True)
  135
+    pub_date = models.DateField()
  136
+
  137
+    def __str__(self):
  138
+        try:
  139
+            return self.active_translation.title
  140
+        except ArticleTranslation.DoesNotExist:
  141
+            return '[No translation found]'
  142
+
  143
+class ArticleTranslation(models.Model):
  144
+    article = models.ForeignKey(Article)
  145
+    lang = models.CharField(max_length='2')
  146
+    title = models.CharField(max_length=100)
  147
+    body = models.TextField()
  148
+    abstract = models.CharField(max_length=400, null=True)
  149
+
  150
+    class Meta:
  151
+        unique_together = ('article', 'lang')
  152
+        ordering = ('active_translation__title',)
338  tests/foreign_object/tests.py
... ...
@@ -0,0 +1,338 @@
  1
+import datetime
  2
+from operator import attrgetter
  3
+
  4
+from .models import Country, Person, Group, Membership, Friendship, Article, ArticleTranslation
  5
+from django.test import TestCase
  6
+from django.utils.translation import activate
  7
+from django import forms
  8
+
  9
+class MultiColumnFKTests(TestCase):
  10
+    def setUp(self):
  11
+        # Creating countries
  12
+        self.usa = Country.objects.create(name="United States of America")
  13
+        self.soviet_union = Country.objects.create(name="Soviet Union")
  14
+        Person()
  15
+        # Creating People
  16
+        self.bob = Person()
  17
+        self.bob.name = 'Bob'
  18
+        self.bob.person_country = self.usa
  19
+        self.bob.save()
  20
+        self.jim = Person.objects.create(name='Jim', person_country=self.usa)
  21
+        self.george = Person.objects.create(name='George', person_country=self.usa)
  22
+
  23
+        self.jane = Person.objects.create(name='Jane', person_country=self.soviet_union)
  24
+        self.mark = Person.objects.create(name='Mark', person_country=self.soviet_union)
  25
+        self.sam = Person.objects.create(name='Sam', person_country=self.soviet_union)
  26
+
  27
+        # Creating Groups
  28
+        self.kgb = Group.objects.create(name='KGB', group_country=self.soviet_union)
  29
+        self.cia = Group.objects.create(name='CIA', group_country=self.usa)
  30
+        self.republican = Group.objects.create(name='Republican', group_country=self.usa)
  31
+        self.democrat = Group.objects.create(name='Democrat', group_country=self.usa)
  32
+
  33
+    def test_get_succeeds_on_multicolumn_match(self):
  34
+        # Membership objects have access to their related Person if both
  35
+        # country_ids match between them
  36
+        membership = Membership.objects.create(
  37
+            membership_country_id=self.usa.id, person_id=self.bob.id, group_id=self.cia.id)
  38
+
  39
+        person = membership.person
  40
+        self.assertEqual((person.id, person.name), (self.bob.id, "Bob"))
  41
+
  42
+    def test_get_fails_on_multicolumn_mismatch(self):
  43
+        # Membership objects returns DoesNotExist error when the there is no
  44
+        # Person with the same id and country_id
  45
+        membership = Membership.objects.create(
  46
+            membership_country_id=self.usa.id, person_id=self.jane.id, group_id=self.cia.id)
  47
+
  48
+        self.assertRaises(Person.DoesNotExist, getattr, membership, 'person')
  49
+
  50
+    def test_reverse_query_returns_correct_result(self):
  51
+        # Creating a valid membership because it has the same country has the person
  52
+        Membership.objects.create(
  53
+            membership_country_id=self.usa.id, person_id=self.bob.id, group_id=self.cia.id)
  54
+
  55
+        # Creating an invalid membership because it has a different country has the person
  56
+        Membership.objects.create(
  57
+            membership_country_id=self.soviet_union.id, person_id=self.bob.id,
  58
+            group_id=self.republican.id)
  59
+
  60
+        self.assertQuerysetEqual(
  61
+            self.bob.membership_set.all(), [
  62
+                self.cia.id
  63
+            ],
  64
+            attrgetter("group_id")
  65
+        )
  66
+
  67
+    def test_query_filters_correctly(self):
  68
+
  69
+        # Creating a to valid memberships
  70
+        Membership.objects.create(
  71
+            membership_country_id=self.usa.id, person_id=self.bob.id, group_id=self.cia.id)
  72
+        Membership.objects.create(
  73
+            membership_country_id=self.usa.id, person_id=self.jim.id,
  74
+            group_id=self.cia.id)
  75
+
  76
+        # Creating an invalid membership
  77
+        Membership.objects.create(membership_country_id=self.soviet_union.id,
  78
+                                  person_id=self.george.id, group_id=self.cia.id)
  79
+
  80
+        self.assertQuerysetEqual(
  81
+            Membership.objects.filter(person__name__contains='o'), [
  82
+                self.bob.id
  83
+            ],
  84
+            attrgetter("person_id")
  85
+        )
  86
+
  87
+    def test_reverse_query_filters_correctly(self):
  88
+
  89
+        timemark = datetime.datetime.utcnow()
  90
+        timedelta = datetime.timedelta(days=1)
  91
+
  92
+        # Creating a to valid memberships
  93
+        Membership.objects.create(
  94
+            membership_country_id=self.usa.id, person_id=self.bob.id,
  95
+            group_id=self.cia.id, date_joined=timemark - timedelta)
  96
+        Membership.objects.create(
  97
+            membership_country_id=self.usa.id, person_id=self.jim.id,
  98
+            group_id=self.cia.id, date_joined=timemark + timedelta)
  99
+
  100
+        # Creating an invalid membership
  101
+        Membership.objects.create(
  102
+            membership_country_id=self.soviet_union.id, person_id=self.george.id,
  103
+            group_id=self.cia.id, date_joined=timemark + timedelta)
  104
+
  105
+        self.assertQuerysetEqual(
  106
+            Person.objects.filter(membership__date_joined__gte=timemark), [
  107
+                'Jim'
  108
+            ],
  109
+            attrgetter('name')
  110
+        )
  111
+
  112
+    def test_forward_in_lookup_filters_correctly(self):
  113
+        Membership.objects.create(membership_country_id=self.usa.id, person_id=self.bob.id,
  114
+                                  group_id=self.cia.id)
  115
+        Membership.objects.create(membership_country_id=self.usa.id, person_id=self.jim.id,
  116
+                                  group_id=self.cia.id)
  117
+
  118
+        # Creating an invalid membership
  119
+        Membership.objects.create(
  120
+            membership_country_id=self.soviet_union.id, person_id=self.george.id,
  121
+            group_id=self.cia.id)
  122
+
  123
+        self.assertQuerysetEqual(
  124
+            Membership.objects.filter(person__in=[self.george, self.jim]), [
  125
+                self.jim.id,
  126
+            ],
  127
+            attrgetter('person_id')
  128
+        )
  129
+
  130
+        self.assertQuerysetEqual(
  131
+            Membership.objects.filter(person__in=Person.objects.filter(name='Jim')), [
  132
+                self.jim.id,
  133
+            ],
  134
+            attrgetter('person_id')
  135
+        )
  136
+
  137
+    def test_select_related_foreignkey_forward_works(self):
  138
+        Membership.objects.create(membership_country=self.usa, person=self.bob, group=self.cia)
  139
+        Membership.objects.create(membership_country=self.usa, person=self.jim, group=self.democrat)
  140
+
  141
+        with self.assertNumQueries(1):
  142
+            people = [m.person for m in Membership.objects.select_related('person')]
  143
+
  144
+        normal_people = [m.person for m in Membership.objects.all()]
  145
+        self.assertEqual(people, normal_people)
  146
+
  147
+    def test_prefetch_foreignkey_forward_works(self):
  148
+        Membership.objects.create(membership_country=self.usa, person=self.bob, group=self.cia)
  149
+        Membership.objects.create(membership_country=self.usa, person=self.jim, group=self.democrat)
  150
+
  151
+        with self.assertNumQueries(2):
  152
+            people = [m.person for m in Membership.objects.prefetch_related('person')]
  153
+
  154
+        normal_people = [m.person for m in Membership.objects.all()]
  155
+        self.assertEqual(people, normal_people)
  156
+
  157
+    def test_prefetch_foreignkey_reverse_works(self):
  158
+        Membership.objects.create(membership_country=self.usa, person=self.bob, group=self.cia)
  159
+        Membership.objects.create(membership_country=self.usa, person=self.jim, group=self.democrat)
  160
+        with self.assertNumQueries(2):
  161
+            membership_sets = [list(p.membership_set.all())
  162
+                               for p in Person.objects.prefetch_related('membership_set')]
  163
+
  164
+        normal_membership_sets = [list(p.membership_set.all()) for p in Person.objects.all()]
  165
+        self.assertEqual(membership_sets, normal_membership_sets)
  166
+
  167
+    def test_m2m_through_forward_returns_valid_members(self):
  168
+        # We start out by making sure that the Group 'CIA' has no members.
  169
+        self.assertQuerysetEqual(
  170
+            self.cia.members.all(),
  171
+            []
  172
+        )
  173
+
  174
+        Membership.objects.create(membership_country=self.usa, person=self.bob, group=self.cia)
  175
+        Membership.objects.create(membership_country=self.usa, person=self.jim, group=self.cia)
  176
+
  177
+        # Let's check to make sure that it worked.  Bob and Jim should be members of the CIA.
  178
+
  179
+        self.assertQuerysetEqual(
  180
+            self.cia.members.all(), [
  181
+                'Bob',
  182
+                'Jim'
  183
+            ], attrgetter("name")
  184
+        )
  185
+
  186
+    def test_m2m_through_reverse_returns_valid_members(self):
  187
+        # We start out by making sure that Bob is in no groups.
  188
+        self.assertQuerysetEqual(
  189
+            self.bob.groups.all(),
  190
+            []
  191
+        )
  192
+
  193
+        Membership.objects.create(membership_country=self.usa, person=self.bob, group=self.cia)
  194
+        Membership.objects.create(membership_country=self.usa, person=self.bob,
  195
+                                  group=self.republican)
  196
+
  197
+        # Bob should be in the CIA and a Republican
  198
+        self.assertQuerysetEqual(
  199
+            self.bob.groups.all(), [
  200
+                'CIA',
  201
+                'Republican'
  202
+            ], attrgetter("name")
  203
+        )
  204
+
  205
+    def test_m2m_through_forward_ignores_invalid_members(self):
  206
+        # We start out by making sure that the Group 'CIA' has no members.
  207
+        self.assertQuerysetEqual(
  208
+            self.cia.members.all(),
  209
+            []
  210
+        )
  211
+
  212
+        # Something adds jane to group CIA but Jane is in Soviet Union which isn't CIA's country
  213
+        Membership.objects.create(membership_country=self.usa, person=self.jane, group=self.cia)
  214
+
  215
+        # There should still be no members in CIA
  216
+        self.assertQuerysetEqual(
  217
+            self.cia.members.all(),
  218
+            []
  219
+        )
  220
+
  221
+    def test_m2m_through_reverse_ignores_invalid_members(self):
  222
+        # We start out by making sure that Jane has no groups.
  223
+        self.assertQuerysetEqual(
  224
+            self.jane.groups.all(),
  225
+            []
  226
+        )
  227
+
  228
+        # Something adds jane to group CIA but Jane is in Soviet Union which isn't CIA's country
  229
+        Membership.objects.create(membership_country=self.usa, person=self.jane, group=self.cia)
  230
+
  231
+        # Jane should still not be in any groups
  232
+        self.assertQuerysetEqual(
  233
+            self.jane.groups.all(),
  234
+            []
  235
+        )
  236
+
  237
+    def test_m2m_through_on_self_works(self):
  238
+        self.assertQuerysetEqual(
  239
+            self.jane.friends.all(),
  240
+            []
  241
+        )
  242
+
  243
+        Friendship.objects.create(
  244
+            from_friend_country=self.jane.person_country, from_friend=self.jane,
  245
+            to_friend_country=self.george.person_country, to_friend=self.george)
  246
+
  247
+        self.assertQuerysetEqual(
  248
+            self.jane.friends.all(),
  249
+            ['George'], attrgetter("name")
  250
+        )
  251
+
  252
+    def test_m2m_through_on_self_ignores_mismatch_columns(self):
  253
+        self.assertQuerysetEqual(self.jane.friends.all(), [])
  254
+
  255
+        # Note that we use ids instead of instances. This is because instances on ForeignObject
  256
+        # properties will set all related field off of the given instance
  257
+        Friendship.objects.create(
  258
+            from_friend_id=self.jane.id, to_friend_id=self.george.id,
  259
+            to_friend_country_id=self.jane.person_country_id,
  260
+            from_friend_country_id=self.george.person_country_id)
  261
+
  262
+        self.assertQuerysetEqual(self.jane.friends.all(), [])
  263
+
  264
+    def test_prefetch_related_m2m_foward_works(self):
  265
+        Membership.objects.create(membership_country=self.usa, person=self.bob, group=self.cia)
  266
+        Membership.objects.create(membership_country=self.usa, person=self.jim, group=self.democrat)
  267
+
  268
+        with self.assertNumQueries(2):
  269
+            members_lists = [list(g.members.all())
  270
+                             for g in Group.objects.prefetch_related('members')]
  271
+
  272
+        normal_members_lists = [list(g.members.all()) for g in Group.objects.all()]
  273
+        self.assertEqual(members_lists, normal_members_lists)
  274
+
  275
+    def test_prefetch_related_m2m_reverse_works(self):
  276
+        Membership.objects.create(membership_country=self.usa, person=self.bob, group=self.cia)
  277
+        Membership.objects.create(membership_country=self.usa, person=self.jim, group=self.democrat)
  278
+
  279
+        with self.assertNumQueries(2):
  280
+            groups_lists = [list(p.groups.all()) for p in Person.objects.prefetch_related('groups')]
  281
+
  282
+        normal_groups_lists = [list(p.groups.all()) for p in Person.objects.all()]
  283
+        self.assertEqual(groups_lists, normal_groups_lists)
  284
+
  285
+    def test_translations(self):
  286
+        activate('fi')
  287
+        a1 = Article.objects.create(pub_date=datetime.date.today())
  288
+        at1_fi = ArticleTranslation(article=a1, lang='fi', title='Otsikko', body='Diipadaapa')
  289
+        at1_fi.save()
  290
+        at2_en = ArticleTranslation(article=a1, lang='en', title='Title', body='Lalalalala')
  291
+        at2_en.save()
  292
+        with self.assertNumQueries(1):
  293
+            fetched = Article.objects.select_related('active_translation').get(
  294
+                active_translation__title='Otsikko')
  295
+            self.assertTrue(fetched.active_translation.title == 'Otsikko')
  296
+        a2 = Article.objects.create(pub_date=datetime.date.today())
  297
+        at2_fi = ArticleTranslation(article=a2, lang='fi', title='Atsikko', body='Diipadaapa',
  298
+                                    abstract='dipad')
  299
+        at2_fi.save()
  300
+        a3 = Article.objects.create(pub_date=datetime.date.today())
  301
+        at3_en = ArticleTranslation(article=a3, lang='en', title='A title', body='lalalalala',
  302
+                                    abstract='lala')
  303
+        at3_en.save()
  304
+        # Test model initialization with active_translation field.
  305
+        a3 = Article(id=a3.id, pub_date=a3.pub_date, active_translation=at3_en)
  306
+        a3.save()
  307
+        self.assertEqual(
  308
+            list(Article.objects.filter(active_translation__abstract=None)),
  309
+            [a1, a3])
  310
+        self.assertEqual(
  311
+            list(Article.objects.filter(active_translation__abstract=None,
  312
+                                        active_translation__pk__isnull=False)),
  313
+            [a1])
  314
+        activate('en')
  315
+        self.assertEqual(
  316
+            list(Article.objects.filter(active_translation__abstract=None)),
  317
+            [a1, a2])
  318
+
  319
+class FormsTests(TestCase):
  320
+    # ForeignObjects should not have any form fields, currently the user needs
  321
+    # to manually deal with the foreignobject relation.
  322
+    class ArticleForm(forms.ModelForm):
  323
+        class Meta:
  324
+            model = Article
  325
+
  326
+    def test_foreign_object_form(self):
  327
+        # A very crude test checking that the non-concrete fields do not get form fields.
  328
+        form = FormsTests.ArticleForm()
  329
+        self.assertIn('id_pub_date', form.as_table())
  330
+        self.assertNotIn('active_translation', form.as_table())
  331
+        form = FormsTests.ArticleForm(data={'pub_date': str(datetime.date.today())})
  332
+        self.assertTrue(form.is_valid())
  333
+        a = form.save()
  334
+        self.assertEqual(a.pub_date, datetime.date.today())
  335
+        form = FormsTests.ArticleForm(instance=a, data={'pub_date': '2013-01-01'})
  336
+        a2 = form.save()
  337
+        self.assertEqual(a.pk, a2.pk)
  338
+        self.assertEqual(a2.pub_date, datetime.date(2013, 1, 1))

0 notes on commit 266de5f

Please sign in to comment.
Something went wrong with that request. Please try again.