Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixed #373 -- Added CompositePrimaryKey-based Meta.primary_key. #18056

Open
wants to merge 72 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
72 commits
Select commit Hold shift + click to select a range
4be1c68
Fixed #373 -- Added CompositeField-based Meta.primary_key.
csirmazbendeguz Apr 7, 2024
cee213d
Fixed #373 -- Add field to error message, simplify tuple operation er…
csirmazbendeguz May 8, 2024
e2d8868
Fixed #373 -- Rename CompositeField to CompositePrimaryKey
csirmazbendeguz May 8, 2024
259605a
Fixed #373 -- Add __len__ to CompositePrimaryKey
csirmazbendeguz May 8, 2024
9e01443
Fixed #373 -- Add some docs
csirmazbendeguz May 9, 2024
fa4af00
Fixed #373 -- Fix docs formatting
csirmazbendeguz May 9, 2024
8354bd0
Fixed #373 -- Add more docs
csirmazbendeguz May 9, 2024
d20f8e0
Fixed #373 -- Fix docs
csirmazbendeguz May 9, 2024
dcd78fc
Fixed #373 -- Fix integrity error tests
csirmazbendeguz May 9, 2024
3576dec
Fixed #373 -- Remove unnecessary asserts
csirmazbendeguz May 9, 2024
1ef76e4
Fixed #373 -- Make a test simpler
csirmazbendeguz May 9, 2024
3d0dcf3
Fixed #373 -- Make get tests better
csirmazbendeguz May 9, 2024
6a2bb92
Fixed #373 -- Fix test
csirmazbendeguz May 9, 2024
a2ec10c
Fixed #373 -- Address create tests comments
csirmazbendeguz May 10, 2024
3071ad9
Fixed #373 -- Address delete tests comments
csirmazbendeguz May 10, 2024
b7179b6
Fixed #373 -- Address filter tests comments
csirmazbendeguz May 10, 2024
cb4f300
Fixed #373 -- Add test for ordering comments
csirmazbendeguz May 10, 2024
6922337
Fixed #373 -- Fix ordering tes
csirmazbendeguz May 10, 2024
b7b9580
Fixed #373 - Simplify tests
csirmazbendeguz May 11, 2024
9b17831
Fixed #373 - Simplify tests
csirmazbendeguz May 11, 2024
6285cff
Fixed #373 - Don't check for CompositePrimaryKey directly in SQL comp…
csirmazbendeguz May 11, 2024
22d6c9d
Fixed #373 - Simplify statement
csirmazbendeguz May 11, 2024
ad51da4
Fixed #373 - Use MultiColSource instead of Cols
csirmazbendeguz May 11, 2024
9c9e57d
Fixed #373 - Add system checks
csirmazbendeguz May 13, 2024
c8f796e
Fixed #373 - Add CompositeValue class
csirmazbendeguz May 13, 2024
26fb669
Fixed #373 - Fix test
csirmazbendeguz May 14, 2024
87b1a23
Fixed #373 - Test related names
csirmazbendeguz May 14, 2024
389daa9
Fixed #373 - Add test for composite pk attnames
csirmazbendeguz May 14, 2024
2515066
Fixed #373 - Add test for composite pk state
csirmazbendeguz May 14, 2024
350c58d
Fixed #373 - Fix error message
csirmazbendeguz May 16, 2024
220b3ed
Fixed #373 - Simplify checking if pk is set
csirmazbendeguz May 16, 2024
b9a8143
Fixed #373 - Fix error message
csirmazbendeguz May 16, 2024
141b27d
Fixed #373 - Fix error message
csirmazbendeguz May 16, 2024
926822f
Fixed #373 - Add explanatory comment
csirmazbendeguz May 16, 2024
ac29218
Fixed #373 - Add explanatory comment
csirmazbendeguz May 16, 2024
0b194eb
Fixed #373 - Minor refactoring in test
csirmazbendeguz May 16, 2024
b6727d2
Fixed #373 - Small refactor for select mask
csirmazbendeguz May 16, 2024
21d48d5
Fixed #373 - Revert auto field changes
csirmazbendeguz May 16, 2024
b839b2b
Fixed #373 - A minor improvement to sql update compiler
csirmazbendeguz May 16, 2024
006736c
Fixed #373 - Fix source expressions
csirmazbendeguz May 16, 2024
3bf4004
Fixed #373 - Simplify error messages
csirmazbendeguz May 16, 2024
90a8d43
Fixed #373 - Fix linter error
csirmazbendeguz May 16, 2024
7bc3f40
Fixed #373 - Fix set source expressions
csirmazbendeguz May 17, 2024
f4db656
Fixed #373 - Don't ignore params in as_sql
csirmazbendeguz May 17, 2024
e7deac6
Fixed #373 - Fix ordering issue
csirmazbendeguz May 17, 2024
e15f55d
Fixed #373 - Revert using MultiColSource
csirmazbendeguz May 19, 2024
79bf5e8
Fixed #373 - Implement gt, gte lookups for tuples
csirmazbendeguz May 19, 2024
92f2807
Fixed #373 - Implement lt, lte lookups for tuples
csirmazbendeguz May 19, 2024
c5c5239
Fixed #373 - Refactor in lookup
csirmazbendeguz May 20, 2024
8e408b6
Fixed #373 - Refactor tests
csirmazbendeguz May 20, 2024
69143d9
Fixed #373 - Improve error messages
csirmazbendeguz May 20, 2024
586434c
Fixed #373 - Fix subTest message
csirmazbendeguz May 20, 2024
0478d46
Fixed #373 - Add lookup error tests
csirmazbendeguz May 21, 2024
23e0e0b
Fixed #373 - Add null check for primary key fields
csirmazbendeguz May 21, 2024
a9ef232
Fixed #373 - Raise value error instead of check error for Meta.primar…
csirmazbendeguz May 22, 2024
694fcc8
Fixed #373 - Add test for checking foreign object's unique target
csirmazbendeguz May 22, 2024
b49ea71
Fixed #373 - Simplify checks
csirmazbendeguz May 23, 2024
9fbb766
Fixed #373 - Add pk_fields to Options
csirmazbendeguz May 23, 2024
96156dd
Fixed #373 - Adjust indentation
csirmazbendeguz May 23, 2024
13c0053
Fixed #373 - Remove try catch checking pk_set
csirmazbendeguz May 23, 2024
9ce6781
Fixed #373 - Remove try catch from function
csirmazbendeguz May 23, 2024
2f24f77
Fixed #373 - Rename to _unnest_composite_fields
csirmazbendeguz May 23, 2024
2c8d38c
Fixed #373 - Refactor is_set
csirmazbendeguz May 23, 2024
cff46d5
Fixed #373 - Add docs
csirmazbendeguz May 23, 2024
39a2a09
Fixed #373 - Add duplicate pk check
csirmazbendeguz May 23, 2024
2478e14
Fixed #373 - Throw ValueError for duplicates instead of check error
csirmazbendeguz May 23, 2024
76191b5
Fixed #373 - Add docs about migration
csirmazbendeguz May 23, 2024
208a457
Fixed #373 - Refactor tuple lookups to classes
csirmazbendeguz May 23, 2024
b74f3ef
Fixed #373 - Revert new line
csirmazbendeguz May 23, 2024
70fd4be
Fixed #373 - Make variable names more consistent
csirmazbendeguz May 23, 2024
4caee01
Fixed #373 - Add some comments to lookups
csirmazbendeguz May 23, 2024
272184d
Fixed #373 - Add validation error to clean
csirmazbendeguz May 24, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
13 changes: 13 additions & 0 deletions django/db/backends/base/schema.py
Expand Up @@ -105,6 +105,7 @@ class BaseDatabaseSchemaEditor:
sql_check_constraint = "CHECK (%(check)s)"
sql_delete_constraint = "ALTER TABLE %(table)s DROP CONSTRAINT %(name)s"
sql_constraint = "CONSTRAINT %(name)s %(constraint)s"
sql_pk_constraint = "PRIMARY KEY (%(columns)s)"

sql_create_check = "ALTER TABLE %(table)s ADD CONSTRAINT %(name)s CHECK (%(check)s)"
sql_delete_check = sql_delete_constraint
Expand Down Expand Up @@ -268,6 +269,13 @@ def table_sql(self, model):
constraint.constraint_sql(model, self)
for constraint in model._meta.constraints
]

# If the model defines Meta.primary_key, add the primary key constraint
# to the table definition.
# It's expected primary_key=True isn't set on any fields (see E042).
if model._meta.primary_key:
constraints.append(self._pk_constraint_sql(model._meta.primary_key))

sql = self.sql_create_table % {
"table": self.quote_name(model._meta.db_table),
"definition": ", ".join(
Expand Down Expand Up @@ -1967,6 +1975,11 @@ def _constraint_names(
result.append(name)
return result

def _pk_constraint_sql(self, fields):
return self.sql_pk_constraint % {
"columns": ", ".join(self.quote_name(field) for field in fields)
}

def _delete_primary_key(self, model, strict=False):
constraint_names = self._constraint_names(model, primary_key=True)
if strict and len(constraint_names) != 1:
Expand Down
32 changes: 31 additions & 1 deletion django/db/models/base.py
Expand Up @@ -30,6 +30,7 @@
from django.db.models.constants import LOOKUP_SEP
from django.db.models.deletion import CASCADE, Collector
from django.db.models.expressions import DatabaseDefault
from django.db.models.fields.composite import is_pk_set
from django.db.models.fields.related import (
ForeignObjectRel,
OneToOneField,
Expand Down Expand Up @@ -1080,7 +1081,7 @@ def _save_table(
if pk_val is None:
pk_val = meta.pk.get_pk_value_on_save(self)
setattr(self, meta.pk.attname, pk_val)
pk_set = pk_val is not None
csirmazbendeguz marked this conversation as resolved.
Show resolved Hide resolved
pk_set = is_pk_set(pk_val)
if not pk_set and (force_update or update_fields):
raise ValueError("Cannot force an update in save() with no primary key.")
updated = False
Expand Down Expand Up @@ -1686,6 +1687,7 @@ def check(cls, **kwargs):
*cls._check_constraints(databases),
*cls._check_default_pk(),
*cls._check_db_table_comment(databases),
*cls._check_composite_pk(),
]

return errors
Expand All @@ -1694,6 +1696,9 @@ def check(cls, **kwargs):
def _check_default_pk(cls):
if (
not cls._meta.abstract
# If the model defines Meta.primary_key, the check should be skipped,
# since there's no default primary key.
and not cls._meta.primary_key
and cls._meta.pk.auto_created
and
# Inherited PKs are checked in parents models.
Expand Down Expand Up @@ -1722,6 +1727,26 @@ def _check_default_pk(cls):
]
return []

@classmethod
def _check_composite_pk(cls):
errors = []

if cls._meta.primary_key is None:
return errors

for field in cls._meta.fields:
if field.primary_key:
errors.append(
checks.Error(
"%s may not set primary_key=True if Meta.primary_key "
"is defined." % (field.name,),
obj=cls,
id="models.E042",
)
)

return errors

@classmethod
def _check_db_table_comment(cls, databases):
if not cls._meta.db_table_comment:
Expand Down Expand Up @@ -1842,6 +1867,11 @@ def _check_m2m_through_same_relationship(cls):
@classmethod
def _check_id_field(cls):
"""Check if `id` field is a primary key."""
# If the model defines Meta.primary_key, the check should be skipped,
# since primary_key=True can't be set on any fields (including `id`).
if cls._meta.primary_key:
return []

fields = [
f for f in cls._meta.local_fields if f.name == "id" and f != cls._meta.pk
]
Expand Down
7 changes: 6 additions & 1 deletion django/db/models/fields/__init__.py
Expand Up @@ -2794,6 +2794,11 @@ def check(self, **kwargs):
]

def _check_primary_key(self):
# If the model defines Meta.primary_key, primary_key=True can't be set on
# any field (including AutoFields).
if self.model._meta.primary_key:
return []

if not self.primary_key:
return [
checks.Error(
Expand All @@ -2808,7 +2813,7 @@ def _check_primary_key(self):
def deconstruct(self):
name, path, args, kwargs = super().deconstruct()
del kwargs["blank"]
kwargs["primary_key"] = True
kwargs["primary_key"] = self.primary_key
return name, path, args, kwargs

def validate(self, value, model_instance):
Expand Down
179 changes: 179 additions & 0 deletions django/db/models/fields/composite.py
@@ -0,0 +1,179 @@
from django.core.exceptions import FieldDoesNotExist
from django.db.models import Field
from django.db.models.expressions import Col, Expression
from django.db.models.lookups import Exact, In
from django.db.models.signals import class_prepared
from django.utils.functional import cached_property


class TupleExact(Exact):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if there might be a way to define a TupleLookupMixin that can be mixed with Exact, In, GreaterThan, and others to do the right thing by default.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I could remove the TupleExact class and handle the case of comparing tuples inside Exact, if that's what you're suggesting? It would be a bit more elegant I suppose.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So I looked into it and I started implementing the other lookups. I don't think it's possible to define a mixin that does the right thing by default, but it's possible to make some abstractions to reduce duplication.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

gt, gte, lt, lte have been implemented. Also, TupleLookupMixin has been added. Please look into it when you have some time. It's not exactly what you wanted, there's no default behavior as each lookup has their own complexity, but I think this is the only way to implement this.

def get_prep_lookup(self):
if not isinstance(self.lhs, Cols):
raise ValueError(
"The left-hand side of the 'exact' lookup must be an instance of Cols"
csirmazbendeguz marked this conversation as resolved.
Show resolved Hide resolved
)
if not isinstance(self.rhs, (tuple, list)):
raise ValueError(
"The right-hand side of the 'exact' lookup must be a tuple or a list"
)
if len(self.lhs) != len(self.rhs):
raise ValueError(
"The left-hand side and right-hand side of the 'exact' lookup must "
"have the same number of elements"
)

return super().get_prep_lookup()

def as_sql(self, compiler, connection):
from django.db.models.sql.where import AND, WhereNode

cols = self.lhs.get_source_expressions()
exprs = [Exact(col, val) for col, val in zip(cols, self.rhs)]

return compiler.compile(WhereNode(exprs, connector=AND))


class TupleIn(In):
def get_prep_lookup(self):
if not isinstance(self.lhs, Cols):
raise ValueError(
"The left-hand side of the 'in' lookup must be an instance of Cols"
csirmazbendeguz marked this conversation as resolved.
Show resolved Hide resolved
)
if not isinstance(self.rhs, (tuple, list)):
raise ValueError(
"The right-hand side of the 'in' lookup must be a tuple or a list"
)
if not all(isinstance(vals, (tuple, list)) for vals in self.rhs):
raise ValueError(
"The right-hand side of the 'in' lookup must be a set of "
csirmazbendeguz marked this conversation as resolved.
Show resolved Hide resolved
"tuples or lists"
)
if not all(len(self.lhs) == len(vals) for vals in self.rhs):
raise ValueError(
"The left-hand side and right-hand side of the 'in' lookup must "
"have the same number of elements"
)

return super().get_prep_lookup()

def as_sql(self, compiler, connection):
from django.db.models.sql.where import AND, OR, WhereNode
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Have you explored overriding resolve_expression to build Q objects instead?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have never seen Q objects used in as_sql, I don't think we're supposed to use them here, are we?


exprs = []
cols = self.lhs.get_source_expressions()

for vals in self.rhs:
exprs.append(
WhereNode(
[Exact(col, val) for col, val in zip(cols, vals)], connector=AND
)
)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It'd be nice to eventually use row level comparison instead for backends that support it.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I could work on this in the next PR, however, I'm not sure if there's any real benefit to it other than generating nicer SQL.


return compiler.compile(WhereNode(exprs, connector=OR))


class Cols(Expression):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if there is an opportunity to merge this TuplesIn, Cols, and friends logic with MultiColSource so it's less of an 👽. They both do very similar thing.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @charettes , I'll need to look into this, I wasn't aware.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I merged Cols with MultiColSource (ad51da4) however, I'm not sure this is correct.

As far as I understand, MultiColSource was meant to represent columns in a JOIN, and as such, it has a sources field. Cols, on the other hand, was meant to represent a list of columns and it doesn't need a sources field. WDYT?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I reverted back to Cols. Please resolve if you agree.

def __init__(self, alias, targets, output_field):
super().__init__(output_field=output_field)
self.alias, self.targets = alias, targets

def get_source_expressions(self):
return [Col(self.alias, target) for target in self.targets]

def set_source_expressions(self, exprs):
assert all(isinstance(expr, Col) for expr in exprs)
assert len(exprs) == len(self.targets)
csirmazbendeguz marked this conversation as resolved.
Show resolved Hide resolved

def as_sql(self, compiler, connection):
sqls = []
cols = self.get_source_expressions()

for col in cols:
sql, _ = col.as_sql(compiler, connection)
sqls.append(sql)

return ", ".join(sqls), []

def __iter__(self):
return iter(self.get_source_expressions())
csirmazbendeguz marked this conversation as resolved.
Show resolved Hide resolved

def __len__(self):
return len(self.targets)


def is_pk_not_set(pk):
return pk is None or (isinstance(pk, tuple) and any(f is None for f in pk))


def is_pk_set(pk):
return not is_pk_not_set(pk)


class CompositeAttribute:
def __init__(self, field):
self.field = field

def __get__(self, instance, cls=None):
return tuple(
getattr(instance, field_name) for field_name in self.field.field_names
)

def __set__(self, instance, values):
if values is None:
values = (None,) * len(self.field.field_names)

for field_name, value in zip(self.field.field_names, values):
setattr(instance, field_name, value)
csirmazbendeguz marked this conversation as resolved.
Show resolved Hide resolved


class CompositePrimaryKey(Field):
descriptor_class = CompositeAttribute

def __init__(self, *args, **kwargs):
kwargs["db_column"] = None
kwargs["editable"] = False
super().__init__(**kwargs)
self.field_names = args
self.fields = None

def contribute_to_class(self, cls, name, **_):
super().contribute_to_class(cls, name, private_only=True)
cls._meta.pk = self
csirmazbendeguz marked this conversation as resolved.
Show resolved Hide resolved
setattr(cls, self.attname, self.descriptor_class(self))

def get_attname_column(self):
return self.get_attname(), self.db_column

def __iter__(self):
return iter(self.fields)
csirmazbendeguz marked this conversation as resolved.
Show resolved Hide resolved

def __len__(self):
return len(self.fields)

@cached_property
def cached_col(self):
return Cols(self.model._meta.db_table, self.fields, self)

def get_col(self, alias, output_field=None):
return self.cached_col
csirmazbendeguz marked this conversation as resolved.
Show resolved Hide resolved

def get_lookup(self, lookup_name):
if lookup_name == "exact":
return TupleExact
elif lookup_name == "in":
return TupleIn

return super().get_lookup(lookup_name)
csirmazbendeguz marked this conversation as resolved.
Show resolved Hide resolved


def resolve_fields(*args, **kwargs):
meta = kwargs["sender"]._meta
for field in meta.private_fields:
if isinstance(field, CompositePrimaryKey) and field.fields is None:
try:
field.fields = tuple(meta.get_field(name) for name in field.field_names)
except FieldDoesNotExist:
continue
csirmazbendeguz marked this conversation as resolved.
Show resolved Hide resolved


class_prepared.connect(resolve_fields)
10 changes: 10 additions & 0 deletions django/db/models/fields/related.py
Expand Up @@ -615,6 +615,16 @@ def _check_unique_target(self):
if not self.foreign_related_fields:
return []

# If a model defines Meta.primary_key and a foreign key refers to it,
# the check should be skipped (since primary keys are unique).
pk = self.remote_field.model._meta.primary_key
if pk:
pk = set(pk)
if pk == {f.attname for f in self.foreign_related_fields}:
return []
elif pk == {f.name for f in self.foreign_related_fields}:
return []
csirmazbendeguz marked this conversation as resolved.
Show resolved Hide resolved

has_unique_constraint = any(
rel_field.unique for rel_field in self.foreign_related_fields
)
Expand Down
8 changes: 7 additions & 1 deletion django/db/models/options.py
Expand Up @@ -7,6 +7,7 @@
from django.core.exceptions import FieldDoesNotExist, ImproperlyConfigured
from django.db import connections
from django.db.models import AutoField, Manager, OrderWrt, UniqueConstraint
from django.db.models.fields.composite import CompositePrimaryKey
from django.db.models.query_utils import PathInfo
from django.utils.datastructures import ImmutableList, OrderedSet
from django.utils.functional import cached_property
Expand All @@ -24,6 +25,7 @@
)

DEFAULT_NAMES = (
"primary_key",
"verbose_name",
"verbose_name_plural",
"db_table",
Expand Down Expand Up @@ -106,6 +108,7 @@ def __init__(self, meta, app_label=None):
self.base_manager_name = None
self.default_manager_name = None
self.model_name = None
self.primary_key = None
self.verbose_name = None
self.verbose_name_plural = None
self.db_table = ""
Expand Down Expand Up @@ -296,7 +299,10 @@ def _prepare(self, model):
self.order_with_respect_to = None

if self.pk is None:
if self.parents:
if self.primary_key:
pk = CompositePrimaryKey(*self.primary_key)
model.add_to_class("primary_key", pk)
elif self.parents:
# Promote the first parent link in lieu of adding yet another
# field.
field = next(iter(self.parents.values()))
Expand Down
5 changes: 4 additions & 1 deletion django/db/models/query.py
Expand Up @@ -24,6 +24,7 @@
from django.db.models.constants import LOOKUP_SEP, OnConflict
from django.db.models.deletion import Collector
from django.db.models.expressions import Case, F, Value, When
from django.db.models.fields.composite import is_pk_not_set
from django.db.models.functions import Cast, Trunc
from django.db.models.query_utils import FilteredRelation, Q
from django.db.models.sql.constants import CURSOR, GET_ITERATOR_CHUNK_SIZE
Expand Down Expand Up @@ -813,7 +814,9 @@ def bulk_create(
objs = list(objs)
self._prepare_for_bulk_create(objs)
with transaction.atomic(using=self.db, savepoint=False):
objs_with_pk, objs_without_pk = partition(lambda o: o.pk is None, objs)
objs_with_pk, objs_without_pk = partition(
lambda o: is_pk_not_set(o.pk), objs
)
if objs_with_pk:
returned_columns = self._batched_insert(
objs_with_pk,
Expand Down