From b5a8b10c61e5778fb1cd3bdb30a5aebaea5acc40 Mon Sep 17 00:00:00 2001 From: Tom Sparrow <793763+sparrowt@users.noreply.github.com> Date: Sat, 18 Apr 2020 14:06:15 +0100 Subject: [PATCH 01/10] Fix #45 correctly reinstate nullable unique constraints (#47) Ensure that the unique/unique_together constraint is reinstated exactly the same as the one that was dropped earlier in `_alter_field` This fixes both cases described on the bug report, that is when `AlterField` modifies a column which is: 1) individually `unique=True` as well as `null=True` 2) in a `unique_together` where at least 1 column is nullable --- sql_server/pyodbc/schema.py | 23 +++++--- .../0002_test_unique_nullable_part1.py | 1 + .../0003_test_unique_nullable_part2.py | 1 + ...4_test_issue45_unique_type_change_part1.py | 32 +++++++++++ ...5_test_issue45_unique_type_change_part2.py | 33 ++++++++++++ testapp/models.py | 16 ++++++ testapp/tests/test_constraints.py | 54 +++++++++++++++++++ testapp/tests/test_expressions.py | 18 +------ 8 files changed, 155 insertions(+), 23 deletions(-) create mode 100644 testapp/migrations/0004_test_issue45_unique_type_change_part1.py create mode 100644 testapp/migrations/0005_test_issue45_unique_type_change_part2.py create mode 100644 testapp/tests/test_constraints.py diff --git a/sql_server/pyodbc/schema.py b/sql_server/pyodbc/schema.py index 2bd2dcc5..cf75b27b 100644 --- a/sql_server/pyodbc/schema.py +++ b/sql_server/pyodbc/schema.py @@ -454,21 +454,30 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, # True | True | True | False if (not old_field.db_index or old_field.unique) and new_field.db_index and not new_field.unique: self.execute(self._create_index_sql(model, [new_field])) - # Restore an index, SQL Server requires explicit restoration + + # Restore indexes & unique constraints deleted above, SQL Server requires explicit restoration if (old_type != new_type or (old_field.null and not new_field.null)) and ( old_field.column == new_field.column ): - unique_columns = [] + # Restore unique constraints + # Note: if nullable they are implemented via an explicit filtered UNIQUE INDEX (not CONSTRAINT) + # in order to get ANSI-compliant NULL behaviour (i.e. NULL != NULL, multiple are allowed) if old_field.unique and new_field.unique: - unique_columns.append([old_field.column]) + if new_field.null: + self.execute( + self._create_index_sql( + model, [old_field], sql=self.sql_create_unique_null, suffix="_uniq" + ) + ) + else: + self.execute(self._create_unique_sql(model, columns=[old_field.column])) else: for fields in model._meta.unique_together: columns = [model._meta.get_field(field).column for field in fields] if old_field.column in columns: - unique_columns.append(columns) - if unique_columns: - for columns in unique_columns: - self.execute(self._create_unique_sql(model, columns)) + condition = ' AND '.join(["[%s] IS NOT NULL" % col for col in columns]) + self.execute(self._create_unique_sql(model, columns, condition=condition)) + # Restore indexes index_columns = [] if old_field.db_index and new_field.db_index: index_columns.append([old_field]) diff --git a/testapp/migrations/0002_test_unique_nullable_part1.py b/testapp/migrations/0002_test_unique_nullable_part1.py index 1c0e48d2..33ab86a6 100644 --- a/testapp/migrations/0002_test_unique_nullable_part1.py +++ b/testapp/migrations/0002_test_unique_nullable_part1.py @@ -8,6 +8,7 @@ class Migration(migrations.Migration): ] operations = [ + # Issue #38 test prep # Create with a field that is unique *and* nullable so it is implemented with a filtered unique index. migrations.CreateModel( name='TestUniqueNullableModel', diff --git a/testapp/migrations/0003_test_unique_nullable_part2.py b/testapp/migrations/0003_test_unique_nullable_part2.py index d6fc61e0..ade35429 100644 --- a/testapp/migrations/0003_test_unique_nullable_part2.py +++ b/testapp/migrations/0003_test_unique_nullable_part2.py @@ -8,6 +8,7 @@ class Migration(migrations.Migration): ] operations = [ + # Issue #38 test # Now remove the null=True to check this transition is correctly handled. migrations.AlterField( model_name='testuniquenullablemodel', diff --git a/testapp/migrations/0004_test_issue45_unique_type_change_part1.py b/testapp/migrations/0004_test_issue45_unique_type_change_part1.py new file mode 100644 index 00000000..2f3b9fba --- /dev/null +++ b/testapp/migrations/0004_test_issue45_unique_type_change_part1.py @@ -0,0 +1,32 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('testapp', '0003_test_unique_nullable_part2'), + ] + + # Issue #45 test prep + operations = [ + # for case 1: + migrations.AddField( + model_name='testuniquenullablemodel', + name='x', + field=models.CharField(max_length=10, null=True, unique=True), + ), + + # for case 2: + migrations.CreateModel( + name='TestNullableUniqueTogetherModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('a', models.CharField(max_length=50, null=True)), + ('b', models.CharField(max_length=50)), + ('c', models.CharField(max_length=50)), + ], + options={ + 'unique_together': {('a', 'b')}, + }, + ), + ] diff --git a/testapp/migrations/0005_test_issue45_unique_type_change_part2.py b/testapp/migrations/0005_test_issue45_unique_type_change_part2.py new file mode 100644 index 00000000..a938fe2a --- /dev/null +++ b/testapp/migrations/0005_test_issue45_unique_type_change_part2.py @@ -0,0 +1,33 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('testapp', '0004_test_issue45_unique_type_change_part1'), + ] + + # Issue #45 test + operations = [ + # Case 1: changing max_length changes the column type - the filtered UNIQUE INDEX which implements + # the nullable unique constraint, should be correctly reinstated after this change of column type + # (see also the specific unit test which checks that multiple rows with NULL are allowed) + migrations.AlterField( + model_name='testuniquenullablemodel', + name='x', + field=models.CharField(max_length=11, null=True, unique=True), + ), + + # Case 2: the filtered UNIQUE INDEX implementing the partially nullable `unique_together` constraint + # should be correctly reinstated after this column type change + migrations.AlterField( + model_name='testnullableuniquetogethermodel', + name='a', + field=models.CharField(max_length=51, null=True), + ), + # ...similarly adding another field to the `unique_together` should preserve the constraint correctly + migrations.AlterUniqueTogether( + name='testnullableuniquetogethermodel', + unique_together={('a', 'b', 'c')}, + ), + ] diff --git a/testapp/models.py b/testapp/models.py index 503d81ce..7b993165 100644 --- a/testapp/models.py +++ b/testapp/models.py @@ -44,7 +44,23 @@ def __str__(self): class TestUniqueNullableModel(models.Model): + # Issue #38: # This field started off as unique=True *and* null=True so it is implemented with a filtered unique index # Then it is made non-nullable by a subsequent migration, to check this is correctly handled (the index # should be dropped, then a normal unique constraint should be added, now that the column is not nullable) test_field = models.CharField(max_length=100, unique=True) + + # Issue #45 (case 1) + # Field used for testing changing the 'type' of a field that's both unique & nullable + x = models.CharField(max_length=11, null=True, unique=True) + + +class TestNullableUniqueTogetherModel(models.Model): + class Meta: + unique_together = (('a', 'b', 'c'),) + + # Issue #45 (case 2) + # Fields used for testing changing the 'type of a field that is in a `unique_together` + a = models.CharField(max_length=51, null=True) + b = models.CharField(max_length=50) + c = models.CharField(max_length=50) diff --git a/testapp/tests/test_constraints.py b/testapp/tests/test_constraints.py new file mode 100644 index 00000000..523a2c85 --- /dev/null +++ b/testapp/tests/test_constraints.py @@ -0,0 +1,54 @@ +from django.db.utils import IntegrityError +from django.test import TestCase, skipUnlessDBFeature + +from ..models import ( + Author, Editor, Post, + TestUniqueNullableModel, TestNullableUniqueTogetherModel, +) + + +@skipUnlessDBFeature('supports_nullable_unique_constraints') +class TestNullableUniqueColumn(TestCase): + def test_multiple_nulls(self): + # Issue #45 (case 1) - after field `x` has had its type changed, the filtered UNIQUE + # INDEX which is implementing the nullable unique constraint should still be correctly + # in place - i.e. allowing multiple NULLs but still enforcing uniqueness of non-NULLs + + # Allowed + TestUniqueNullableModel.objects.create(x=None, test_field='randomness') + TestUniqueNullableModel.objects.create(x=None, test_field='doesntmatter') + + # Disallowed + TestUniqueNullableModel.objects.create(x="foo", test_field='irrelevant') + with self.assertRaises(IntegrityError): + TestUniqueNullableModel.objects.create(x="foo", test_field='nonsense') + + +@skipUnlessDBFeature('supports_partially_nullable_unique_constraints') +class TestPartiallyNullableUniqueTogether(TestCase): + def test_partially_nullable(self): + # Check basic behaviour of `unique_together` where at least 1 of the columns is nullable + + # It should be possible to have 2 rows both with NULL `alt_editor` + author = Author.objects.create(name="author") + Post.objects.create(title="foo", author=author) + Post.objects.create(title="foo", author=author) + + # But `unique_together` is still enforced for non-NULL values + editor = Editor.objects.create(name="editor") + Post.objects.create(title="foo", author=author, alt_editor=editor) + with self.assertRaises(IntegrityError): + Post.objects.create(title="foo", author=author, alt_editor=editor) + + def test_after_type_change(self): + # Issue #45 (case 2) - after one of the fields in the `unique_together` has had its + # type changed in a migration, the constraint should still be correctly enforced + + # Multiple rows with a=NULL are considered different + TestNullableUniqueTogetherModel.objects.create(a=None, b='bbb', c='ccc') + TestNullableUniqueTogetherModel.objects.create(a=None, b='bbb', c='ccc') + + # Uniqueness still enforced for non-NULL values + TestNullableUniqueTogetherModel.objects.create(a='aaa', b='bbb', c='ccc') + with self.assertRaises(IntegrityError): + TestNullableUniqueTogetherModel.objects.create(a='aaa', b='bbb', c='ccc') diff --git a/testapp/tests/test_expressions.py b/testapp/tests/test_expressions.py index 720c542b..90623753 100644 --- a/testapp/tests/test_expressions.py +++ b/testapp/tests/test_expressions.py @@ -3,10 +3,9 @@ from django import VERSION from django.db.models import IntegerField from django.db.models.expressions import Case, Exists, OuterRef, Subquery, Value, When -from django.db.utils import IntegrityError -from django.test import TestCase, skipUnlessDBFeature +from django.test import TestCase -from ..models import Author, Comment, Editor, Post +from ..models import Author, Comment, Post DJANGO3 = VERSION[0] >= 3 @@ -52,16 +51,3 @@ def test_order_by_exists(self): authors_by_posts = Author.objects.order_by(Exists(Post.objects.filter(author=OuterRef('pk'))).asc()) self.assertSequenceEqual(authors_by_posts, [author_without_posts, self.author]) - - -@skipUnlessDBFeature('supports_partially_nullable_unique_constraints') -class TestPartiallyNullableUniqueTogether(TestCase): - def test_partially_nullable(self): - author = Author.objects.create(name="author") - Post.objects.create(title="foo", author=author) - Post.objects.create(title="foo", author=author) - - editor = Editor.objects.create(name="editor") - Post.objects.create(title="foo", author=author, alt_editor=editor) - with self.assertRaises(IntegrityError): - Post.objects.create(title="foo", author=author, alt_editor=editor) From 76a1317121fa1526eb3bc022dbe85f9921b69af8 Mon Sep 17 00:00:00 2001 From: Henrik Ek Date: Tue, 21 Apr 2020 15:57:32 +0200 Subject: [PATCH 02/10] Exclude indexes when deleting unique constraints (#51) --- sql_server/pyodbc/schema.py | 13 ++++++++--- .../0006_test_remove_onetoone_field_part1.py | 22 +++++++++++++++++++ .../0007_test_remove_onetoone_field_part2.py | 17 ++++++++++++++ testapp/models.py | 7 ++++++ 4 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 testapp/migrations/0006_test_remove_onetoone_field_part1.py create mode 100644 testapp/migrations/0007_test_remove_onetoone_field_part2.py diff --git a/sql_server/pyodbc/schema.py b/sql_server/pyodbc/schema.py index cf75b27b..9abcbd04 100644 --- a/sql_server/pyodbc/schema.py +++ b/sql_server/pyodbc/schema.py @@ -2,10 +2,16 @@ import datetime from django.db.backends.base.schema import ( - BaseDatabaseSchemaEditor, logger, _is_relevant_relation, _related_non_m2m_objects, + BaseDatabaseSchemaEditor, + _is_relevant_relation, + _related_non_m2m_objects, + logger, ) from django.db.backends.ddl_references import ( - Columns, IndexName, Statement as DjStatement, Table, + Columns, + IndexName, + Statement as DjStatement, + Table, ) from django.db.models import Index from django.db.models.fields import AutoField, BigAutoField @@ -935,7 +941,8 @@ def remove_field(self, model, field): }) # Drop unique constraints, SQL Server requires explicit deletion for name, infodict in constraints.items(): - if field.column in infodict['columns'] and infodict['unique'] and not infodict['primary_key']: + if (field.column in infodict['columns'] and infodict['unique'] and + not infodict['primary_key'] and not infodict['index']): self.execute(self.sql_delete_unique % { "table": self.quote_name(model._meta.db_table), "name": self.quote_name(name), diff --git a/testapp/migrations/0006_test_remove_onetoone_field_part1.py b/testapp/migrations/0006_test_remove_onetoone_field_part1.py new file mode 100644 index 00000000..e7e61473 --- /dev/null +++ b/testapp/migrations/0006_test_remove_onetoone_field_part1.py @@ -0,0 +1,22 @@ +# Generated by Django 3.0.4 on 2020-04-20 14:59 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('testapp', '0005_test_issue45_unique_type_change_part2'), + ] + + operations = [ + migrations.CreateModel( + name='TestRemoveOneToOneFieldModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('a', models.CharField(max_length=50)), + ('b', models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, to='testapp.TestRemoveOneToOneFieldModel')), + ], + ), + ] diff --git a/testapp/migrations/0007_test_remove_onetoone_field_part2.py b/testapp/migrations/0007_test_remove_onetoone_field_part2.py new file mode 100644 index 00000000..cc64ff69 --- /dev/null +++ b/testapp/migrations/0007_test_remove_onetoone_field_part2.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.4 on 2020-04-20 14:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('testapp', '0006_test_remove_onetoone_field_part1'), + ] + + operations = [ + migrations.RemoveField( + model_name='testremoveonetoonefieldmodel', + name='b', + ), + ] diff --git a/testapp/models.py b/testapp/models.py index 7b993165..c87f797b 100644 --- a/testapp/models.py +++ b/testapp/models.py @@ -64,3 +64,10 @@ class Meta: a = models.CharField(max_length=51, null=True) b = models.CharField(max_length=50) c = models.CharField(max_length=50) + + +class TestRemoveOneToOneFieldModel(models.Model): + # Fields used for testing removing OneToOne field. Verifies that delete_unique do not try to remove indexes + # thats already is removed. + # b = models.OneToOneField('self', on_delete=models.SET_NULL, null=True) + a = models.CharField(max_length=50) From b81a024940bb40db698c6ae075a9d531bc1d5c74 Mon Sep 17 00:00:00 2001 From: Henrik Ek Date: Wed, 22 Apr 2020 08:36:06 +0200 Subject: [PATCH 03/10] Bump version 2.8.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8c6b4ad6..57f920c6 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ setup( name='django-mssql-backend', - version='2.8.0', + version='2.8.1', description='Django backend for Microsoft SQL Server', long_description=open('README.rst').read(), author='ES Solutions AB', From 05f3fe1121046fb45969b3345baa8fe105200f69 Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Fri, 16 Oct 2020 07:04:24 +0200 Subject: [PATCH 04/10] Fix startup errors in Django 3.1 (#64) --- .github/workflows/main.yml | 12 ++++++++++++ .travis.yml | 6 ++++++ sql_server/pyodbc/creation.py | 13 ++++++++++--- sql_server/pyodbc/features.py | 1 + sql_server/pyodbc/operations.py | 19 +++++++++++++++++++ sql_server/pyodbc/schema.py | 2 ++ tox.ini | 2 ++ 7 files changed, 52 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 03205a8b..4cdf48d8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,11 +41,14 @@ jobs: tox_env: - "py36-django22" - "py36-django30" + - "py36-django31" - "py37-django22" - "py37-django30" + - "py37-django31" - "py38-django30" + - "py38-django31" include: - python: "3.6" @@ -54,15 +57,24 @@ jobs: - python: "3.6" tox_env: "py36-django30" + - python: "3.6" + tox_env: "py36-django31" + - python: "3.7" tox_env: "py37-django22" - python: "3.7" tox_env: "py37-django30" + - python: "3.7" + tox_env: "py37-django31" + - python: "3.8" tox_env: "py38-django30" + - python: "3.8" + tox_env: "py38-django31" + steps: - uses: actions/checkout@v2 diff --git a/.travis.yml b/.travis.yml index a938aad3..6b8f8425 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,19 +41,25 @@ matrix: - { before_install: *linux_before_install, python: "3.6", os: linux, env: TOX_ENV=py36-django22 } - { before_install: *linux_before_install, python: "3.6", os: linux, env: TOX_ENV=py36-django30 } + - { before_install: *linux_before_install, python: "3.6", os: linux, env: TOX_ENV=py36-django31 } - { before_install: *linux_before_install, python: "3.7", os: linux, env: TOX_ENV=py37-django22 } - { before_install: *linux_before_install, python: "3.7", os: linux, env: TOX_ENV=py37-django30 } + - { before_install: *linux_before_install, python: "3.7", os: linux, env: TOX_ENV=py37-django31 } - { before_install: *linux_before_install, python: "3.8", os: linux, env: TOX_ENV=py38-django30 } + - { before_install: *linux_before_install, python: "3.8", os: linux, env: TOX_ENV=py38-django31 } - { before_install: *win_before_install, language: sh, python: "3.6", os: windows, env: TOX_ENV=py36-django22 } - { before_install: *win_before_install, language: sh, python: "3.6", os: windows, env: TOX_ENV=py36-django30 } + - { before_install: *win_before_install, language: sh, python: "3.6", os: windows, env: TOX_ENV=py36-django31 } - { before_install: *win_before_install, language: sh, python: "3.7", os: windows, env: TOX_ENV=py37-django22 } - { before_install: *win_before_install, language: sh, python: "3.7", os: windows, env: TOX_ENV=py37-django30 } + - { before_install: *win_before_install, language: sh, python: "3.7", os: windows, env: TOX_ENV=py37-django31 } - { before_install: *win_before_install, language: sh, python: "3.8", os: windows, env: TOX_ENV=py38-django30 } + - { before_install: *win_before_install, language: sh, python: "3.8", os: windows, env: TOX_ENV=py38-django31 } diff --git a/sql_server/pyodbc/creation.py b/sql_server/pyodbc/creation.py index 61745b57..eb0cc890 100644 --- a/sql_server/pyodbc/creation.py +++ b/sql_server/pyodbc/creation.py @@ -1,10 +1,17 @@ import binascii import os +import django from django.db.backends.base.creation import BaseDatabaseCreation class DatabaseCreation(BaseDatabaseCreation): + @property + def cursor(self): + if django.VERSION >= (3, 1): + return self.connection._nodb_cursor + + return self.connection._nodb_connection.cursor def _destroy_test_db(self, test_database_name, verbosity): """ @@ -14,7 +21,7 @@ def _destroy_test_db(self, test_database_name, verbosity): # ourselves. Connect to the previous database (not the test database) # to do so, because it's not allowed to delete a database while being # connected to it. - with self.connection._nodb_connection.cursor() as cursor: + with self.cursor() as cursor: to_azure_sql_db = self.connection.to_azure_sql_db if not to_azure_sql_db: cursor.execute("ALTER DATABASE %s SET SINGLE_USER WITH ROLLBACK IMMEDIATE" @@ -36,7 +43,7 @@ def enable_clr(self): This function will not fail if current user doesn't have permissions to enable clr, and clr is already enabled """ - with self._nodb_connection.cursor() as cursor: + with self.cursor() as cursor: # check whether clr is enabled cursor.execute(''' SELECT value FROM sys.configurations @@ -86,7 +93,7 @@ def install_regex_clr(self, database_name): self.enable_clr() - with self._nodb_connection.cursor() as cursor: + with self.cursor() as cursor: for s in sql: cursor.execute(s) diff --git a/sql_server/pyodbc/features.py b/sql_server/pyodbc/features.py index 6563b9d9..1e184217 100644 --- a/sql_server/pyodbc/features.py +++ b/sql_server/pyodbc/features.py @@ -22,6 +22,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): requires_literal_defaults = True requires_sqlparse_for_splitting = False supports_boolean_expr_in_select_clause = False + supports_deferrable_unique_constraints = False supports_ignore_conflicts = False supports_index_on_text_field = False supports_paramstyle_pyformat = False diff --git a/sql_server/pyodbc/operations.py b/sql_server/pyodbc/operations.py index 74b1c009..6af85bab 100644 --- a/sql_server/pyodbc/operations.py +++ b/sql_server/pyodbc/operations.py @@ -1,9 +1,13 @@ import datetime import uuid import warnings +import django from django.conf import settings from django.db.backends.base.operations import BaseDatabaseOperations +from django.db.models import Exists, ExpressionWrapper +from django.db.models.expressions import RawSQL +from django.db.models.sql.where import WhereNode from django.utils import timezone from django.utils.encoding import force_str @@ -440,3 +444,18 @@ def time_trunc_sql(self, lookup_type, field_name): elif lookup_type == 'second': sql = "CONVERT(time, SUBSTRING(CONVERT(varchar, %s, 114), 0, 9))" % field_name return sql + + def conditional_expression_supported_in_where_clause(self, expression): + """ + Following "Moved conditional expression wrapping to the Exact lookup" in django 3.1 + https://github.com/django/django/commit/37e6c5b79bd0529a3c85b8c478e4002fd33a2a1d + """ + if django.VERSION >= (3, 1): + if isinstance(expression, (Exists, WhereNode)): + return True + if isinstance(expression, ExpressionWrapper) and expression.conditional: + return self.conditional_expression_supported_in_where_clause(expression.expression) + if isinstance(expression, RawSQL) and expression.conditional: + return True + return False + return True diff --git a/sql_server/pyodbc/schema.py b/sql_server/pyodbc/schema.py index 9abcbd04..38bc80aa 100644 --- a/sql_server/pyodbc/schema.py +++ b/sql_server/pyodbc/schema.py @@ -694,6 +694,7 @@ def create_unique_name(*args, **kwargs): name=name, columns=columns, condition=' WHERE ' + condition, + deferrable='' ) if self.connection.features.supports_partial_indexes else None else: return Statement( @@ -701,6 +702,7 @@ def create_unique_name(*args, **kwargs): table=table, name=name, columns=columns, + deferrable='' ) def _create_index_sql(self, model, fields, *, name=None, suffix='', using='', diff --git a/tox.ini b/tox.ini index 1b56e027..6bc0e3ae 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,7 @@ envlist = {py36,py37}-django22, {py36,py37,py38}-django30, + {py36,py37,py38}-django31, [testenv] passenv = @@ -19,4 +20,5 @@ commands = deps = django22: django==2.2.* django30: django>=3.0a1,<3.1 + django31: django>=3.1,<3.2 dj-database-url==0.5.0 From 7982506f0bb012aa10f28e2825343f1267172241 Mon Sep 17 00:00:00 2001 From: Henrik Ek Date: Thu, 22 Oct 2020 14:24:44 +0200 Subject: [PATCH 05/10] Fix better support for django 3.1.2 (#82) - Support for JSONField - Updates parameter structure for sql_flush - reset_sequences - Add an mssql specific feature flag - supports_order_by_is_nulls --- sql_server/pyodbc/base.py | 25 +++++----- sql_server/pyodbc/features.py | 2 + sql_server/pyodbc/functions.py | 29 ++++++++++++ sql_server/pyodbc/operations.py | 84 ++++++++++++++++++--------------- 4 files changed, 89 insertions(+), 51 deletions(-) diff --git a/sql_server/pyodbc/base.py b/sql_server/pyodbc/base.py index 297a90b8..590fc7c4 100644 --- a/sql_server/pyodbc/base.py +++ b/sql_server/pyodbc/base.py @@ -12,28 +12,28 @@ except ImportError as e: raise ImproperlyConfigured("Error loading pyodbc module: %s" % e) -from django.utils.version import get_version_tuple # noqa +from django.utils.version import get_version_tuple # noqa pyodbc_ver = get_version_tuple(Database.version) if pyodbc_ver < (3, 0): raise ImproperlyConfigured("pyodbc 3.0 or newer is required; you have %s" % Database.version) -from django.conf import settings # noqa -from django.db import NotSupportedError # noqa -from django.db.backends.base.base import BaseDatabaseWrapper # noqa -from django.utils.encoding import smart_str # noqa -from django.utils.functional import cached_property # noqa +from django.conf import settings # noqa +from django.db import NotSupportedError # noqa +from django.db.backends.base.base import BaseDatabaseWrapper # noqa +from django.utils.encoding import smart_str # noqa +from django.utils.functional import cached_property # noqa if hasattr(settings, 'DATABASE_CONNECTION_POOLING'): if not settings.DATABASE_CONNECTION_POOLING: Database.pooling = False -from .client import DatabaseClient # noqa -from .creation import DatabaseCreation # noqa -from .features import DatabaseFeatures # noqa -from .introspection import DatabaseIntrospection # noqa -from .operations import DatabaseOperations # noqa -from .schema import DatabaseSchemaEditor # noqa +from .client import DatabaseClient # noqa +from .creation import DatabaseCreation # noqa +from .features import DatabaseFeatures # noqa +from .introspection import DatabaseIntrospection # noqa +from .operations import DatabaseOperations # noqa +from .schema import DatabaseSchemaEditor # noqa EDITION_AZURE_SQL_DB = 5 @@ -95,6 +95,7 @@ class DatabaseWrapper(BaseDatabaseWrapper): 'TextField': 'nvarchar(max)', 'TimeField': 'time', 'UUIDField': 'char(32)', + 'JSONField': 'nvarchar(max)', } data_type_check_constraints = { 'PositiveIntegerField': '[%(column)s] >= 0', diff --git a/sql_server/pyodbc/features.py b/sql_server/pyodbc/features.py index 1e184217..f78ae730 100644 --- a/sql_server/pyodbc/features.py +++ b/sql_server/pyodbc/features.py @@ -34,6 +34,8 @@ class DatabaseFeatures(BaseDatabaseFeatures): supports_timezones = False supports_transactions = True uses_savepoints = True + supports_order_by_nulls_modifier = False + supports_order_by_is_nulls = False @cached_property def has_bulk_insert(self): diff --git a/sql_server/pyodbc/functions.py b/sql_server/pyodbc/functions.py index cc043281..967f5ffb 100644 --- a/sql_server/pyodbc/functions.py +++ b/sql_server/pyodbc/functions.py @@ -12,6 +12,34 @@ class TryCast(Cast): function = 'TRY_CAST' +def sqlserver_as_sql(self, compiler, connection, template=None, **extra_context): + template = template or self.template + if connection.features.supports_order_by_nulls_modifier: + if self.nulls_last: + template = '%s NULLS LAST' % template + elif self.nulls_first: + template = '%s NULLS FIRST' % template + else: + if self.nulls_last and not ( + self.descending and connection.features.order_by_nulls_first + ) and connection.features.supports_order_by_is_nulls: + template = '%%(expression)s IS NULL, %s' % template + elif self.nulls_first and not ( + not self.descending and connection.features.order_by_nulls_first + ) and connection.features.supports_order_by_is_nulls: + template = '%%(expression)s IS NOT NULL, %s' % template + connection.ops.check_expression_support(self) + expression_sql, params = compiler.compile(self.expression) + placeholders = { + 'expression': expression_sql, + 'ordering': 'DESC' if self.descending else 'ASC', + **extra_context, + } + template = template or self.template + params *= template.count('%(expression)s') + return (template % placeholders).rstrip(), params + + def sqlserver_atan2(self, compiler, connection, **extra_context): return self.as_sql(compiler, connection, function='ATN2', **extra_context) @@ -85,3 +113,4 @@ def sqlserver_orderby(self, compiler, connection): Exists.as_microsoft = sqlserver_exists OrderBy.as_microsoft = sqlserver_orderby +OrderBy.as_sql = sqlserver_as_sql diff --git a/sql_server/pyodbc/operations.py b/sql_server/pyodbc/operations.py index 6af85bab..64ef7e69 100644 --- a/sql_server/pyodbc/operations.py +++ b/sql_server/pyodbc/operations.py @@ -314,7 +314,7 @@ def savepoint_rollback_sql(self, sid): """ return "ROLLBACK TRANSACTION %s" % sid - def sql_flush(self, style, tables, sequences, allow_cascade=False): + def sql_flush(self, style, tables, *, reset_sequences=False, allow_cascade=False): """ Returns a list of SQL statements required to remove all data from the given database tables (without actually removing the tables @@ -329,14 +329,21 @@ def sql_flush(self, style, tables, sequences, allow_cascade=False): The `allow_cascade` argument determines whether truncation may cascade to tables with foreign keys pointing the tables being truncated. """ - if tables: - # Cannot use TRUNCATE on tables that are referenced by a FOREIGN KEY - # So must use the much slower DELETE - from django.db import connections - cursor = connections[self.connection.alias].cursor() - # Try to minimize the risks of the braindeaded inconsistency in - # DBCC CHEKIDENT(table, RESEED, n) behavior. - seqs = [] + if not tables: + return [] + + # Cannot use TRUNCATE on tables that are referenced by a FOREIGN KEY + # So must use the much slower DELETE + from django.db import connections + cursor = connections[self.connection.alias].cursor() + # Try to minimize the risks of the braindeaded inconsistency in + # DBCC CHEKIDENT(table, RESEED, n) behavior. + seqs = [] + if reset_sequences: + sequences = [ + sequence + for sequence in self.connection.introspection.sequence_list() + ] for seq in sequences: cursor.execute("SELECT COUNT(*) FROM %s" % self.quote_name(seq["table"])) rowcnt = cursor.fetchone()[0] @@ -347,37 +354,36 @@ def sql_flush(self, style, tables, sequences, allow_cascade=False): elem['start_id'] = 1 elem.update(seq) seqs.append(elem) - COLUMNS = "TABLE_NAME, CONSTRAINT_NAME" - WHERE = "CONSTRAINT_TYPE not in ('PRIMARY KEY','UNIQUE')" - cursor.execute( - "SELECT {} FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE {}".format(COLUMNS, WHERE)) - fks = cursor.fetchall() - sql_list = ['ALTER TABLE %s NOCHECK CONSTRAINT %s;' % - (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks] - sql_list.extend(['%s %s %s;' % (style.SQL_KEYWORD('DELETE'), style.SQL_KEYWORD('FROM'), - style.SQL_FIELD(self.quote_name(table))) for table in tables]) - - if self.connection.to_azure_sql_db and self.connection.sql_server_version < 2014: - warnings.warn("Resetting identity columns is not supported " - "on this versios of Azure SQL Database.", - RuntimeWarning) - else: - # Then reset the counters on each table. - sql_list.extend(['%s %s (%s, %s, %s) %s %s;' % ( - style.SQL_KEYWORD('DBCC'), - style.SQL_KEYWORD('CHECKIDENT'), - style.SQL_FIELD(self.quote_name(seq["table"])), - style.SQL_KEYWORD('RESEED'), - style.SQL_FIELD('%d' % seq['start_id']), - style.SQL_KEYWORD('WITH'), - style.SQL_KEYWORD('NO_INFOMSGS'), - ) for seq in seqs]) - - sql_list.extend(['ALTER TABLE %s CHECK CONSTRAINT %s;' % - (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks]) - return sql_list + + COLUMNS = "TABLE_NAME, CONSTRAINT_NAME" + WHERE = "CONSTRAINT_TYPE not in ('PRIMARY KEY','UNIQUE')" + cursor.execute( + "SELECT {} FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE {}".format(COLUMNS, WHERE)) + fks = cursor.fetchall() + sql_list = ['ALTER TABLE %s NOCHECK CONSTRAINT %s;' % + (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks] + sql_list.extend(['%s %s %s;' % (style.SQL_KEYWORD('DELETE'), style.SQL_KEYWORD('FROM'), + style.SQL_FIELD(self.quote_name(table))) for table in tables]) + + if self.connection.to_azure_sql_db and self.connection.sql_server_version < 2014: + warnings.warn("Resetting identity columns is not supported " + "on this versios of Azure SQL Database.", + RuntimeWarning) else: - return [] + # Then reset the counters on each table. + sql_list.extend(['%s %s (%s, %s, %s) %s %s;' % ( + style.SQL_KEYWORD('DBCC'), + style.SQL_KEYWORD('CHECKIDENT'), + style.SQL_FIELD(self.quote_name(seq["table"])), + style.SQL_KEYWORD('RESEED'), + style.SQL_FIELD('%d' % seq['start_id']), + style.SQL_KEYWORD('WITH'), + style.SQL_KEYWORD('NO_INFOMSGS'), + ) for seq in seqs]) + + sql_list.extend(['ALTER TABLE %s CHECK CONSTRAINT %s;' % + (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks]) + return sql_list def start_transaction_sql(self): """ From 33e62643836c86fff998192f10641e35bc77ff5f Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Sun, 3 Jan 2021 19:36:34 +0100 Subject: [PATCH 06/10] Clean tests (#94) --- .github/workflows/main.yml | 2 +- sql_server/pyodbc/features.py | 1 + sql_server/pyodbc/functions.py | 7 +- sql_server/pyodbc/operations.py | 58 ++++++++----- test.sh | 9 +- testapp/runner.py | 21 +++++ testapp/settings.py | 146 ++++++++++++++++++++++++++++++++ 7 files changed, 215 insertions(+), 29 deletions(-) create mode 100644 testapp/runner.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4cdf48d8..75c9174d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,7 +29,7 @@ jobs: pip install flake8 - name: Linting run: | - flake8 + flake8 --exclude testapp build: runs-on: ${{ matrix.os }} diff --git a/sql_server/pyodbc/features.py b/sql_server/pyodbc/features.py index f78ae730..4eb39115 100644 --- a/sql_server/pyodbc/features.py +++ b/sql_server/pyodbc/features.py @@ -36,6 +36,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): uses_savepoints = True supports_order_by_nulls_modifier = False supports_order_by_is_nulls = False + order_by_nulls_first = True @cached_property def has_bulk_insert(self): diff --git a/sql_server/pyodbc/functions.py b/sql_server/pyodbc/functions.py index 967f5ffb..c2cc6656 100644 --- a/sql_server/pyodbc/functions.py +++ b/sql_server/pyodbc/functions.py @@ -1,7 +1,7 @@ from django import VERSION from django.db.models import BooleanField from django.db.models.functions import Cast -from django.db.models.functions.math import ATan2, Log, Ln, Round +from django.db.models.functions.math import ATan2, Log, Ln, Mod, Round from django.db.models.expressions import Case, Exists, OrderBy, When from django.db.models.lookups import Lookup @@ -54,6 +54,10 @@ def sqlserver_ln(self, compiler, connection, **extra_context): return self.as_sql(compiler, connection, function='LOG', **extra_context) +def sqlserver_mod(self, compiler, connection, **extra_context): + return self.as_sql(compiler, connection, template='%(expressions)s', arg_joiner='%%', **extra_context) + + def sqlserver_round(self, compiler, connection, **extra_context): return self.as_sql(compiler, connection, template='%(function)s(%(expressions)s, 0)', **extra_context) @@ -105,6 +109,7 @@ def sqlserver_orderby(self, compiler, connection): ATan2.as_microsoft = sqlserver_atan2 Log.as_microsoft = sqlserver_log Ln.as_microsoft = sqlserver_ln +Mod.as_microsoft = sqlserver_mod Round.as_microsoft = sqlserver_round if DJANGO3: diff --git a/sql_server/pyodbc/operations.py b/sql_server/pyodbc/operations.py index 64ef7e69..00eebec0 100644 --- a/sql_server/pyodbc/operations.py +++ b/sql_server/pyodbc/operations.py @@ -114,6 +114,8 @@ def date_extract_sql(self, lookup_type, field_name): return "DATEPART(weekday, %s)" % field_name elif lookup_type == 'week': return "DATEPART(iso_week, %s)" % field_name + elif lookup_type == 'iso_year': + return "YEAR(DATEADD(day, 26 - DATEPART(isoww, %s), %s))" % (field_name, field_name) else: return "DATEPART(%s, %s)" % (lookup_type, field_name) @@ -314,7 +316,33 @@ def savepoint_rollback_sql(self, sid): """ return "ROLLBACK TRANSACTION %s" % sid - def sql_flush(self, style, tables, *, reset_sequences=False, allow_cascade=False): + def _build_sequences(self, sequences, cursor): + seqs = [] + for seq in sequences: + cursor.execute("SELECT COUNT(*) FROM %s" % self.quote_name(seq["table"])) + rowcnt = cursor.fetchone()[0] + elem = {} + if rowcnt: + elem['start_id'] = 0 + else: + elem['start_id'] = 1 + elem.update(seq) + seqs.append(elem) + return seqs + + def _sql_flush_new(self, style, tables, *, reset_sequences=False, allow_cascade=False): + if reset_sequences: + return [ + sequence + for sequence in self.connection.introspection.sequence_list() + ] + + return [] + + def _sql_flush_old(self, style, tables, sequences, allow_cascade=False): + return sequences + + def sql_flush(self, style, tables, *args, **kwargs): """ Returns a list of SQL statements required to remove all data from the given database tables (without actually removing the tables @@ -329,31 +357,19 @@ def sql_flush(self, style, tables, *, reset_sequences=False, allow_cascade=False The `allow_cascade` argument determines whether truncation may cascade to tables with foreign keys pointing the tables being truncated. """ + if not tables: return [] - # Cannot use TRUNCATE on tables that are referenced by a FOREIGN KEY - # So must use the much slower DELETE + if django.VERSION >= (3, 1): + sequences = self._sql_flush_new(style, tables, *args, **kwargs) + else: + sequences = self._sql_flush_old(style, tables, *args, **kwargs) + from django.db import connections cursor = connections[self.connection.alias].cursor() - # Try to minimize the risks of the braindeaded inconsistency in - # DBCC CHEKIDENT(table, RESEED, n) behavior. - seqs = [] - if reset_sequences: - sequences = [ - sequence - for sequence in self.connection.introspection.sequence_list() - ] - for seq in sequences: - cursor.execute("SELECT COUNT(*) FROM %s" % self.quote_name(seq["table"])) - rowcnt = cursor.fetchone()[0] - elem = {} - if rowcnt: - elem['start_id'] = 0 - else: - elem['start_id'] = 1 - elem.update(seq) - seqs.append(elem) + + seqs = self._build_sequences(sequences, cursor) COLUMNS = "TABLE_NAME, CONSTRAINT_NAME" WHERE = "CONSTRAINT_TYPE not in ('PRIMARY KEY','UNIQUE')" diff --git a/test.sh b/test.sh index 644249f8..b03216d8 100755 --- a/test.sh +++ b/test.sh @@ -8,9 +8,9 @@ set -e DJANGO_VERSION="$(python -m django --version)" cd django -git fetch --depth=1 origin +refs/tags/*:refs/tags/* -git checkout $DJANGO_VERSION -pip install -r tests/requirements/py3.txt +git fetch -q --depth=1 origin +refs/tags/*:refs/tags/* +git checkout -q $DJANGO_VERSION +pip install -q -r tests/requirements/py3.txt python tests/runtests.py --settings=testapp.settings --noinput --keepdb \ aggregation \ @@ -77,9 +77,6 @@ python tests/runtests.py --settings=testapp.settings --noinput --keepdb \ many_to_one \ max_lengths \ migrate_signals \ - migration_test_data_persistence \ - migrations \ - migrations2 \ model_fields \ model_indexes \ model_options \ diff --git a/testapp/runner.py b/testapp/runner.py new file mode 100644 index 00000000..e7e73070 --- /dev/null +++ b/testapp/runner.py @@ -0,0 +1,21 @@ +from unittest import skip +from django.test.runner import DiscoverRunner +from django.conf import settings + + +EXCLUDED_TESTS = getattr(settings, 'EXCLUDED_TESTS', []) + + +class ExcludeTestSuiteRunner(DiscoverRunner): + def build_suite(self, *args, **kwargs): + suite = super().build_suite(*args, **kwargs) + for case in suite: + cls = case.__class__ + for attr in dir(cls): + if not attr.startswith('test_'): + continue + fullname = f'{cls.__module__}.{cls.__name__}.{attr}' + if len(list(filter(fullname.startswith, EXCLUDED_TESTS))): + setattr(cls, attr, skip(getattr(cls, attr))) + + return suite diff --git a/testapp/settings.py b/testapp/settings.py index 07106562..eada7f9f 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -13,6 +13,152 @@ 'testapp', ) + +TEST_RUNNER = 'testapp.runner.ExcludeTestSuiteRunner' +EXCLUDED_TESTS = ( + 'aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation_exists', + 'aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation_values_collision', + 'aggregation.tests.AggregateTestCase.test_count_star', + 'aggregation.tests.AggregateTestCase.test_distinct_on_aggregate', + 'aggregation.tests.AggregateTestCase.test_expression_on_aggregation', + 'aggregation_regress.tests.AggregationTests.test_annotated_conditional_aggregate', + 'aggregation_regress.tests.AggregationTests.test_annotation_with_value', + 'aggregation_regress.tests.AggregationTests.test_more_more', + 'aggregation_regress.tests.AggregationTests.test_more_more_more', + 'aggregation_regress.tests.AggregationTests.test_ticket_11293', + 'aggregation_regress.tests.AggregationTests.test_values_list_annotation_args_ordering', + 'annotations.tests.NonAggregateAnnotationTestCase.test_annotate_exists', + 'annotations.tests.NonAggregateAnnotationTestCase.test_combined_expression_annotation_with_aggregation', + 'backends.tests.BackendTestCase.test_queries', + 'backends.tests.BackendTestCase.test_unicode_password', + 'backends.tests.FkConstraintsTests.test_disable_constraint_checks_context_manager', + 'backends.tests.FkConstraintsTests.test_disable_constraint_checks_manually', + 'backends.tests.LastExecutedQueryTest.test_last_executed_query', + 'bulk_create.tests.BulkCreateTests.test_bulk_insert_nullable_fields', + 'constraints.tests.CheckConstraintTests.test_abstract_name', + 'constraints.tests.CheckConstraintTests.test_database_constraint', + 'constraints.tests.CheckConstraintTests.test_database_constraint_expression', + 'constraints.tests.CheckConstraintTests.test_database_constraint_expressionwrapper', + 'constraints.tests.CheckConstraintTests.test_name', + 'constraints.tests.UniqueConstraintTests.test_database_constraint', + 'constraints.tests.UniqueConstraintTests.test_database_constraint_with_condition', + 'constraints.tests.UniqueConstraintTests.test_name', + 'custom_lookups.tests.BilateralTransformTests.test_transform_order_by', + 'datatypes.tests.DataTypesTestCase.test_error_on_timezone', + 'datetimes.tests.DateTimesTests.test_datetimes_ambiguous_and_invalid_times', + 'datetimes.tests.DateTimesTests.test_datetimes_returns_available_dates_for_given_scope_and_given_field', + 'datetimes.tests.DateTimesTests.test_related_model_traverse', + 'db_functions.comparison.test_cast.CastTests.test_cast_to_integer', + 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_func', + 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_iso_weekday_func', + 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_year_exact_lookup', + 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_year_greaterthan_lookup', + 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_year_lessthan_lookup', + 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_trunc_func', + 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_trunc_week_func', + 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_func', + 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_func_with_timezone', + 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_iso_weekday_func', + 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_year_exact_lookup', + 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_year_greaterthan_lookup', + 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_year_lessthan_lookup', + 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_ambiguous_and_invalid_times', + 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_func_with_timezone', + 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_none', + 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_week_func', + 'db_functions.math.test_degrees.DegreesTests.test_integer', + 'db_functions.math.test_mod.ModTests.test_float', + 'db_functions.math.test_power.PowerTests.test_integer', + 'db_functions.math.test_radians.RadiansTests.test_integer', + 'db_functions.text.test_md5', + 'db_functions.text.test_pad.PadTests.test_pad', + 'db_functions.text.test_replace.ReplaceTests.test_case_sensitive', + 'db_functions.text.test_sha1', + 'db_functions.text.test_sha224', + 'db_functions.text.test_sha256', + 'db_functions.text.test_sha384', + 'db_functions.text.test_sha512', + 'dbshell.tests.DbshellCommandTestCase.test_command_missing', + 'defer_regress.tests.DeferRegressionTest.test_ticket_23270', + 'delete.tests.DeletionTests.test_only_referenced_fields_selected', + 'expressions.tests.BasicExpressionsTests.test_case_in_filter_if_boolean_output_field', + 'expressions.tests.BasicExpressionsTests.test_filtering_on_annotate_that_uses_q', + 'expressions.tests.BasicExpressionsTests.test_order_by_exists', + 'expressions.tests.BasicExpressionsTests.test_subquery_in_filter', + 'expressions.tests.ExpressionOperatorTests.test_lefthand_bitwise_right_shift_operator', + 'expressions.tests.ExpressionOperatorTests.test_lefthand_bitwise_xor', + 'expressions.tests.ExpressionOperatorTests.test_lefthand_bitwise_xor_null', + 'expressions.tests.ExpressionOperatorTests.test_righthand_power', + 'expressions.tests.FTimeDeltaTests.test_date_subquery_subtraction', + 'expressions.tests.FTimeDeltaTests.test_datetime_subquery_subtraction', + 'expressions.tests.FTimeDeltaTests.test_datetime_subtraction_microseconds', + 'expressions.tests.FTimeDeltaTests.test_duration_with_datetime_microseconds', + 'expressions.tests.FTimeDeltaTests.test_invalid_operator', + 'expressions.tests.FTimeDeltaTests.test_time_subquery_subtraction', + 'expressions.tests.IterableLookupInnerExpressionsTests.test_expressions_in_lookups_join_choice', + 'expressions_case.tests.CaseExpressionTests.test_annotate_with_in_clause', + 'fixtures_regress.tests.TestFixtures.test_loaddata_raises_error_when_fixture_has_invalid_foreign_key', + 'fixtures_regress.tests.TestFixtures.test_loaddata_with_m2m_to_self', + 'fixtures_regress.tests.TestFixtures.test_loaddata_with_valid_fixture_dirs', + 'fixtures_regress.tests.TestFixtures.test_loaddata_works_when_fixture_has_forward_refs', + 'fixtures_regress.tests.TestFixtures.test_path_containing_dots', + 'fixtures_regress.tests.TestFixtures.test_pg_sequence_resetting_checks', + 'fixtures_regress.tests.TestFixtures.test_pretty_print_xml', + 'fixtures_regress.tests.TestFixtures.test_proxy_model_included', + 'fixtures_regress.tests.TestFixtures.test_relative_path', + 'fixtures_regress.tests.TestFixtures.test_relative_path_in_fixture_dirs', + 'fixtures_regress.tests.TestFixtures.test_ticket_20820', + 'fixtures_regress.tests.TestFixtures.test_ticket_22421', + 'get_or_create.tests.UpdateOrCreateTransactionTests.test_creation_in_transaction', + 'indexes.tests.PartialIndexTests.test_multiple_conditions', + 'indexes.tests.SchemaIndexesNotPostgreSQLTests.test_create_index_ignores_opclasses', + 'inspectdb.tests.InspectDBTestCase.test_introspection_errors', + 'inspectdb.tests.InspectDBTestCase.test_json_field', + 'inspectdb.tests.InspectDBTestCase.test_number_field_types', + 'introspection.tests.IntrospectionTests.test_get_constraints', + 'introspection.tests.IntrospectionTests.test_get_table_description_types', + 'introspection.tests.IntrospectionTests.test_smallautofield', + 'invalid_models_tests.test_ordinary_fields.TextFieldTests.test_max_length_warning', + 'migrate_signals.tests.MigrateSignalTests.test_migrations_only', + 'model_fields.test_integerfield.PositiveBigIntegerFieldTests', + 'model_fields.test_jsonfield', + 'model_indexes.tests.IndexesTests.test_db_tablespace', + 'ordering.tests.OrderingTests.test_deprecated_values_annotate', + 'ordering.tests.OrderingTests.test_order_by_fk_attname', + 'ordering.tests.OrderingTests.test_order_by_pk', + 'ordering.tests.OrderingTests.test_orders_nulls_first_on_filtered_subquery', + 'prefetch_related.tests.GenericRelationTests.test_prefetch_GFK_nonint_pk', + 'queries.test_bulk_update.BulkUpdateNoteTests.test_set_field_to_null', + 'queries.test_bulk_update.BulkUpdateTests.test_json_field', + 'queries.test_db_returning', + 'queries.test_qs_combinators.QuerySetSetOperationTests.test_limits', + 'queries.test_qs_combinators.QuerySetSetOperationTests.test_ordering_by_f_expression_and_alias', + 'schema.tests.SchemaTests.test_add_foreign_key_quoted_db_table', + 'schema.tests.SchemaTests.test_alter_auto_field_quoted_db_column', + 'schema.tests.SchemaTests.test_alter_auto_field_to_char_field', + 'schema.tests.SchemaTests.test_alter_auto_field_to_integer_field', + 'schema.tests.SchemaTests.test_alter_autofield_pk_to_bigautofield_pk_sequence_owner', + 'schema.tests.SchemaTests.test_alter_autofield_pk_to_smallautofield_pk_sequence_owner', + 'schema.tests.SchemaTests.test_alter_implicit_id_to_explicit', + 'schema.tests.SchemaTests.test_alter_int_pk_to_autofield_pk', + 'schema.tests.SchemaTests.test_alter_int_pk_to_bigautofield_pk', + 'schema.tests.SchemaTests.test_alter_pk_with_self_referential_field', + 'schema.tests.SchemaTests.test_alter_primary_key_quoted_db_table', + 'schema.tests.SchemaTests.test_alter_smallint_pk_to_smallautofield_pk', + 'schema.tests.SchemaTests.test_char_field_pk_to_auto_field', + 'schema.tests.SchemaTests.test_inline_fk', + 'schema.tests.SchemaTests.test_no_db_constraint_added_during_primary_key_change', + 'schema.tests.SchemaTests.test_remove_field_check_does_not_remove_meta_constraints', + 'schema.tests.SchemaTests.test_remove_field_unique_does_not_remove_meta_constraints', + 'schema.tests.SchemaTests.test_remove_unique_together_does_not_remove_meta_constraints', + 'schema.tests.SchemaTests.test_text_field_with_db_index', + 'schema.tests.SchemaTests.test_unique_and_reverse_m2m', + 'schema.tests.SchemaTests.test_unique_no_unnecessary_fk_drops', + 'schema.tests.SchemaTests.test_unique_together_with_fk', + 'schema.tests.SchemaTests.test_unique_together_with_fk_with_existing_index', + 'select_for_update.tests.SelectForUpdateTests.test_for_update_after_from', +) + SECRET_KEY = "django_tests_secret_key" # Use a fast hasher to speed up tests. From 8d76f0febb185c6142a445ef956d5495ab8ff27b Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Sun, 3 Jan 2021 21:36:54 +0100 Subject: [PATCH 07/10] Provide reason for skipping tests --- testapp/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testapp/runner.py b/testapp/runner.py index e7e73070..4e5e99a7 100644 --- a/testapp/runner.py +++ b/testapp/runner.py @@ -16,6 +16,6 @@ def build_suite(self, *args, **kwargs): continue fullname = f'{cls.__module__}.{cls.__name__}.{attr}' if len(list(filter(fullname.startswith, EXCLUDED_TESTS))): - setattr(cls, attr, skip(getattr(cls, attr))) + setattr(cls, attr, skip('Does not work on MSSQL')(getattr(cls, attr))) return suite From d166fc96dec33c28b590f5e8353f1f4b0fd4529c Mon Sep 17 00:00:00 2001 From: David Beitey Date: Mon, 4 Jan 2021 07:16:13 +1000 Subject: [PATCH 08/10] Fix host_is_server URL and expand docs in README (#92) --- README.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 625b1d15..17e2108a 100644 --- a/README.rst +++ b/README.rst @@ -149,10 +149,12 @@ Dictionary. Current available keys are: definition present in the ``freetds.conf`` FreeTDS configuration file instead of a hostname or an IP address. - But if this option is present and it's value is ``True``, this - special behavior is turned off. + But if this option is present and its value is ``True``, this + special behavior is turned off. Instead, connections to the database + server will be established using ``HOST`` and ``PORT`` options, without + requiring ``freetds.conf`` to be configured. - See http://www.freetds.org/userguide/dsnless.htm for more information. + See https://www.freetds.org/userguide/dsnless.html for more information. - unicode_results From 33389b66550451c818a7d98cec8e6de486b7221a Mon Sep 17 00:00:00 2001 From: David Beitey Date: Mon, 11 Jan 2021 19:24:39 +0000 Subject: [PATCH 09/10] Accept deferrable kwarg for schema editor SQL (#86) This follows Django 3.1.x's lead in adding `deferrable` as a keyword argument to the `_create_unique_sql` function, following their implementation in the schema editor object. Without accepting this keyword arg, migrations will crash like so: ``` ... File "lib/python3.8/site-packages/django/db/backends/base/schema.py", line 360, in add_constraint sql = constraint.create_sql(model, self) File "lib/python3.8/site-packages/django/db/models/constraints.py", line 118, in create_sql return schema_editor._create_unique_sql( TypeError: _create_unique_sql() got an unexpected keyword argument 'deferrable' ``` This also adjusts the implementation to call `self_deferrable_constraint_sql(...)` with the deferrable arg; in this backend's case the return value is an empty string (like it is currently hardcoded). Essentially it's the same result but allows flexibility if this backend ever supported deferrable constraints. Backwards compatibility is maintained by checking for the passing of a deferrable keyword argument to the create sql function (not provided on earlier Django versions before 3.1) and that the database backend supports deferral for unique indexes. --- sql_server/pyodbc/schema.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sql_server/pyodbc/schema.py b/sql_server/pyodbc/schema.py index 38bc80aa..b7188228 100644 --- a/sql_server/pyodbc/schema.py +++ b/sql_server/pyodbc/schema.py @@ -1,5 +1,6 @@ import binascii import datetime +import django from django.db.backends.base.schema import ( BaseDatabaseSchemaEditor, @@ -677,7 +678,10 @@ def add_field(self, model, field): if self.connection.features.connection_persists_old_columns: self.connection.close() - def _create_unique_sql(self, model, columns, name=None, condition=None): + def _create_unique_sql(self, model, columns, name=None, condition=None, deferrable=None): + if (deferrable and not getattr(self.connection.features, 'supports_deferrable_unique_constraints', False)): + return None + def create_unique_name(*args, **kwargs): return self.quote_name(self._create_index_name(*args, **kwargs)) @@ -687,6 +691,10 @@ def create_unique_name(*args, **kwargs): else: name = self.quote_name(name) columns = Columns(table, columns, self.quote_name) + statement_args = { + "deferrable": self._deferrable_constraint_sql(deferrable) + } if django.VERSION >= (3, 1) else {} + if condition: return Statement( self.sql_create_unique_index, @@ -694,7 +702,7 @@ def create_unique_name(*args, **kwargs): name=name, columns=columns, condition=' WHERE ' + condition, - deferrable='' + **statement_args ) if self.connection.features.supports_partial_indexes else None else: return Statement( @@ -702,7 +710,7 @@ def create_unique_name(*args, **kwargs): table=table, name=name, columns=columns, - deferrable='' + **statement_args ) def _create_index_sql(self, model, fields, *, name=None, suffix='', using='', From 79e421ad377204580b7fb89b7e126be0c04dca75 Mon Sep 17 00:00:00 2001 From: aceofwings Date: Thu, 14 Jan 2021 13:17:41 -0500 Subject: [PATCH 10/10] Fix inspectdb (#96) --- sql_server/pyodbc/base.py | 1 + sql_server/pyodbc/features.py | 10 ++++++++++ sql_server/pyodbc/introspection.py | 5 ++++- testapp/settings.py | 2 -- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/sql_server/pyodbc/base.py b/sql_server/pyodbc/base.py index 590fc7c4..a6bbb993 100644 --- a/sql_server/pyodbc/base.py +++ b/sql_server/pyodbc/base.py @@ -89,6 +89,7 @@ class DatabaseWrapper(BaseDatabaseWrapper): 'OneToOneField': 'int', 'PositiveIntegerField': 'int', 'PositiveSmallIntegerField': 'smallint', + 'PositiveBigIntegerField': 'bigint', 'SlugField': 'nvarchar(%(max_length)s)', 'SmallAutoField': 'smallint IDENTITY (1, 1)', 'SmallIntegerField': 'smallint', diff --git a/sql_server/pyodbc/features.py b/sql_server/pyodbc/features.py index 4eb39115..17455dee 100644 --- a/sql_server/pyodbc/features.py +++ b/sql_server/pyodbc/features.py @@ -3,6 +3,8 @@ class DatabaseFeatures(BaseDatabaseFeatures): + can_introspect_json_field = False + has_native_json_field = False has_native_uuid_field = False allow_sliced_subqueries_with_in = False can_introspect_autofield = True @@ -57,3 +59,11 @@ def supports_partial_indexes(self): @cached_property def supports_functions_in_partial_indexes(self): return self.connection.sql_server_version > 2005 + + @cached_property + def introspected_field_types(self): + return { + **super().introspected_field_types, + 'GenericIPAddressField': 'CharField', + 'PositiveBigIntegerField': 'BigIntegerField' + } diff --git a/sql_server/pyodbc/introspection.py b/sql_server/pyodbc/introspection.py index 4b92f9e6..3961f5ce 100644 --- a/sql_server/pyodbc/introspection.py +++ b/sql_server/pyodbc/introspection.py @@ -1,13 +1,16 @@ import pyodbc as Database +from collections import namedtuple from django.db.backends.base.introspection import ( - BaseDatabaseIntrospection, FieldInfo, TableInfo, + BaseDatabaseIntrospection, TableInfo, ) from django.db.models.indexes import Index SQL_AUTOFIELD = -777555 SQL_BIGAUTOFIELD = -777444 +FieldInfo = namedtuple('FieldInfo', 'name type_code display_size internal_size precision scale null_ok default') + class DatabaseIntrospection(BaseDatabaseIntrospection): # Map type codes to Django Field types. diff --git a/testapp/settings.py b/testapp/settings.py index eada7f9f..a427aeb8 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -113,8 +113,6 @@ 'indexes.tests.PartialIndexTests.test_multiple_conditions', 'indexes.tests.SchemaIndexesNotPostgreSQLTests.test_create_index_ignores_opclasses', 'inspectdb.tests.InspectDBTestCase.test_introspection_errors', - 'inspectdb.tests.InspectDBTestCase.test_json_field', - 'inspectdb.tests.InspectDBTestCase.test_number_field_types', 'introspection.tests.IntrospectionTests.test_get_constraints', 'introspection.tests.IntrospectionTests.test_get_table_description_types', 'introspection.tests.IntrospectionTests.test_smallautofield',