From e8c53c89b81d5e1e3e778c4a29d8c2a528f37fc4 Mon Sep 17 00:00:00 2001 From: David Beitey Date: Wed, 3 Mar 2021 12:26:02 +1000 Subject: [PATCH 01/41] Put hashbang on first line of manage.py --- manage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/manage.py b/manage.py index a57ed35e..e88b9965 100755 --- a/manage.py +++ b/manage.py @@ -1,7 +1,8 @@ +#!/usr/bin/env python + # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -#!/usr/bin/env python import os import sys From 23524b5a3b8b6221ee393242f1ea6dbe91c428c5 Mon Sep 17 00:00:00 2001 From: David Beitey Date: Sun, 6 Dec 2020 23:39:13 +0000 Subject: [PATCH 02/41] Fix host_is_server URL and expand docs in README --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2d6be07b..5a89d288 100644 --- a/README.md +++ b/README.md @@ -135,10 +135,12 @@ Dictionary. Current available keys are: definition present in the ``freetds.conf`` FreeTDS configuration file instead of a hostname or an IP address. - But if this option is present and it's value is ``True``, this - special behavior is turned off. + But if this option is present and its value is ``True``, this + special behavior is turned off. Instead, connections to the database + server will be established using ``HOST`` and ``PORT`` options, without + requiring ``freetds.conf`` to be configured. - See http://www.freetds.org/userguide/dsnless.htm for more information. + See https://www.freetds.org/userguide/dsnless.html for more information. - unicode_results From 762c4643853632fbc19530fad07f6fc16231e4aa Mon Sep 17 00:00:00 2001 From: David Beitey Date: Mon, 11 Jan 2021 19:24:39 +0000 Subject: [PATCH 03/41] Backwards compatibility for deferrable kwarg This was previously accepted to the original repository in https://github.com/ESSolutions/django-mssql-backend/pull/86. Backwards compatibility for Django < 3.1 is maintained by not directly trying to load the supports_deferrable_unique_constraints via dot notation, but rather by getattr with a default. --- mssql/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mssql/schema.py b/mssql/schema.py index f85046fb..e617cb5f 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -689,7 +689,7 @@ def add_field(self, model, field): self.connection.close() def _create_unique_sql(self, model, columns, name=None, condition=None, deferrable=None): - if (deferrable and not self.connection.features.supports_deferrable_unique_constraints): + if (deferrable and not getattr(self.connection.features, 'supports_deferrable_unique_constraints', False)): return None def create_unique_name(*args, **kwargs): From c063460d4627b8b2054f0e97cb3f15a48d3cc565 Mon Sep 17 00:00:00 2001 From: David Beitey Date: Fri, 15 Jan 2021 19:03:26 +1000 Subject: [PATCH 04/41] Error on unsupported unique constraint conditions CREATE INDEX in SQL Server only supports AND conditions (not OR) as part of its WHERE syntax. This change handles that situation by raising an error from the schema editor class. This change adds unit tests to confirm this happens against a SQL Server database. Previously opened at https://github.com/ESSolutions/django-mssql-backend/pull/97 --- mssql/schema.py | 9 ++- .../0010_test_unique_constraints.py | 68 +++++++++++++++++++ testapp/models.py | 37 ++++++++++ testapp/tests/test_constraints.py | 61 ++++++++++++++++- 4 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 testapp/migrations/0010_test_unique_constraints.py diff --git a/mssql/schema.py b/mssql/schema.py index e617cb5f..b7b27c4b 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -17,8 +17,9 @@ Table, ) from django import VERSION as django_version -from django.db.models import Index +from django.db.models import Index, UniqueConstraint from django.db.models.fields import AutoField, BigAutoField +from django.db.models.sql.where import AND from django.db.transaction import TransactionManagementError from django.utils.encoding import force_str @@ -955,3 +956,9 @@ def remove_field(self, model, field): for sql in list(self.deferred_sql): if isinstance(sql, Statement) and sql.references_column(model._meta.db_table, field.column): self.deferred_sql.remove(sql) + + def add_constraint(self, model, constraint): + if isinstance(constraint, UniqueConstraint) and constraint.condition and constraint.condition.connector != AND: + raise NotImplementedError("The backend does not support %s conditions on unique constraint %s." % + (constraint.condition.connector, constraint.name)) + super().add_constraint(model, constraint) diff --git a/testapp/migrations/0010_test_unique_constraints.py b/testapp/migrations/0010_test_unique_constraints.py new file mode 100644 index 00000000..6c84c05e --- /dev/null +++ b/testapp/migrations/0010_test_unique_constraints.py @@ -0,0 +1,68 @@ +# Generated by Django 3.1.5 on 2021-01-18 00:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('testapp', '0009_test_drop_table_with_foreign_key_reference_part2'), + ] + + operations = [ + migrations.CreateModel( + name='TestUnsupportableUniqueConstraint', + fields=[ + ( + 'id', + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + ), + ), + ('_type', models.CharField(max_length=50)), + ('status', models.CharField(max_length=50)), + ], + options={ + 'managed': False, + }, + ), + migrations.CreateModel( + name='TestSupportableUniqueConstraint', + fields=[ + ( + 'id', + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + ), + ), + ('_type', models.CharField(max_length=50)), + ('status', models.CharField(max_length=50)), + ], + ), + migrations.AddConstraint( + model_name='testsupportableuniqueconstraint', + constraint=models.UniqueConstraint( + condition=models.Q( + ('status', 'in_progress'), + ('status', 'needs_changes'), + ('status', 'published'), + ), + fields=('_type',), + name='and_constraint', + ), + ), + migrations.AddConstraint( + model_name='testsupportableuniqueconstraint', + constraint=models.UniqueConstraint( + condition=models.Q(status__in=['in_progress', 'needs_changes']), + fields=('_type',), + name='in_constraint', + ), + ), + ] diff --git a/testapp/models.py b/testapp/models.py index 85a571d3..3ef2be3a 100644 --- a/testapp/models.py +++ b/testapp/models.py @@ -4,6 +4,7 @@ import uuid from django.db import models +from django.db.models import Q from django.utils import timezone @@ -74,3 +75,39 @@ class TestRemoveOneToOneFieldModel(models.Model): # thats already is removed. # b = models.OneToOneField('self', on_delete=models.SET_NULL, null=True) a = models.CharField(max_length=50) + + +class TestUnsupportableUniqueConstraint(models.Model): + class Meta: + managed = False + constraints = [ + models.UniqueConstraint( + name='or_constraint', + fields=['_type'], + condition=(Q(status='in_progress') | Q(status='needs_changes')), + ), + ] + + _type = models.CharField(max_length=50) + status = models.CharField(max_length=50) + + +class TestSupportableUniqueConstraint(models.Model): + class Meta: + constraints = [ + models.UniqueConstraint( + name='and_constraint', + fields=['_type'], + condition=( + Q(status='in_progress') & Q(status='needs_changes') & Q(status='published') + ), + ), + models.UniqueConstraint( + name='in_constraint', + fields=['_type'], + condition=(Q(status__in=['in_progress', 'needs_changes'])), + ), + ] + + _type = models.CharField(max_length=50) + status = models.CharField(max_length=50) diff --git a/testapp/tests/test_constraints.py b/testapp/tests/test_constraints.py index 635fa1b1..2c743f11 100644 --- a/testapp/tests/test_constraints.py +++ b/testapp/tests/test_constraints.py @@ -1,12 +1,18 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from django.db import connections, migrations, models +from django.db.migrations.state import ProjectState from django.db.utils import IntegrityError -from django.test import TestCase, skipUnlessDBFeature +from django.test import TestCase, TransactionTestCase, skipUnlessDBFeature +from mssql.base import DatabaseWrapper from ..models import ( - Author, Editor, Post, - TestUniqueNullableModel, TestNullableUniqueTogetherModel, + Author, + Editor, + Post, + TestUniqueNullableModel, + TestNullableUniqueTogetherModel, ) @@ -55,3 +61,52 @@ def test_after_type_change(self): TestNullableUniqueTogetherModel.objects.create(a='aaa', b='bbb', c='ccc') with self.assertRaises(IntegrityError): TestNullableUniqueTogetherModel.objects.create(a='aaa', b='bbb', c='ccc') + + +class TestUniqueConstraints(TransactionTestCase): + def test_unsupportable_unique_constraint(self): + # Only execute tests when running against SQL Server + connection = connections['default'] + if isinstance(connection, DatabaseWrapper): + + class TestMigration(migrations.Migration): + initial = True + + operations = [ + migrations.CreateModel( + name='TestUnsupportableUniqueConstraint', + fields=[ + ( + 'id', + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + ), + ), + ('_type', models.CharField(max_length=50)), + ('status', models.CharField(max_length=50)), + ], + ), + migrations.AddConstraint( + model_name='testunsupportableuniqueconstraint', + constraint=models.UniqueConstraint( + condition=models.Q( + ('status', 'in_progress'), + ('status', 'needs_changes'), + _connector='OR', + ), + fields=('_type',), + name='or_constraint', + ), + ), + ] + + migration = TestMigration('testapp', 'test_unsupportable_unique_constraint') + + with connection.schema_editor(atomic=True) as editor: + with self.assertRaisesRegex( + NotImplementedError, "does not support OR conditions" + ): + return migration.apply(ProjectState(), editor) From cd436a8d962811295dcabe9b5ca6ddd46704d6c6 Mon Sep 17 00:00:00 2001 From: Kapil Kaushik Date: Fri, 19 Mar 2021 10:07:39 +0530 Subject: [PATCH 05/41] Allow Trusted connection parameter oveririding Trusted connection parameter is automatically set to yes which causes authentication failure when a different authentication method is used. e.g. when using managed identities with azure sql and authentication as ActiveDirectoryMsi --- mssql/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mssql/base.py b/mssql/base.py index 17c27da2..d2b00a69 100644 --- a/mssql/base.py +++ b/mssql/base.py @@ -249,6 +249,7 @@ def get_new_connection(self, conn_params): user = conn_params.get('USER', None) password = conn_params.get('PASSWORD', None) port = conn_params.get('PORT', None) + trusted_connection = conn_params.get('Trusted_Connection', 'yes') options = conn_params.get('OPTIONS', {}) driver = options.get('driver', 'ODBC Driver 13 for SQL Server') @@ -289,7 +290,7 @@ def get_new_connection(self, conn_params): cstr_parts['PWD'] = password else: if ms_drivers.match(driver): - cstr_parts['Trusted_Connection'] = 'yes' + cstr_parts['Trusted_Connection'] = trusted_connection else: cstr_parts['Integrated Security'] = 'SSPI' From c446bbbde0c819dab7a067fc165b4aa727ca943f Mon Sep 17 00:00:00 2001 From: Kapil Kaushik Date: Fri, 19 Mar 2021 10:21:20 +0530 Subject: [PATCH 06/41] Updated README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 21d97baa..34666686 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,10 @@ in DATABASES control the behavior of the backend: Boolean. Set this to `False` if you want to disable Django's transaction management and implement your own. +- Trusted_Connection + + String. Default is `"yes"`. Can be set to `"no"` if required. + and the following entries are also available in the `TEST` dictionary for any given database-level settings dictionary: From 00bbdca7206192b12752d75ba33c78755e41e20d Mon Sep 17 00:00:00 2001 From: Warren Chu <60903512+vwarchu@users.noreply.github.com> Date: Fri, 19 Mar 2021 16:48:38 -0700 Subject: [PATCH 07/41] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ea3dfdc9..b22ff614 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ We hope you enjoy using the MSSQL-Django 3rd party backend. ## Features -- Supports Django 2.2, 3.0 +- Supports Django 2.2, 3.0 and 3.1 - Tested on Microsoft SQL Server 2016, 2017, 2019 - Passes most of the tests of the Django test suite - Compatible with @@ -20,12 +20,12 @@ We hope you enjoy using the MSSQL-Django 3rd party backend. ## Dependencies -- Django 2.2 or 3.0 +- Django 2.2, 3.0 or 3.1 - pyodbc 3.0 or newer ## Installation -1. Install pyodbc 3.0 (or newer) and Django 2.2 (or 3.0) +1. Install pyodbc 3.0 (or newer) and Django 2.2, 3.0 or 3.1 2. Install mssql-django: From 941060318c0086a5d077e398410993e94c859171 Mon Sep 17 00:00:00 2001 From: Tim Nyborg Date: Mon, 22 Mar 2021 10:49:49 +0000 Subject: [PATCH 08/41] mark supports_order_by_nulls_modifier unsupported The backend has an existing replacement, using CASE flag order_by_nulls_first feature as true Prevents erroneous attempts to order by "field IS NULL" --- mssql/features.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mssql/features.py b/mssql/features.py index 6e840b98..e186143d 100644 --- a/mssql/features.py +++ b/mssql/features.py @@ -21,6 +21,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): has_select_for_update_skip_locked = True ignores_table_name_case = True ignores_quoted_identifier_case = True + order_by_nulls_first = True requires_literal_defaults = True requires_sqlparse_for_splitting = False supports_boolean_expr_in_select_clause = False @@ -28,6 +29,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): supports_ignore_conflicts = False supports_index_on_text_field = False supports_json_field = False + supports_order_by_nulls_modifier = False supports_paramstyle_pyformat = False supports_regex_backreferencing = True supports_sequence_reset = False From b41e8b4cd31ce5e3c7e074c4a82e7da303654a54 Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Wed, 24 Mar 2021 17:33:28 -0700 Subject: [PATCH 09/41] Add docs and support for Azure AD Auth In base.py, if the user would like to use interactive auth, then password is not needed. --- README.md | 6 +++--- mssql/base.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b22ff614..d6b79204 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Welcome to the MSSQL-Django 3rd party backend project! *mssql-django* is a fork of [django-mssql-backend](https://pypi.org/project/django-mssql-backend/). This project provides an enterprise database connectivity option for the Django Web Framework, with support for Microsoft SQL Server and Azure SQL Database. -We'd like to give thanks to the community that made this project possible, with particular recognition of the contributors: OskarPersson, michiya, dlo and the original Google Code django-pyodbc team. Moving forward we encourage partipation in this project from both old and new contributors! +We'd like to give thanks to the community that made this project possible, with particular recognition of the contributors: OskarPersson, michiya, dlo and the original Google Code django-pyodbc team. Moving forward we encourage partipation in this project from both old and new contributors! We hope you enjoy using the MSSQL-Django 3rd party backend. @@ -81,7 +81,7 @@ for any given database-level settings dictionary: String. The name of database to use when running the test suite. If the default value (`None`) is used, the test database will use - the name `"test\_" + NAME`. + the name `"test_" + NAME`. - COLLATION @@ -151,7 +151,7 @@ Dictionary. Current available keys are: - extra_params String. Additional parameters for the ODBC connection. The format is - ``"param=value;param=value"``. + ``"param=value;param=value"``, [Azure AD Authentication](https://github.com/microsoft/mssql-django/wiki/Azure-AD-Authentication) can be added to this field. - collation diff --git a/mssql/base.py b/mssql/base.py index 17c27da2..5d1a6444 100644 --- a/mssql/base.py +++ b/mssql/base.py @@ -286,7 +286,8 @@ def get_new_connection(self, conn_params): if user: cstr_parts['UID'] = user - cstr_parts['PWD'] = password + if 'Authentication=ActiveDirectoryInteractive' not in options.get('extra_params', ''): + cstr_parts['PWD'] = password else: if ms_drivers.match(driver): cstr_parts['Trusted_Connection'] = 'yes' From 23757b63d687b6b682d5734e2a338a96fe949719 Mon Sep 17 00:00:00 2001 From: randlet Date: Fri, 26 Mar 2021 12:38:40 -0400 Subject: [PATCH 10/41] Add "OFFSET 0 ROWS" queries without an existing ordering or limit clause Without this patch, subqueries with orderings result in an exception: ProgrammingError: ('42000', '[42000] [Microsoft][ODBC Driver 13 for SQL Server][SQL Server]The ORDER BY clause is invalid in views, inline functions, derived tables, subqueries, and common table expressions, unless TOP, OFFSET or FOR XML is also specified. (1033) (SQLExecDirectW)') Resolves issue #12. --- mssql/compiler.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mssql/compiler.py b/mssql/compiler.py index 28d1fbd2..9df8081e 100644 --- a/mssql/compiler.py +++ b/mssql/compiler.py @@ -307,6 +307,12 @@ def as_sql(self, with_limits=True, with_col_aliases=False): params.extend(o_params) result.append('ORDER BY %s' % ', '.join(ordering)) + # For subqueres with an ORDER BY clause, SQL Server also + # requires a TOP or OFFSET clause which is not generated for + # Django 2.x. See https://github.com/microsoft/mssql-django/issues/12 + if django.VERSION < (3, 0, 0) and not (do_offset or do_limit): + result.append("OFFSET 0 ROWS") + # SQL Server requires the backend-specific emulation (2008 or earlier) # or an offset clause (2012 or newer) for offsetting if do_offset: From c373dfa2e81d19e45b1d6999b6f67b8b67e7545b Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Wed, 7 Apr 2021 17:31:22 -0700 Subject: [PATCH 11/41] Disable check for existing data after enable constraint checking Django expect database not to check existing data after enable constraint checking, This also fixed two errors in backends unit test. --- mssql/base.py | 2 +- testapp/settings.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/mssql/base.py b/mssql/base.py index ea307ba0..28923440 100644 --- a/mssql/base.py +++ b/mssql/base.py @@ -494,7 +494,7 @@ def disable_constraint_checking(self): def enable_constraint_checking(self): if not self.needs_rollback: - self.cursor().execute('EXEC sp_msforeachtable "ALTER TABLE ? WITH CHECK CHECK CONSTRAINT ALL"') + self.cursor().execute('EXEC sp_msforeachtable "ALTER TABLE ? WITH NOCHECK CHECK CONSTRAINT ALL"') class CursorWrapper(object): diff --git a/testapp/settings.py b/testapp/settings.py index ccc57c77..ef14ac72 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -101,8 +101,6 @@ 'ordering.tests.OrderingTests.test_deprecated_values_annotate', 'queries.test_qs_combinators.QuerySetSetOperationTests.test_limits', 'backends.tests.BackendTestCase.test_unicode_password', - 'backends.tests.FkConstraintsTests.test_disable_constraint_checks_context_manager', - 'backends.tests.FkConstraintsTests.test_disable_constraint_checks_manually', 'introspection.tests.IntrospectionTests.test_get_table_description_types', 'migrations.test_commands.MigrateTests.test_migrate_syncdb_app_label', 'migrations.test_commands.MigrateTests.test_migrate_syncdb_deferred_sql_executed_with_schemaeditor', From 2e20cad6f694a34405bdff48b26be5a8a3fb0f3a Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Wed, 7 Apr 2021 18:03:12 -0700 Subject: [PATCH 12/41] Issue #23 -- Change the default driver to match the README file --- mssql/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mssql/base.py b/mssql/base.py index 28923440..9cc78d00 100644 --- a/mssql/base.py +++ b/mssql/base.py @@ -252,7 +252,7 @@ def get_new_connection(self, conn_params): trusted_connection = conn_params.get('Trusted_Connection', 'yes') options = conn_params.get('OPTIONS', {}) - driver = options.get('driver', 'ODBC Driver 13 for SQL Server') + driver = options.get('driver', 'ODBC Driver 17 for SQL Server') dsn = options.get('dsn', None) # Microsoft driver names assumed here are: From 06417e89da0fbac8ff04b4e97bc79dcba189c6f5 Mon Sep 17 00:00:00 2001 From: Karl Vernet Date: Mon, 12 Apr 2021 15:21:40 +0200 Subject: [PATCH 13/41] Avoid using non documented stored procedure sp_msforeachtable --- mssql/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mssql/base.py b/mssql/base.py index 9cc78d00..328272c9 100644 --- a/mssql/base.py +++ b/mssql/base.py @@ -489,12 +489,12 @@ def check_constraints(self, table_names=None): def disable_constraint_checking(self): if not self.needs_rollback: - self.cursor().execute('EXEC sp_msforeachtable "ALTER TABLE ? NOCHECK CONSTRAINT ALL"') + self._execute_foreach('ALTER TABLE %s NOCHECK CONSTRAINT ALL') return not self.needs_rollback def enable_constraint_checking(self): if not self.needs_rollback: - self.cursor().execute('EXEC sp_msforeachtable "ALTER TABLE ? WITH NOCHECK CHECK CONSTRAINT ALL"') + self._execute_foreach('ALTER TABLE %s WITH NOCHECK CHECK CONSTRAINT ALL') class CursorWrapper(object): From c7e9d3f86c74031f2e3e1e5cc29f1b64510c1c1e Mon Sep 17 00:00:00 2001 From: David Hotham Date: Mon, 12 Apr 2021 22:34:50 +0100 Subject: [PATCH 14/41] Declare dependencies explicitly In particular, django<3.2 until that release is supported --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 55fd3d42..92e4fe22 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,9 @@ license='BSD', packages=find_packages(), install_requires=[ + 'django>=2.2,<3.2', 'pyodbc>=3.0', + 'pytz', ], classifiers=CLASSIFIERS, keywords='django', From c2fbcad82e6263efe338b5fbfe82f3cf8aed445a Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Thu, 15 Apr 2021 11:08:15 -0700 Subject: [PATCH 15/41] Add basic support for JSONField Initial support for Json, does not have lookups like `contains` and `has` yet. --- .gitignore | 2 +- mssql/base.py | 1 + mssql/compiler.py | 9 ++++++++- mssql/features.py | 14 +++++++++++--- mssql/functions.py | 17 ++++++++++++++--- setup.py | 1 + testapp/settings.py | 28 +++++++++++++++++++++++++++- 7 files changed, 63 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 0c91415f..1377bf26 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ Thumbs.db *.egg-info - +*.dll tests/local_settings.py # Virtual Env diff --git a/mssql/base.py b/mssql/base.py index 9cc78d00..b5dd54a1 100644 --- a/mssql/base.py +++ b/mssql/base.py @@ -88,6 +88,7 @@ class DatabaseWrapper(BaseDatabaseWrapper): 'IntegerField': 'int', 'IPAddressField': 'nvarchar(15)', 'GenericIPAddressField': 'nvarchar(39)', + 'JSONField': 'nvarchar(max)', 'NullBooleanField': 'bit', 'OneToOneField': 'int', 'PositiveIntegerField': 'int', diff --git a/mssql/compiler.py b/mssql/compiler.py index 9df8081e..b76b3c69 100644 --- a/mssql/compiler.py +++ b/mssql/compiler.py @@ -13,7 +13,7 @@ from django.db.models.sql import compiler from django.db.transaction import TransactionManagementError from django.db.utils import NotSupportedError - +from django.db.models.fields.json import compile_json_path, KeyTransform as json_KeyTransform def _as_sql_agv(self, compiler, connection): return self.as_sql(compiler, connection, template='%(function)s(CONVERT(float, %(field)s))') @@ -42,6 +42,11 @@ def _as_sql_greatest(self, compiler, connection): template = '(SELECT MAX(value) FROM (VALUES (%(expressions)s)) AS _%(function)s(value))' return self.as_sql(compiler, connection, arg_joiner='), (', template=template) +def _as_sql_json_keytransform(self, compiler, connection): + lhs, params, key_transforms = self.preprocess_lhs(compiler, connection) + json_path = compile_json_path(key_transforms) + + return 'JSON_VALUE(%s, %%s)' % lhs, tuple(params) + (json_path,) def _as_sql_least(self, compiler, connection): # SQL Server does not provide LEAST function, @@ -382,6 +387,8 @@ def _as_microsoft(self, node): as_microsoft = _as_sql_count elif isinstance(node, Greatest): as_microsoft = _as_sql_greatest + if isinstance(node, json_KeyTransform): + as_microsoft = _as_sql_json_keytransform elif isinstance(node, Least): as_microsoft = _as_sql_least elif isinstance(node, Length): diff --git a/mssql/features.py b/mssql/features.py index e186143d..d1562bdd 100644 --- a/mssql/features.py +++ b/mssql/features.py @@ -6,21 +6,24 @@ class DatabaseFeatures(BaseDatabaseFeatures): - has_native_uuid_field = False allow_sliced_subqueries_with_in = False can_introspect_autofield = True + can_introspect_json_field = False can_introspect_small_integer_field = True can_return_columns_from_insert = True can_return_id_from_insert = True can_use_chunked_reads = False for_update_after_from = True greatest_least_ignores_nulls = True + has_json_operators = False + has_native_json_field = False + has_native_uuid_field = False has_real_datatype = True has_select_for_update = True has_select_for_update_nowait = True has_select_for_update_skip_locked = True - ignores_table_name_case = True ignores_quoted_identifier_case = True + ignores_table_name_case = True order_by_nulls_first = True requires_literal_defaults = True requires_sqlparse_for_splitting = False @@ -28,9 +31,10 @@ class DatabaseFeatures(BaseDatabaseFeatures): supports_deferrable_unique_constraints = False supports_ignore_conflicts = False supports_index_on_text_field = False - supports_json_field = False + supports_json_field_contains = False supports_order_by_nulls_modifier = False supports_paramstyle_pyformat = False + supports_primitives_in_json_field = False supports_regex_backreferencing = True supports_sequence_reset = False supports_subqueries_in_group_by = False @@ -65,3 +69,7 @@ def has_zoneinfo_database(self): with self.connection.cursor() as cursor: cursor.execute("SELECT TOP 1 1 FROM sys.time_zone_info") return cursor.fetchone() is not None + + @cached_property + def supports_json_field(self): + return self.connection.sql_server_version >= 2016 diff --git a/mssql/functions.py b/mssql/functions.py index b1a5b99f..1f50c6cf 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -6,7 +6,8 @@ from django.db.models.functions import Cast from django.db.models.functions.math import ATan2, Log, Ln, Mod, Round from django.db.models.expressions import Case, Exists, OrderBy, When -from django.db.models.lookups import Lookup, In +from django.db.models.lookups import Lookup, In, Exact +from django.db.models.fields.json import KeyTransform, KeyTransformExact DJANGO3 = VERSION[0] >= 3 @@ -108,12 +109,22 @@ def split_parameter_list_as_sql(self, compiler, connection): return in_clause, () +def KeyTransformExact_process_rhs(self, compiler, connection): + if isinstance(self.rhs, KeyTransform): + return super(Exact, self).process_rhs(compiler, connection) + rhs, rhs_params = super(Exact, self).process_rhs(compiler, connection) + if connection.vendor == 'microsoft': + if rhs_params != [None]: + rhs_params = [params.strip('"') for params in rhs_params] + return rhs, rhs_params + ATan2.as_microsoft = sqlserver_atan2 -Log.as_microsoft = sqlserver_log +In.split_parameter_list_as_sql = split_parameter_list_as_sql +KeyTransformExact.process_rhs = KeyTransformExact_process_rhs Ln.as_microsoft = sqlserver_ln +Log.as_microsoft = sqlserver_log Mod.as_microsoft = sqlserver_mod Round.as_microsoft = sqlserver_round -In.split_parameter_list_as_sql = split_parameter_list_as_sql if DJANGO3: Lookup.as_microsoft = sqlserver_lookup diff --git a/setup.py b/setup.py index 55fd3d42..49e9e824 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ install_requires=[ 'pyodbc>=3.0', ], + package_data={'mssql': ['regex_clr.dll']}, classifiers=CLASSIFIERS, keywords='django', ) diff --git a/testapp/settings.py b/testapp/settings.py index ef14ac72..15694aba 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -186,7 +186,33 @@ 'queries.test_db_returning.ReturningValuesTests.test_insert_returning_multiple', 'dbshell.tests.DbshellCommandTestCase.test_command_missing', 'schema.tests.SchemaTests.test_char_field_pk_to_auto_field', - 'datetimes.tests.DateTimesTests.test_21432' + 'datetimes.tests.DateTimesTests.test_21432', + + #JSONFields + 'model_fields.test_jsonfield.TestQuerying.test_has_any_keys', + 'model_fields.test_jsonfield.TestQuerying.test_has_key', + 'model_fields.test_jsonfield.TestQuerying.test_has_key_deep', + 'model_fields.test_jsonfield.TestQuerying.test_has_key_list', + 'model_fields.test_jsonfield.TestQuerying.test_has_key_null_value', + 'model_fields.test_jsonfield.TestQuerying.test_has_keys', + 'model_fields.test_jsonfield.TestQuerying.test_key_iregex', + 'model_fields.test_jsonfield.TestQuerying.test_key_quoted_string', + 'model_fields.test_jsonfield.TestQuerying.test_key_regex', + 'model_fields.test_jsonfield.TestQuerying.test_lookups_with_key_transform', + 'model_fields.test_jsonfield.TestQuerying.test_order_grouping_custom_decoder', + 'model_fields.test_jsonfield.TestQuerying.test_ordering_grouping_by_count', + 'model_fields.test_jsonfield.TestQuerying.test_ordering_grouping_by_key_transform', + 'model_fields.test_jsonfield.JSONFieldTests.test_db_check_constraints', + 'model_fields.test_jsonfield.TestQuerying.test_isnull_key', + 'model_fields.test_jsonfield.TestQuerying.test_key_in', + 'model_fields.test_jsonfield.TestQuerying.test_key_transform_expression', + 'model_fields.test_jsonfield.TestQuerying.test_key_values', + 'model_fields.test_jsonfield.TestQuerying.test_nested_key_transform_expression', + 'model_fields.test_jsonfield.TestQuerying.test_none_key', + 'model_fields.test_jsonfield.TestQuerying.test_none_key_and_exact_lookup', + 'model_fields.test_jsonfield.TestQuerying.test_none_key_exclude', + 'model_fields.test_jsonfield.TestQuerying.test_ordering_by_transform', + 'model_fields.test_jsonfield.TestQuerying.test_shallow_lookup_obj_target' ] REGEX_TESTS = ['lookup.tests.LookupTests.test_regex', From 6fe445a8399970a10b546ce9ae25675dc4ec3926 Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Thu, 15 Apr 2021 13:03:12 -0700 Subject: [PATCH 16/41] Issue #25 -- Fix Django 3.2 startup issue --- mssql/compiler.py | 8 +++++--- mssql/functions.py | 6 ++++-- mssql/schema.py | 12 ++++++++---- testapp/settings.py | 2 ++ tox.ini | 2 ++ 5 files changed, 21 insertions(+), 9 deletions(-) diff --git a/mssql/compiler.py b/mssql/compiler.py index b76b3c69..a61b5999 100644 --- a/mssql/compiler.py +++ b/mssql/compiler.py @@ -13,7 +13,8 @@ from django.db.models.sql import compiler from django.db.transaction import TransactionManagementError from django.db.utils import NotSupportedError -from django.db.models.fields.json import compile_json_path, KeyTransform as json_KeyTransform +if django.VERSION >= (3, 1): + from django.db.models.fields.json import compile_json_path, KeyTransform as json_KeyTransform def _as_sql_agv(self, compiler, connection): return self.as_sql(compiler, connection, template='%(function)s(CONVERT(float, %(field)s))') @@ -387,8 +388,6 @@ def _as_microsoft(self, node): as_microsoft = _as_sql_count elif isinstance(node, Greatest): as_microsoft = _as_sql_greatest - if isinstance(node, json_KeyTransform): - as_microsoft = _as_sql_json_keytransform elif isinstance(node, Least): as_microsoft = _as_sql_least elif isinstance(node, Length): @@ -409,6 +408,9 @@ def _as_microsoft(self, node): as_microsoft = _as_sql_trim elif isinstance(node, Variance): as_microsoft = _as_sql_variance + if django.VERSION >= (3, 1): + if isinstance(node, json_KeyTransform): + as_microsoft = _as_sql_json_keytransform if as_microsoft: node = node.copy() node.as_microsoft = types.MethodType(as_microsoft, node) diff --git a/mssql/functions.py b/mssql/functions.py index 1f50c6cf..43b11243 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -7,7 +7,8 @@ from django.db.models.functions.math import ATan2, Log, Ln, Mod, Round from django.db.models.expressions import Case, Exists, OrderBy, When from django.db.models.lookups import Lookup, In, Exact -from django.db.models.fields.json import KeyTransform, KeyTransformExact +if VERSION >= (3, 1): + from django.db.models.fields.json import KeyTransform, KeyTransformExact DJANGO3 = VERSION[0] >= 3 @@ -120,7 +121,8 @@ def KeyTransformExact_process_rhs(self, compiler, connection): ATan2.as_microsoft = sqlserver_atan2 In.split_parameter_list_as_sql = split_parameter_list_as_sql -KeyTransformExact.process_rhs = KeyTransformExact_process_rhs +if VERSION >= (3, 1): + KeyTransformExact.process_rhs = KeyTransformExact_process_rhs Ln.as_microsoft = sqlserver_ln Log.as_microsoft = sqlserver_log Mod.as_microsoft = sqlserver_mod diff --git a/mssql/schema.py b/mssql/schema.py index b7b27c4b..6d151c15 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -235,6 +235,7 @@ def _db_table_delete_constraint_sql(self, template, db_table, name): template, table=Table(db_table, self.quote_name), name=self.quote_name(name), + include='' ) def alter_db_table(self, model, old_db_table, new_db_table): @@ -689,7 +690,7 @@ def add_field(self, model, field): if self.connection.features.connection_persists_old_columns: self.connection.close() - def _create_unique_sql(self, model, columns, name=None, condition=None, deferrable=None): + def _create_unique_sql(self, model, columns, name=None, condition=None, deferrable=None, include=None, opclasses=None): if (deferrable and not getattr(self.connection.features, 'supports_deferrable_unique_constraints', False)): return None @@ -713,7 +714,8 @@ def create_unique_name(*args, **kwargs): name=name, columns=columns, condition=' WHERE ' + condition, - **statement_args + **statement_args, + include='', ) if self.connection.features.supports_partial_indexes else None else: return Statement( @@ -721,12 +723,13 @@ def create_unique_name(*args, **kwargs): table=table, name=name, columns=columns, - **statement_args + **statement_args, + include='', ) def _create_index_sql(self, model, fields, *, name=None, suffix='', using='', db_tablespace=None, col_suffixes=(), sql=None, opclasses=(), - condition=None): + condition=None, include=None, expressions=None): """ Return the SQL statement to create the index for one or several fields. `sql` can be specified if the syntax differs from the standard (GIS @@ -751,6 +754,7 @@ def create_index_name(*args, **kwargs): columns=self._index_columns(table, columns, col_suffixes, opclasses), extra=tablespace_sql, condition=(' WHERE ' + condition) if condition else '', + include='' ) def create_model(self, model): diff --git a/testapp/settings.py b/testapp/settings.py index 15694aba..53901cd4 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -36,6 +36,8 @@ 'django.contrib.auth.hashers.PBKDF2PasswordHasher', ] +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' + ENABLE_REGEX_TESTS = False TEST_RUNNER = "testapp.runners.ExcludedTestSuiteRunner" diff --git a/tox.ini b/tox.ini index 19e2d25e..cef93c0f 100644 --- a/tox.ini +++ b/tox.ini @@ -3,6 +3,7 @@ envlist = {py36,py37}-django22, {py36,py37,py38}-django30, {py36,py37,py38}-django31, + {py36,py37,py38,py39}-django32 [testenv] allowlist_externals = @@ -17,3 +18,4 @@ deps = django22: django==2.2.* django30: django>=3.0,<3.1 django31: django>=3.1,<3.2 + django32: django==3.2.* From 13f79cb58e577a7af1ba3bab1f0a151850cd3838 Mon Sep 17 00:00:00 2001 From: Daniel Hillier Date: Tue, 20 Apr 2021 10:25:46 +1000 Subject: [PATCH 17/41] Increase max bulk_batch_size The previous limit was a max of 1000 query parameters. This is changed to a max of 1000 rows (the max number of rows allowed for inserting in a single statement by mssql) or however many rows can be inserted while keeping the number of query parameters below 2050 per insert statement (mssql reports a max allowed of 2100 parameters but a few parameters are reserved for executing the query). --- mssql/operations.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/mssql/operations.py b/mssql/operations.py index 5f8ebb47..3c797659 100644 --- a/mssql/operations.py +++ b/mssql/operations.py @@ -49,12 +49,20 @@ def bulk_batch_size(self, fields, objs): are the fields going to be inserted in the batch, the objs contains all the objects to be inserted. """ - objs_len, fields_len, max_row_values = len(objs), len(fields), 1000 - if (objs_len * fields_len) <= max_row_values: - size = objs_len - else: - size = max_row_values // fields_len - return size + max_insert_rows = 1000 + fields_len = len(fields) + if fields_len == 0: + # Required for empty model + # (bulk_create.tests.BulkCreateTests.test_empty_model) + return max_insert_rows + + # MSSQL allows a query to have 2100 parameters but some parameters are + # taken up defining `NVARCHAR` parameters to store the query text and + # query parameters for the `sp_executesql` call. This should only take + # up 2 parameters but I've had this error when sending 2098 parameters. + max_query_params = 2050 + # inserts are capped at 1000 rows regardless of number of query params. + return min(max_insert_rows, max_query_params // fields_len) def bulk_insert_sql(self, fields, placeholder_rows): placeholder_rows_sql = (", ".join(row) for row in placeholder_rows) From 6cf15c9176c8daa99062977f0eef765c2082286d Mon Sep 17 00:00:00 2001 From: David Beitey Date: Tue, 20 Apr 2021 16:17:02 +1000 Subject: [PATCH 18/41] Avoid invalid SQL syntax from OrderBy.as_sql By making a copy of the OrderBy expression and disabling its nulls_first/nulls_last flags, this prevents Django's core OrderBy.as_sql function from modifying the custom templates SQL Server requires. See for details on why this isn't implemented as a feature flag. Fixes #31 --- mssql/functions.py | 15 +++++++++------ testapp/tests/test_expressions.py | 29 ++++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/mssql/functions.py b/mssql/functions.py index b1a5b99f..f6b20b94 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -71,24 +71,27 @@ def sqlserver_lookup(self, compiler, connection): def sqlserver_orderby(self, compiler, connection): - # MSSQL doesn't allow ORDER BY EXISTS() unless it's wrapped in - # a CASE WHEN. - template = None if self.nulls_last: template = 'CASE WHEN %(expression)s IS NULL THEN 1 ELSE 0 END, %(expression)s %(ordering)s' if self.nulls_first: template = 'CASE WHEN %(expression)s IS NULL THEN 0 ELSE 1 END, %(expression)s %(ordering)s' + copy = self.copy() + + # Prevent OrderBy.as_sql() from modifying supplied templates + copy.nulls_first = False + copy.nulls_last = False + + # MSSQL doesn't allow ORDER BY EXISTS() unless it's wrapped in a CASE WHEN. if isinstance(self.expression, Exists): - copy = self.copy() copy.expression = Case( When(self.expression, then=True), default=False, output_field=BooleanField(), ) - return copy.as_sql(compiler, connection, template=template) - return self.as_sql(compiler, connection, template=template) + + return copy.as_sql(compiler, connection, template=template) def split_parameter_list_as_sql(self, compiler, connection): diff --git a/testapp/tests/test_expressions.py b/testapp/tests/test_expressions.py index e89a0def..a23f68b2 100644 --- a/testapp/tests/test_expressions.py +++ b/testapp/tests/test_expressions.py @@ -4,11 +4,12 @@ from unittest import skipUnless from django import VERSION -from django.db.models import IntegerField +from django.db.models import IntegerField, F from django.db.models.expressions import Case, Exists, OuterRef, Subquery, Value, When -from django.test import TestCase +from django.test import TestCase, skipUnlessDBFeature -from ..models import Author, Comment, Post + +from ..models import Author, Comment, Post, Editor DJANGO3 = VERSION[0] >= 3 @@ -54,3 +55,25 @@ def test_order_by_exists(self): authors_by_posts = Author.objects.order_by(Exists(Post.objects.filter(author=OuterRef('pk'))).asc()) self.assertSequenceEqual(authors_by_posts, [author_without_posts, self.author]) + + +@skipUnless(DJANGO3, "Django 3 specific tests") +@skipUnlessDBFeature("order_by_nulls_first") +class TestOrderBy(TestCase): + def setUp(self): + self.author = Author.objects.create(name="author") + self.post = Post.objects.create(title="foo", author=self.author) + self.editor = Editor.objects.create(name="editor") + self.post_alt = Post.objects.create(title="Post with editor", author=self.author, alt_editor=self.editor) + + def test_order_by_nulls_last(self): + results = Post.objects.order_by(F("alt_editor").asc(nulls_last=True)).all() + self.assertEqual(len(results), 2) + self.assertIsNotNone(results[0].alt_editor) + self.assertIsNone(results[1].alt_editor) + + def test_order_by_nulls_first(self): + results = Post.objects.order_by(F("alt_editor").desc(nulls_first=True)).all() + self.assertEqual(len(results), 2) + self.assertIsNone(results[0].alt_editor) + self.assertIsNotNone(results[1].alt_editor) From 6bf76554921597a35cae78254b930a06d04bbac1 Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Wed, 28 Apr 2021 18:39:47 -0700 Subject: [PATCH 19/41] Fix JSONField KeyTransformIn, KeyTransformExact --- azure-pipelines.yml | 1 + mssql/compiler.py | 6 ++++-- mssql/functions.py | 36 ++++++++++++++++++++++++++---------- testapp/settings.py | 9 +-------- 4 files changed, 32 insertions(+), 20 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3a70cd6e..611d0db3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,6 +17,7 @@ jobs: - job: Windows pool: Django-agent-pool + timeoutInMinutes: 120 strategy: matrix: diff --git a/mssql/compiler.py b/mssql/compiler.py index a61b5999..56e53fec 100644 --- a/mssql/compiler.py +++ b/mssql/compiler.py @@ -46,8 +46,10 @@ def _as_sql_greatest(self, compiler, connection): def _as_sql_json_keytransform(self, compiler, connection): lhs, params, key_transforms = self.preprocess_lhs(compiler, connection) json_path = compile_json_path(key_transforms) - - return 'JSON_VALUE(%s, %%s)' % lhs, tuple(params) + (json_path,) + return ( + "COALESCE(JSON_QUERY(%s, '%s'), JSON_VALUE(%s, '%s'))" % + ((lhs, json_path) * 2) + ), tuple(params) * 2 def _as_sql_least(self, compiler, connection): # SQL Server does not provide LEAST function, diff --git a/mssql/functions.py b/mssql/functions.py index 43b11243..ce1e9578 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -1,14 +1,18 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import json + from django import VERSION from django.db.models import BooleanField from django.db.models.functions import Cast from django.db.models.functions.math import ATan2, Log, Ln, Mod, Round from django.db.models.expressions import Case, Exists, OrderBy, When -from django.db.models.lookups import Lookup, In, Exact +from django.db.models.lookups import Lookup, In +from django.db.models import lookups + if VERSION >= (3, 1): - from django.db.models.fields.json import KeyTransform, KeyTransformExact + from django.db.models.fields.json import KeyTransform, KeyTransformIn, KeyTransformExact DJANGO3 = VERSION[0] >= 3 @@ -110,19 +114,31 @@ def split_parameter_list_as_sql(self, compiler, connection): return in_clause, () -def KeyTransformExact_process_rhs(self, compiler, connection): +def unquote_json_rhs(rhs_params): + for value in rhs_params: + value = json.loads(value) + if not isinstance(value, (list, dict)): + rhs_params = [param.replace('"', '') for param in rhs_params] + return rhs_params + +def json_KeyTransformExact_process_rhs(self, compiler, connection): if isinstance(self.rhs, KeyTransform): - return super(Exact, self).process_rhs(compiler, connection) - rhs, rhs_params = super(Exact, self).process_rhs(compiler, connection) - if connection.vendor == 'microsoft': - if rhs_params != [None]: - rhs_params = [params.strip('"') for params in rhs_params] - return rhs, rhs_params + return super(lookups.Exact, self).process_rhs(compiler, connection) + rhs, rhs_params = super(KeyTransformExact, self).process_rhs(compiler, connection) + + return rhs, unquote_json_rhs(rhs_params) + +def json_KeyTransformIn(self, compiler, connection): + lhs, _ = super(KeyTransformIn, self).process_lhs(compiler, connection) + rhs, rhs_params = super(KeyTransformIn, self).process_rhs(compiler, connection) + + return (lhs + ' IN ' + rhs, unquote_json_rhs(rhs_params)) ATan2.as_microsoft = sqlserver_atan2 In.split_parameter_list_as_sql = split_parameter_list_as_sql if VERSION >= (3, 1): - KeyTransformExact.process_rhs = KeyTransformExact_process_rhs + KeyTransformIn.as_microsoft = json_KeyTransformIn + KeyTransformExact.process_rhs = json_KeyTransformExact_process_rhs Ln.as_microsoft = sqlserver_ln Log.as_microsoft = sqlserver_log Mod.as_microsoft = sqlserver_mod diff --git a/testapp/settings.py b/testapp/settings.py index 53901cd4..0be1cad5 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -201,20 +201,13 @@ 'model_fields.test_jsonfield.TestQuerying.test_key_quoted_string', 'model_fields.test_jsonfield.TestQuerying.test_key_regex', 'model_fields.test_jsonfield.TestQuerying.test_lookups_with_key_transform', - 'model_fields.test_jsonfield.TestQuerying.test_order_grouping_custom_decoder', 'model_fields.test_jsonfield.TestQuerying.test_ordering_grouping_by_count', - 'model_fields.test_jsonfield.TestQuerying.test_ordering_grouping_by_key_transform', 'model_fields.test_jsonfield.JSONFieldTests.test_db_check_constraints', 'model_fields.test_jsonfield.TestQuerying.test_isnull_key', - 'model_fields.test_jsonfield.TestQuerying.test_key_in', - 'model_fields.test_jsonfield.TestQuerying.test_key_transform_expression', - 'model_fields.test_jsonfield.TestQuerying.test_key_values', - 'model_fields.test_jsonfield.TestQuerying.test_nested_key_transform_expression', 'model_fields.test_jsonfield.TestQuerying.test_none_key', 'model_fields.test_jsonfield.TestQuerying.test_none_key_and_exact_lookup', - 'model_fields.test_jsonfield.TestQuerying.test_none_key_exclude', 'model_fields.test_jsonfield.TestQuerying.test_ordering_by_transform', - 'model_fields.test_jsonfield.TestQuerying.test_shallow_lookup_obj_target' + 'model_fields.test_jsonfield.TestQuerying.test_key_escape', ] REGEX_TESTS = ['lookup.tests.LookupTests.test_regex', From 0153047808842dfb23c538b88e08bf5d118fc432 Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Mon, 3 May 2021 16:38:53 -0700 Subject: [PATCH 20/41] Add Python 3.9 to pipeline for Django 3.0 and 3.1 --- azure-pipelines.yml | 12 ++++++++++++ tox.ini | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 611d0db3..59dc206f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,6 +21,9 @@ jobs: strategy: matrix: + Python 3.9 - Django 3.1: + python.version: '3.9' + tox.env: 'py39-django31' Python 3.8 - Django 3.1: python.version: '3.8' tox.env: 'py38-django31' @@ -31,6 +34,9 @@ jobs: python.version: '3.6' tox.env: 'py36-django31' + Python 3.9 - Django 3.0: + python.version: '3.9' + tox.env: 'py39-django30' Python 3.8 - Django 3.0: python.version: '3.8' tox.env: 'py38-django30' @@ -77,6 +83,9 @@ jobs: strategy: matrix: + Python 3.9 - Django 3.1: + python.version: '3.9' + tox.env: 'py39-django31' Python 3.8 - Django 3.1: python.version: '3.8' tox.env: 'py38-django31' @@ -87,6 +96,9 @@ jobs: python.version: '3.6' tox.env: 'py36-django31' + Python 3.9 - Django 3.0: + python.version: '3.9' + tox.env: 'py39-django30' Python 3.8 - Django 3.0: python.version: '3.8' tox.env: 'py38-django30' diff --git a/tox.ini b/tox.ini index cef93c0f..c47c74e8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] envlist = {py36,py37}-django22, - {py36,py37,py38}-django30, - {py36,py37,py38}-django31, + {py36,py37,py38,py39}-django30, + {py36,py37,py38,py39}-django31, {py36,py37,py38,py39}-django32 [testenv] From 6c921039b4a7ada1fc862e6cb373e93d1d9b57b1 Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Thu, 6 May 2021 15:53:27 +0000 Subject: [PATCH 21/41] Fix JSONField has key lookup --- mssql/functions.py | 35 ++++++++++++++++++++++++++++++++++- testapp/settings.py | 13 +++++-------- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/mssql/functions.py b/mssql/functions.py index ce1e9578..ca8a0029 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -12,7 +12,9 @@ from django.db.models import lookups if VERSION >= (3, 1): - from django.db.models.fields.json import KeyTransform, KeyTransformIn, KeyTransformExact + from django.db.models.fields.json import ( + KeyTransform, KeyTransformIn, KeyTransformExact, + HasKeyLookup, compile_json_path) DJANGO3 = VERSION[0] >= 3 @@ -134,11 +136,42 @@ def json_KeyTransformIn(self, compiler, connection): return (lhs + ' IN ' + rhs, unquote_json_rhs(rhs_params)) +def json_HasKeyLookup(self, compiler, connection): + # Process JSON path from the left-hand side. + if isinstance(self.lhs, KeyTransform): + lhs, _, lhs_key_transforms = self.lhs.preprocess_lhs(compiler, connection) + lhs_json_path = compile_json_path(lhs_key_transforms) + else: + lhs, _ = self.process_lhs(compiler, connection) + lhs_json_path = '$' + sql = lhs + ' IN (SELECT ' + lhs + ' FROM ' + self.lhs.output_field.model._meta.db_table + \ + ' CROSS APPLY OPENJSON(' + lhs + ') WITH ( [json_path_value] char(1) \'%s\') WHERE [json_path_value] IS NOT NULL)' + # Process JSON path from the right-hand side. + rhs = self.rhs + rhs_params = [] + if not isinstance(rhs, (list, tuple)): + rhs = [rhs] + for key in rhs: + if isinstance(key, KeyTransform): + *_, rhs_key_transforms = key.preprocess_lhs(compiler, connection) + else: + rhs_key_transforms = [key] + rhs_params.append('%s%s' % ( + lhs_json_path, + compile_json_path(rhs_key_transforms, include_root=False), + )) + # Add condition for each key. + if self.logical_operator: + sql = '(%s)' % self.logical_operator.join([sql] * len(rhs_params)) + + return sql % tuple(rhs_params), [] + ATan2.as_microsoft = sqlserver_atan2 In.split_parameter_list_as_sql = split_parameter_list_as_sql if VERSION >= (3, 1): KeyTransformIn.as_microsoft = json_KeyTransformIn KeyTransformExact.process_rhs = json_KeyTransformExact_process_rhs + HasKeyLookup.as_microsoft = json_HasKeyLookup Ln.as_microsoft = sqlserver_ln Log.as_microsoft = sqlserver_log Mod.as_microsoft = sqlserver_mod diff --git a/testapp/settings.py b/testapp/settings.py index 0be1cad5..c46c630a 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -191,28 +191,25 @@ 'datetimes.tests.DateTimesTests.test_21432', #JSONFields - 'model_fields.test_jsonfield.TestQuerying.test_has_any_keys', - 'model_fields.test_jsonfield.TestQuerying.test_has_key', - 'model_fields.test_jsonfield.TestQuerying.test_has_key_deep', 'model_fields.test_jsonfield.TestQuerying.test_has_key_list', 'model_fields.test_jsonfield.TestQuerying.test_has_key_null_value', - 'model_fields.test_jsonfield.TestQuerying.test_has_keys', - 'model_fields.test_jsonfield.TestQuerying.test_key_iregex', 'model_fields.test_jsonfield.TestQuerying.test_key_quoted_string', - 'model_fields.test_jsonfield.TestQuerying.test_key_regex', 'model_fields.test_jsonfield.TestQuerying.test_lookups_with_key_transform', 'model_fields.test_jsonfield.TestQuerying.test_ordering_grouping_by_count', 'model_fields.test_jsonfield.JSONFieldTests.test_db_check_constraints', 'model_fields.test_jsonfield.TestQuerying.test_isnull_key', 'model_fields.test_jsonfield.TestQuerying.test_none_key', 'model_fields.test_jsonfield.TestQuerying.test_none_key_and_exact_lookup', - 'model_fields.test_jsonfield.TestQuerying.test_ordering_by_transform', 'model_fields.test_jsonfield.TestQuerying.test_key_escape', + + 'model_fields.test_jsonfield.TestQuerying.test_ordering_by_transform', ] REGEX_TESTS = ['lookup.tests.LookupTests.test_regex', 'lookup.tests.LookupTests.test_regex_backreferencing', 'lookup.tests.LookupTests.test_regex_non_ascii', 'lookup.tests.LookupTests.test_regex_non_string', - 'lookup.tests.LookupTests.test_regex_null' + 'lookup.tests.LookupTests.test_regex_null', + 'model_fields.test_jsonfield.TestQuerying.test_key_iregex', + 'model_fields.test_jsonfield.TestQuerying.test_key_regex', ] From 7b7ed3115b7fbdccc2fb7f3962a583d361cb6cd4 Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Tue, 11 May 2021 18:35:54 +0000 Subject: [PATCH 22/41] Edit README.md for JSONField --- README.md | 4 +++- mssql/functions.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1e225306..8386a3db 100644 --- a/README.md +++ b/README.md @@ -222,7 +222,9 @@ Here is an example of the database settings: The following features are currently not supported: - mssql-django does not support SQL-based regex commands -- Altering a model field from or to AutoField at migration +- Altering a model field from or to AutoField at migration + +Certain limitations for JSONField lookups, more details [here](https://github.com/microsoft/mssql-django/wiki/JSONField). ## Future Plans diff --git a/mssql/functions.py b/mssql/functions.py index 1619ce2a..d22eca2e 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -169,6 +169,7 @@ def json_HasKeyLookup(self, compiler, connection): return sql % tuple(rhs_params), [] + ATan2.as_microsoft = sqlserver_atan2 In.split_parameter_list_as_sql = split_parameter_list_as_sql if VERSION >= (3, 1): From 0c13927a3c24df9f5b941db404c610edf7d4379e Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Wed, 12 May 2021 21:47:29 +0000 Subject: [PATCH 23/41] Add JSONField check constraints --- mssql/base.py | 1 + testapp/settings.py | 1 - tox.ini | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mssql/base.py b/mssql/base.py index 1b824210..963c963c 100644 --- a/mssql/base.py +++ b/mssql/base.py @@ -106,6 +106,7 @@ class DatabaseWrapper(BaseDatabaseWrapper): 'SmallAutoField': 'IDENTITY (1, 1)', } data_type_check_constraints = { + 'JSONField': '(ISJSON ("%(column)s") = 1)', 'PositiveIntegerField': '[%(column)s] >= 0', 'PositiveSmallIntegerField': '[%(column)s] >= 0', } diff --git a/testapp/settings.py b/testapp/settings.py index c46c630a..626fa2c3 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -196,7 +196,6 @@ 'model_fields.test_jsonfield.TestQuerying.test_key_quoted_string', 'model_fields.test_jsonfield.TestQuerying.test_lookups_with_key_transform', 'model_fields.test_jsonfield.TestQuerying.test_ordering_grouping_by_count', - 'model_fields.test_jsonfield.JSONFieldTests.test_db_check_constraints', 'model_fields.test_jsonfield.TestQuerying.test_isnull_key', 'model_fields.test_jsonfield.TestQuerying.test_none_key', 'model_fields.test_jsonfield.TestQuerying.test_none_key_and_exact_lookup', diff --git a/tox.ini b/tox.ini index c47c74e8..3c968719 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ envlist = [testenv] allowlist_externals = /bin/bash + /usr/bin/bash C:\Program Files\Git\bin\bash.EXE commands = From 4dfb39c028dcdf3a1d6f5f5f10bc3404a333fdde Mon Sep 17 00:00:00 2001 From: Warren Chu <60903512+vwarchu@users.noreply.github.com> Date: Thu, 13 May 2021 09:12:07 -0700 Subject: [PATCH 24/41] Update LICENSE.txt Reverting License to previously forked repository License --- LICENSE.txt | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index bdf74cb2..85f33650 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,11 +1,31 @@ Project Name: mssql-django -MIT License +BSD 3-Clause License -Copyright (c) Microsoft Corporation. +Copyright (c) Microsoft Corporation +All rights reserved. -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE \ No newline at end of file +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From fd4b929c8388e2d41e65d912e20d9ab97b734b76 Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Fri, 14 May 2021 21:39:02 +0000 Subject: [PATCH 25/41] Update CONTRIBUTING.md file Fix #11 --- CONTRIBUTING.md | 33 +++++++++++++++++++++++++++++++++ README.md | 2 ++ 2 files changed, 35 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a7aa9f0b..568fd400 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,39 @@ ## How to contribute +### Run unit tests +After changes made to the project, it's a good idea to run the unit tests before making a pull request. + + +1. **Run SQL Server** + Make sure you have SQL Server running in your local machine. + Download and install SQL Server [here](https://www.microsoft.com/en-us/sql-server/sql-server-downloads), or you could use docker. Change `testapp/settings.py` to match your SQL Server login username and password. + + ``` + docker run -e 'ACCEPT_EULA=Y' -e 'SA_PASSWORD=MyPassword42' -p 1433:1433 -d mcr.microsoft.com/mssql/server:2019-latest + ``` +2. **Clone Django** + In `mssql-django` folder. + ``` + # Install your local mssql-django + pip install -e . + + # The unit test suite are in `Django` folder, so we need to clone it + git clone https://github.com/django/django.git --depth 1 + ``` + +3. **Install Tox** + ``` + # we use `tox` to run tests and install dependencies + pip install tox + ``` +4. **Run Tox** + ``` + # eg. run django 3.1 tests with Python 3.7 + tox -e py37-django31 + ``` + +--- This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. diff --git a/README.md b/README.md index 8386a3db..4ab8739e 100644 --- a/README.md +++ b/README.md @@ -233,6 +233,8 @@ The following features and additions are planned: ## Contributing +More details on contributing can be found [Here](CONTRIBUTING.md). + This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. From f785bfa3d9a3f89c110d3bf30bee5d1de48d7cb9 Mon Sep 17 00:00:00 2001 From: Tim Nyborg Date: Thu, 20 May 2021 16:29:14 +0100 Subject: [PATCH 26/41] Enable support for window functions --- mssql/features.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mssql/features.py b/mssql/features.py index d1562bdd..e688012a 100644 --- a/mssql/features.py +++ b/mssql/features.py @@ -33,6 +33,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): supports_index_on_text_field = False supports_json_field_contains = False supports_order_by_nulls_modifier = False + supports_over_clause = True supports_paramstyle_pyformat = False supports_primitives_in_json_field = False supports_regex_backreferencing = True From 2abea1a2bad4af1298e43e3db751a6497bb5933c Mon Sep 17 00:00:00 2001 From: Tim Nyborg Date: Thu, 20 May 2021 20:20:58 +0100 Subject: [PATCH 27/41] default null ordering for window functions --- mssql/functions.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mssql/functions.py b/mssql/functions.py index d22eca2e..09c17ef5 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -4,10 +4,10 @@ import json from django import VERSION -from django.db.models import BooleanField +from django.db.models import BooleanField, Value from django.db.models.functions import Cast from django.db.models.functions.math import ATan2, Log, Ln, Mod, Round -from django.db.models.expressions import Case, Exists, OrderBy, When +from django.db.models.expressions import Case, Exists, OrderBy, When, Window from django.db.models.lookups import Lookup, In from django.db.models import lookups @@ -55,6 +55,13 @@ def sqlserver_round(self, compiler, connection, **extra_context): return self.as_sql(compiler, connection, template='%(function)s(%(expressions)s, 0)', **extra_context) +def sqlserver_window(self, compiler, connection, template=None): + # MSSQL window functions require an OVER clause with ORDER BY + if self.order_by is None: + self.order_by = Value('SELECT NULL') + return self.as_sql(compiler, connection, template) + + def sqlserver_exists(self, compiler, connection, template=None, **extra_context): # MS SQL doesn't allow EXISTS() in the SELECT list, so wrap it with a # CASE WHEN expression. Change the template since the When expression @@ -180,6 +187,7 @@ def json_HasKeyLookup(self, compiler, connection): Log.as_microsoft = sqlserver_log Mod.as_microsoft = sqlserver_mod Round.as_microsoft = sqlserver_round +Window.as_microsoft = sqlserver_window if DJANGO3: Lookup.as_microsoft = sqlserver_lookup @@ -187,3 +195,4 @@ def json_HasKeyLookup(self, compiler, connection): Exists.as_microsoft = sqlserver_exists OrderBy.as_microsoft = sqlserver_orderby + From 810336f6065e7804ae0516806975ee3a24681582 Mon Sep 17 00:00:00 2001 From: Tim Nyborg Date: Thu, 20 May 2021 21:41:10 +0100 Subject: [PATCH 28/41] mark NthValue as unsupported, and skip related tests --- mssql/functions.py | 8 +++++++- testapp/settings.py | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/mssql/functions.py b/mssql/functions.py index 09c17ef5..7c110006 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -4,8 +4,9 @@ import json from django import VERSION +from django.db import NotSupportedError from django.db.models import BooleanField, Value -from django.db.models.functions import Cast +from django.db.models.functions import Cast, NthValue from django.db.models.functions.math import ATan2, Log, Ln, Mod, Round from django.db.models.expressions import Case, Exists, OrderBy, When, Window from django.db.models.lookups import Lookup, In @@ -51,6 +52,10 @@ def sqlserver_mod(self, compiler, connection): ) +def sqlserver_nth_value(self, compiler, connection, **extra_content): + raise NotSupportedError('This backend does not support the NthValue function') + + def sqlserver_round(self, compiler, connection, **extra_context): return self.as_sql(compiler, connection, template='%(function)s(%(expressions)s, 0)', **extra_context) @@ -186,6 +191,7 @@ def json_HasKeyLookup(self, compiler, connection): Ln.as_microsoft = sqlserver_ln Log.as_microsoft = sqlserver_log Mod.as_microsoft = sqlserver_mod +NthValue.as_microsoft = sqlserver_nth_value Round.as_microsoft = sqlserver_round Window.as_microsoft = sqlserver_window diff --git a/testapp/settings.py b/testapp/settings.py index 626fa2c3..436a8ceb 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -54,6 +54,8 @@ 'expressions.tests.FTimeDeltaTests.test_duration_with_datetime_microseconds', 'expressions.tests.IterableLookupInnerExpressionsTests.test_expressions_in_lookups_join_choice', 'expressions_case.tests.CaseExpressionTests.test_annotate_with_in_clause', + 'expressions_window.tests.WindowFunctionTests.test_nth_returns_null', + 'expressions_window.tests.WindowFunctionTests.test_nthvalue', 'ordering.tests.OrderingTests.test_orders_nulls_first_on_filtered_subquery', 'queries.test_bulk_update.BulkUpdateNoteTests.test_set_field_to_null', 'get_or_create.tests.UpdateOrCreateTransactionTests.test_creation_in_transaction', From 6cce3b2a3e31842b996cd1276cf50bf6f0332edf Mon Sep 17 00:00:00 2001 From: Tim Nyborg Date: Thu, 20 May 2021 23:11:42 +0100 Subject: [PATCH 29/41] handle django 2.2 not excluding test via db feature flag --- testapp/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testapp/settings.py b/testapp/settings.py index 436a8ceb..7af5bf96 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -56,6 +56,7 @@ 'expressions_case.tests.CaseExpressionTests.test_annotate_with_in_clause', 'expressions_window.tests.WindowFunctionTests.test_nth_returns_null', 'expressions_window.tests.WindowFunctionTests.test_nthvalue', + 'expressions_window.tests.WindowFunctionTests.test_range_n_preceding_and_following', 'ordering.tests.OrderingTests.test_orders_nulls_first_on_filtered_subquery', 'queries.test_bulk_update.BulkUpdateNoteTests.test_set_field_to_null', 'get_or_create.tests.UpdateOrCreateTransactionTests.test_creation_in_transaction', From 8d7efbc7aec2f5260fdfd5030de277702768ea66 Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Mon, 31 May 2021 18:10:01 -0700 Subject: [PATCH 30/41] Publish test result to pipeline, Fix some Django 3.2 errors --- azure-pipelines.yml | 14 ++++++++++++++ mssql/base.py | 2 ++ mssql/features.py | 2 ++ mssql/functions.py | 8 ++++++++ mssql/introspection.py | 7 ++++++- mssql/operations.py | 4 ++-- test.sh | 6 ++++-- testapp/runners.py | 22 ++++++++++++++++++---- testapp/settings.py | 6 +----- tox.ini | 3 +++ 10 files changed, 60 insertions(+), 14 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 59dc206f..5835672b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -77,6 +77,13 @@ jobs: codeCoverageTool: 'Cobertura' summaryFileLocation: 'C:\Windows\ServiceProfiles\NetworkService\coverage.xml' + - task: PublishTestResults@2 + displayName: Publish test results via jUnit + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: 'C:\Windows\ServiceProfiles\NetworkService\result.xml' + testRunTitle: 'junit-$(Agent.OS)-$(Agent.OSArchitecture)-$(tox.env)' + - job: Linux pool: vmImage: ubuntu-18.04 @@ -144,3 +151,10 @@ jobs: inputs: codeCoverageTool: 'Cobertura' summaryFileLocation: '/home/vsts/coverage.xml' + + - task: PublishTestResults@2 + displayName: Publish test results via jUnit + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: '/home/vsts/result.xml' + testRunTitle: 'junit-$(Agent.OS)-$(Agent.OSArchitecture)-$(tox.env)' diff --git a/mssql/base.py b/mssql/base.py index 963c963c..a955680b 100644 --- a/mssql/base.py +++ b/mssql/base.py @@ -93,6 +93,7 @@ class DatabaseWrapper(BaseDatabaseWrapper): 'OneToOneField': 'int', 'PositiveIntegerField': 'int', 'PositiveSmallIntegerField': 'smallint', + 'PositiveBigIntegerField' : 'bigint', 'SlugField': 'nvarchar(%(max_length)s)', 'SmallAutoField': 'smallint', 'SmallIntegerField': 'smallint', @@ -109,6 +110,7 @@ class DatabaseWrapper(BaseDatabaseWrapper): 'JSONField': '(ISJSON ("%(column)s") = 1)', 'PositiveIntegerField': '[%(column)s] >= 0', 'PositiveSmallIntegerField': '[%(column)s] >= 0', + 'PositiveBigIntegerField': '[%(column)s] >= 0', } operators = { # Since '=' is used not only for string comparision there is no way diff --git a/mssql/features.py b/mssql/features.py index e688012a..fee7c009 100644 --- a/mssql/features.py +++ b/mssql/features.py @@ -15,6 +15,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): can_use_chunked_reads = False for_update_after_from = True greatest_least_ignores_nulls = True + has_json_object_function = False has_json_operators = False has_native_json_field = False has_native_uuid_field = False @@ -29,6 +30,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): requires_sqlparse_for_splitting = False supports_boolean_expr_in_select_clause = False supports_deferrable_unique_constraints = False + supports_expression_indexes = False supports_ignore_conflicts = False supports_index_on_text_field = False supports_json_field_contains = False diff --git a/mssql/functions.py b/mssql/functions.py index 7c110006..6317ef4e 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -17,6 +17,9 @@ KeyTransform, KeyTransformIn, KeyTransformExact, HasKeyLookup, compile_json_path) +if VERSION >= (3, 2): + from django.db.models.functions.math import Random + DJANGO3 = VERSION[0] >= 3 @@ -59,6 +62,8 @@ def sqlserver_nth_value(self, compiler, connection, **extra_content): def sqlserver_round(self, compiler, connection, **extra_context): return self.as_sql(compiler, connection, template='%(function)s(%(expressions)s, 0)', **extra_context) +def sqlserver_random(self, compiler, connection, **extra_context): + return self.as_sql(compiler, connection, function='RAND', **extra_context) def sqlserver_window(self, compiler, connection, template=None): # MSSQL window functions require an OVER clause with ORDER BY @@ -195,6 +200,9 @@ def json_HasKeyLookup(self, compiler, connection): Round.as_microsoft = sqlserver_round Window.as_microsoft = sqlserver_window +if VERSION >= (3, 2): + Random.as_microsoft = sqlserver_random + if DJANGO3: Lookup.as_microsoft = sqlserver_lookup else: diff --git a/mssql/introspection.py b/mssql/introspection.py index 3f794cbd..110e96c2 100644 --- a/mssql/introspection.py +++ b/mssql/introspection.py @@ -3,6 +3,7 @@ import pyodbc as Database +from django import VERSION from django.db.backends.base.introspection import ( BaseDatabaseIntrospection, FieldInfo, TableInfo, ) @@ -97,7 +98,11 @@ def get_table_description(self, cursor, table_name, identity_check=True): """ # map pyodbc's cursor.columns to db-api cursor description - columns = [[c[3], c[4], None, c[6], c[6], c[8], c[10], c[12]] for c in cursor.columns(table=table_name)] + if VERSION >= (3, 2): + columns = [[c[3], c[4], None, c[6], c[6], c[8], c[10], c[12], ''] for c in cursor.columns(table=table_name)] + else: + columns = [[c[3], c[4], None, c[6], c[6], c[8], c[10], c[12]] for c in cursor.columns(table=table_name)] + items = [] for column in columns: if identity_check and self._is_auto_field(cursor, table_name, column[0]): diff --git a/mssql/operations.py b/mssql/operations.py index 3c797659..fcab4d6a 100644 --- a/mssql/operations.py +++ b/mssql/operations.py @@ -143,7 +143,7 @@ def date_interval_sql(self, timedelta): sql = 'DATEADD(microsecond, %d%%s, CAST(%s AS datetime2))' % (timedelta.microseconds, sql) return sql - def date_trunc_sql(self, lookup_type, field_name): + def date_trunc_sql(self, lookup_type, field_name, tzname=''): CONVERT_YEAR = 'CONVERT(varchar, DATEPART(year, %s))' % field_name CONVERT_QUARTER = 'CONVERT(varchar, 1+((DATEPART(quarter, %s)-1)*3))' % field_name CONVERT_MONTH = 'CONVERT(varchar, DATEPART(month, %s))' % field_name @@ -465,7 +465,7 @@ def adapt_datetimefield_value(self, value): value = value.astimezone(self.connection.timezone).replace(tzinfo=None) return value - def time_trunc_sql(self, lookup_type, field_name): + def time_trunc_sql(self, lookup_type, field_name, tzname=''): # if self.connection.sql_server_version >= 2012: # fields = { # 'hour': 'DATEPART(hour, %s)' % field_name, diff --git a/test.sh b/test.sh index 79fd40e1..3920b476 100755 --- a/test.sh +++ b/test.sh @@ -10,7 +10,7 @@ DJANGO_VERSION="$(python -m django --version)" cd django git fetch --depth=1 origin +refs/tags/*:refs/tags/* git checkout $DJANGO_VERSION -pip install -r tests/requirements/py3.txt coverage +pip install -r tests/requirements/py3.txt coverage run tests/runtests.py --settings=testapp.settings --noinput \ aggregation \ @@ -110,4 +110,6 @@ coverage run tests/runtests.py --settings=testapp.settings --noinput \ update_only_fields python -m coverage xml --include '*mssql*' --omit '*virtualenvs*' -mv coverage.xml ~/ + +# For Azure Pipelines +cp -f coverage.xml result.xml ~/ diff --git a/testapp/runners.py b/testapp/runners.py index a7a032ae..acb4c631 100644 --- a/testapp/runners.py +++ b/testapp/runners.py @@ -1,6 +1,9 @@ from django.test.runner import DiscoverRunner from django.conf import settings +from unittest import skip +import xmlrunner + EXCLUDED_TESTS = getattr(settings, 'EXCLUDED_TESTS', []) REGEX_TESTS = getattr(settings, 'REGEX_TESTS', []) @@ -12,11 +15,22 @@ def build_suite(self, *args, **kwargs): suite = super().build_suite(*args, **kwargs) tests = [] for case in suite: + test_name = case._testMethodName if ENABLE_REGEX_TESTS: - if not case.id() in EXCLUDED_TESTS: - tests.append(case) + if case.id() in EXCLUDED_TESTS: + setattr(case, test_name, skip("Not supported")(getattr(case, test_name))) else: - if not case.id() in EXCLUDED_TESTS + REGEX_TESTS: - tests.append(case) + if case.id() in EXCLUDED_TESTS + REGEX_TESTS: + setattr(case, test_name, skip("Not supported")(getattr(case, test_name))) + tests.append(case) suite._tests = tests return suite + + def run_suite(self, suite): + kwargs = dict( + verbosity=1, descriptions=False, + failfast=self.failfast) + + with open('./result.xml', 'wb') as xml: + return xmlrunner.XMLTestRunner( + output=xml, **kwargs).run(suite) diff --git a/testapp/settings.py b/testapp/settings.py index 7af5bf96..eb456264 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -179,10 +179,6 @@ 'expressions.tests.ExpressionOperatorTests.test_lefthand_bitwise_xor_null', 'inspectdb.tests.InspectDBTestCase.test_number_field_types', 'inspectdb.tests.InspectDBTestCase.test_json_field', - 'model_fields.test_integerfield.PositiveBigIntegerFieldTests.test_backend_range_save', - 'model_fields.test_integerfield.PositiveBigIntegerFieldTests.test_coercing', - 'model_fields.test_integerfield.PositiveBigIntegerFieldTests.test_documented_range', - 'model_fields.test_integerfield.PositiveBigIntegerFieldTests.test_types', 'ordering.tests.OrderingTests.test_default_ordering_by_f_expression', 'ordering.tests.OrderingTests.test_order_by_nulls_first', 'ordering.tests.OrderingTests.test_order_by_nulls_last', @@ -203,8 +199,8 @@ 'model_fields.test_jsonfield.TestQuerying.test_none_key', 'model_fields.test_jsonfield.TestQuerying.test_none_key_and_exact_lookup', 'model_fields.test_jsonfield.TestQuerying.test_key_escape', - 'model_fields.test_jsonfield.TestQuerying.test_ordering_by_transform', + 'expressions_window.tests.WindowFunctionTests.test_key_transform', ] REGEX_TESTS = ['lookup.tests.LookupTests.test_regex', diff --git a/tox.ini b/tox.ini index 3c968719..c0693b64 100644 --- a/tox.ini +++ b/tox.ini @@ -16,6 +16,9 @@ commands = bash test.sh deps = + coverage + unittest-xml-reporting + django22: django==2.2.* django30: django>=3.0,<3.1 django31: django>=3.1,<3.2 From 4d1d7007249d2c028a2ed416376c70d97993e648 Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Wed, 2 Jun 2021 15:16:10 -0700 Subject: [PATCH 31/41] Mark current skipped tests expected failures --- testapp/runners.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/testapp/runners.py b/testapp/runners.py index acb4c631..dd3cdb25 100644 --- a/testapp/runners.py +++ b/testapp/runners.py @@ -1,7 +1,6 @@ from django.test.runner import DiscoverRunner from django.conf import settings -from unittest import skip import xmlrunner EXCLUDED_TESTS = getattr(settings, 'EXCLUDED_TESTS', []) @@ -10,6 +9,14 @@ ENABLE_REGEX_TESTS = getattr(settings, 'ENABLE_REGEX_TESTS', False) +def MarkexpectedFailure(): + def decorator(test_item): + def wrapper(): + raise "Expected Failure" + wrapper.__unittest_expecting_failure__ = True + return wrapper + return decorator + class ExcludedTestSuiteRunner(DiscoverRunner): def build_suite(self, *args, **kwargs): suite = super().build_suite(*args, **kwargs) @@ -18,10 +25,14 @@ def build_suite(self, *args, **kwargs): test_name = case._testMethodName if ENABLE_REGEX_TESTS: if case.id() in EXCLUDED_TESTS: - setattr(case, test_name, skip("Not supported")(getattr(case, test_name))) + test_method = getattr(case, test_name) + setattr(case, test_name, MarkexpectedFailure()(test_method)) + # setattr(case, test_name, expectedFailure2()(test_method)) else: if case.id() in EXCLUDED_TESTS + REGEX_TESTS: - setattr(case, test_name, skip("Not supported")(getattr(case, test_name))) + test_method = getattr(case, test_name) + setattr(case, test_name, MarkexpectedFailure()(test_method)) + # setattr(case, test_name, expectedFailure2()(test_method)) tests.append(case) suite._tests = tests return suite From 1b0afbdefe38007eecdae34483ed06e78c785fa9 Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Fri, 4 Jun 2021 14:36:57 -0700 Subject: [PATCH 32/41] Add Django 3.2 to pipeline --- azure-pipelines.yml | 26 ++++++++++++++++++++++++++ testapp/settings.py | 24 +++++++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5835672b..cdb5a17b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,6 +21,19 @@ jobs: strategy: matrix: + Python 3.9 - Django 3.2: + python.version: '3.9' + tox.env: 'py39-django32' + Python 3.8 - Django 3.2: + python.version: '3.8' + tox.env: 'py38-django32' + Python 3.7 - Django 3.2: + python.version: '3.7' + tox.env: 'py37-django32' + Python 3.6 - Django 3.2: + python.version: '3.6' + tox.env: 'py36-django32' + Python 3.9 - Django 3.1: python.version: '3.9' tox.env: 'py39-django31' @@ -90,6 +103,19 @@ jobs: strategy: matrix: + Python 3.9 - Django 3.2: + python.version: '3.9' + tox.env: 'py39-django32' + Python 3.8 - Django 3.2: + python.version: '3.8' + tox.env: 'py38-django32' + Python 3.7 - Django 3.2: + python.version: '3.7' + tox.env: 'py37-django32' + Python 3.6 - Django 3.2: + python.version: '3.6' + tox.env: 'py36-django32' + Python 3.9 - Django 3.1: python.version: '3.9' tox.env: 'py39-django31' diff --git a/testapp/settings.py b/testapp/settings.py index eb456264..5be2cc9c 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -189,7 +189,7 @@ 'schema.tests.SchemaTests.test_char_field_pk_to_auto_field', 'datetimes.tests.DateTimesTests.test_21432', - #JSONFields + # JSONFields 'model_fields.test_jsonfield.TestQuerying.test_has_key_list', 'model_fields.test_jsonfield.TestQuerying.test_has_key_null_value', 'model_fields.test_jsonfield.TestQuerying.test_key_quoted_string', @@ -201,6 +201,28 @@ 'model_fields.test_jsonfield.TestQuerying.test_key_escape', 'model_fields.test_jsonfield.TestQuerying.test_ordering_by_transform', 'expressions_window.tests.WindowFunctionTests.test_key_transform', + + # Django 3.2 + 'model_indexes.tests.IndexesTests.test_func_with_tablespace', + 'migrations.test_operations.OperationTests.test_add_covering_unique_constraint', + 'migrations.test_operations.OperationTests.test_remove_covering_unique_constraint', + 'constraints.tests.CheckConstraintTests.test_database_constraint_unicode', + 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_func_with_timezone', + 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_timezone_applied_before_truncation', + 'expressions.tests.ExistsTests.test_optimizations', + 'expressions.tests.FTimeDeltaTests.test_delta_add', + 'expressions.tests.FTimeDeltaTests.test_delta_subtract', + 'expressions.tests.FTimeDeltaTests.test_delta_update', + 'expressions.tests.FTimeDeltaTests.test_exclude', + 'expressions.tests.FTimeDeltaTests.test_mixed_comparisons1', + 'expressions.tests.FTimeDeltaTests.test_negative_timedelta_update', + 'inspectdb.tests.InspectDBTestCase.test_field_types', + 'lookup.tests.LookupTests.test_in_ignore_none', + 'lookup.tests.LookupTests.test_in_ignore_none_with_unhashable_items', + 'queries.test_qs_combinators.QuerySetSetOperationTests.test_exists_union', + 'introspection.tests.IntrospectionTests.test_get_constraints_unique_indexes_orders', + 'schema.tests.SchemaTests.test_ci_cs_db_collation', + 'select_for_update.tests.SelectForUpdateTests.test_unsuported_no_key_raises_error', ] REGEX_TESTS = ['lookup.tests.LookupTests.test_regex', From 219c8f76bcd3ad981c4bc8abd9e6a574cf802e0e Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Tue, 8 Jun 2021 23:40:51 -0700 Subject: [PATCH 33/41] Cleanup test runner --- testapp/runners.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/testapp/runners.py b/testapp/runners.py index dd3cdb25..d34f97b8 100644 --- a/testapp/runners.py +++ b/testapp/runners.py @@ -27,20 +27,16 @@ def build_suite(self, *args, **kwargs): if case.id() in EXCLUDED_TESTS: test_method = getattr(case, test_name) setattr(case, test_name, MarkexpectedFailure()(test_method)) - # setattr(case, test_name, expectedFailure2()(test_method)) else: if case.id() in EXCLUDED_TESTS + REGEX_TESTS: test_method = getattr(case, test_name) setattr(case, test_name, MarkexpectedFailure()(test_method)) - # setattr(case, test_name, expectedFailure2()(test_method)) tests.append(case) suite._tests = tests return suite def run_suite(self, suite): - kwargs = dict( - verbosity=1, descriptions=False, - failfast=self.failfast) + kwargs = dict(verbosity=1, descriptions=False) with open('./result.xml', 'wb') as xml: return xmlrunner.XMLTestRunner( From 8b9679fc33e4a002cbc318209895c913fb236eef Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Fri, 11 Jun 2021 14:26:59 -0700 Subject: [PATCH 34/41] Cleanup create index function --- mssql/schema.py | 32 +++++++++++--------------------- testapp/settings.py | 1 - 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/mssql/schema.py b/mssql/schema.py index 6d151c15..d4079fc0 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -735,28 +735,18 @@ def _create_index_sql(self, model, fields, *, name=None, suffix='', using='', `sql` can be specified if the syntax differs from the standard (GIS indexes, ...). """ - tablespace_sql = self._get_index_tablespace_sql(model, fields, db_tablespace=db_tablespace) - columns = [field.column for field in fields] - sql_create_index = sql or self.sql_create_index - table = model._meta.db_table - - def create_index_name(*args, **kwargs): - nonlocal name - if name is None: - name = self._create_index_name(*args, **kwargs) - return self.quote_name(name) - - return Statement( - sql_create_index, - table=Table(table, self.quote_name), - name=IndexName(table, columns, suffix, create_index_name), - using=using, - columns=self._index_columns(table, columns, col_suffixes, opclasses), - extra=tablespace_sql, - condition=(' WHERE ' + condition) if condition else '', - include='' + if django_version >= (3, 2): + return super()._create_index_sql( + model, fields=fields, name=name, suffix=suffix, using=using, + db_tablespace=db_tablespace, col_suffixes=col_suffixes, sql=sql, + opclasses=opclasses, condition=condition, include=include, + expressions=expressions, + ) + return super()._create_index_sql( + model, fields=fields, name=name, suffix=suffix, using=using, + db_tablespace=db_tablespace, col_suffixes=col_suffixes, sql=sql, + opclasses=opclasses, condition=condition, ) - def create_model(self, model): """ Takes a model and creates a table for it in the database. diff --git a/testapp/settings.py b/testapp/settings.py index 5be2cc9c..28bd0bdf 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -203,7 +203,6 @@ 'expressions_window.tests.WindowFunctionTests.test_key_transform', # Django 3.2 - 'model_indexes.tests.IndexesTests.test_func_with_tablespace', 'migrations.test_operations.OperationTests.test_add_covering_unique_constraint', 'migrations.test_operations.OperationTests.test_remove_covering_unique_constraint', 'constraints.tests.CheckConstraintTests.test_database_constraint_unicode', From c809d623326117ed9e822ba61170cfa24b29d2b0 Mon Sep 17 00:00:00 2001 From: Martin Zellner Date: Tue, 15 Jun 2021 16:12:32 +0200 Subject: [PATCH 35/41] fix: use parameter data type instead of hardcoded nvarchar(32) --- mssql/functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mssql/functions.py b/mssql/functions.py index 6317ef4e..1ec51416 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -126,7 +126,8 @@ def split_parameter_list_as_sql(self, compiler, connection): with connection.cursor() as cursor: cursor.execute("IF OBJECT_ID('tempdb.dbo.#Temp_params', 'U') IS NOT NULL DROP TABLE #Temp_params; ") - cursor.execute("CREATE TABLE #Temp_params (params nvarchar(32))") + parameter_data_type = self.lhs.field.db_type(connection) + cursor.execute(f"CREATE TABLE #Temp_params (params {parameter_data_type})") for offset in range(0, len(rhs_params), 1000): sqls_params = rhs_params[offset: offset + 1000] sqls_params = ", ".join("('{}')".format(item) for item in sqls_params) From 046479270af0a3825d00ff169f48c4a123884ec4 Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Mon, 21 Jun 2021 17:01:59 -0700 Subject: [PATCH 36/41] Migrate Windows pipeline to 1ES --- azure-pipelines.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cdb5a17b..1db9bcbe 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,7 +16,9 @@ schedules: jobs: - job: Windows pool: - Django-agent-pool + name: SqlClientDrivers-1ES-Windows-agent-pool + demands: + - imageOverride -equals Django_Win2019 timeoutInMinutes: 120 strategy: @@ -77,6 +79,8 @@ jobs: displayName: Use Python $(python.version) - script: | + docker pull mcr.microsoft.com/windows/servercore:1809 + docker run -e 'ACCEPT_EULA=Y' -e 'SA_PASSWORD=MyPassword42' -p 1433:1433 -d mcr.microsoft.com/windows/servercore:1809 python -m pip install --upgrade pip wheel setuptools pip install tox git clone https://github.com/django/django.git @@ -85,17 +89,6 @@ jobs: - script: tox -e $(tox.env) displayName: Run tox - - task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: 'Cobertura' - summaryFileLocation: 'C:\Windows\ServiceProfiles\NetworkService\coverage.xml' - - - task: PublishTestResults@2 - displayName: Publish test results via jUnit - inputs: - testResultsFormat: 'JUnit' - testResultsFiles: 'C:\Windows\ServiceProfiles\NetworkService\result.xml' - testRunTitle: 'junit-$(Agent.OS)-$(Agent.OSArchitecture)-$(tox.env)' - job: Linux pool: From 05a3b323db156db830856719388c3b45a74f2ba5 Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Wed, 23 Jun 2021 16:49:03 -0700 Subject: [PATCH 37/41] Remove windows pipeline temporary --- azure-pipelines.yml | 76 --------------------------------------------- 1 file changed, 76 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1db9bcbe..fde089bb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -14,82 +14,6 @@ schedules: always: true jobs: - - job: Windows - pool: - name: SqlClientDrivers-1ES-Windows-agent-pool - demands: - - imageOverride -equals Django_Win2019 - timeoutInMinutes: 120 - - strategy: - matrix: - Python 3.9 - Django 3.2: - python.version: '3.9' - tox.env: 'py39-django32' - Python 3.8 - Django 3.2: - python.version: '3.8' - tox.env: 'py38-django32' - Python 3.7 - Django 3.2: - python.version: '3.7' - tox.env: 'py37-django32' - Python 3.6 - Django 3.2: - python.version: '3.6' - tox.env: 'py36-django32' - - Python 3.9 - Django 3.1: - python.version: '3.9' - tox.env: 'py39-django31' - Python 3.8 - Django 3.1: - python.version: '3.8' - tox.env: 'py38-django31' - Python 3.7 - Django 3.1: - python.version: '3.7' - tox.env: 'py37-django31' - Python 3.6 - Django 3.1: - python.version: '3.6' - tox.env: 'py36-django31' - - Python 3.9 - Django 3.0: - python.version: '3.9' - tox.env: 'py39-django30' - Python 3.8 - Django 3.0: - python.version: '3.8' - tox.env: 'py38-django30' - Python 3.7 - Django 3.0: - python.version: '3.7' - tox.env: 'py37-django30' - Python 3.6 - Django 3.0: - python.version: '3.6' - tox.env: 'py36-django30' - - Python 3.7 - Django 2.2: - python.version: '3.7' - tox.env: 'py37-django22' - Python 3.6 - Django 2.2: - python.version: '3.6' - tox.env: 'py36-django22' - - steps: - - task: CredScan@2 - inputs: - toolMajorVersion: 'V2' - - task: UsePythonVersion@0 - inputs: - versionSpec: "$(python.version)" - displayName: Use Python $(python.version) - - - script: | - docker pull mcr.microsoft.com/windows/servercore:1809 - docker run -e 'ACCEPT_EULA=Y' -e 'SA_PASSWORD=MyPassword42' -p 1433:1433 -d mcr.microsoft.com/windows/servercore:1809 - python -m pip install --upgrade pip wheel setuptools - pip install tox - git clone https://github.com/django/django.git - displayName: Install requirements - - - script: tox -e $(tox.env) - displayName: Run tox - - - job: Linux pool: vmImage: ubuntu-18.04 From 848c847cec23f18501010d97889b2cbcb7631fbf Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Wed, 23 Jun 2021 17:12:40 -0700 Subject: [PATCH 38/41] Add max_length parameter to binaryfield --- mssql/base.py | 2 +- mssql/functions.py | 11 +++++++++++ testapp/settings.py | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/mssql/base.py b/mssql/base.py index a955680b..e904785d 100644 --- a/mssql/base.py +++ b/mssql/base.py @@ -75,7 +75,7 @@ class DatabaseWrapper(BaseDatabaseWrapper): 'AutoField': 'int', 'BigAutoField': 'bigint', 'BigIntegerField': 'bigint', - 'BinaryField': 'varbinary(max)', + 'BinaryField': 'varbinary(%(max_length)s)', 'BooleanField': 'bit', 'CharField': 'nvarchar(%(max_length)s)', 'DateField': 'date', diff --git a/mssql/functions.py b/mssql/functions.py index 1ec51416..afb3e414 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -11,6 +11,8 @@ from django.db.models.expressions import Case, Exists, OrderBy, When, Window from django.db.models.lookups import Lookup, In from django.db.models import lookups +from django.db.models.fields import BinaryField, Field +from django.core import validators if VERSION >= (3, 1): from django.db.models.fields.json import ( @@ -187,6 +189,14 @@ def json_HasKeyLookup(self, compiler, connection): return sql % tuple(rhs_params), [] +def BinaryField_init(self, *args, **kwargs): + # Add max_length option for BinaryField, default to max + kwargs.setdefault('editable', False) + Field.__init__(self, *args, **kwargs) + if self.max_length is not None: + self.validators.append(validators.MaxLengthValidator(self.max_length)) + else: + self.max_length = 'max' ATan2.as_microsoft = sqlserver_atan2 In.split_parameter_list_as_sql = split_parameter_list_as_sql @@ -200,6 +210,7 @@ def json_HasKeyLookup(self, compiler, connection): NthValue.as_microsoft = sqlserver_nth_value Round.as_microsoft = sqlserver_round Window.as_microsoft = sqlserver_window +BinaryField.__init__ = BinaryField_init if VERSION >= (3, 2): Random.as_microsoft = sqlserver_random diff --git a/testapp/settings.py b/testapp/settings.py index 28bd0bdf..784b6ef3 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -57,6 +57,7 @@ 'expressions_window.tests.WindowFunctionTests.test_nth_returns_null', 'expressions_window.tests.WindowFunctionTests.test_nthvalue', 'expressions_window.tests.WindowFunctionTests.test_range_n_preceding_and_following', + 'field_deconstruction.tests.FieldDeconstructionTests.test_binary_field', 'ordering.tests.OrderingTests.test_orders_nulls_first_on_filtered_subquery', 'queries.test_bulk_update.BulkUpdateNoteTests.test_set_field_to_null', 'get_or_create.tests.UpdateOrCreateTransactionTests.test_creation_in_transaction', From 10f05255159a029e45c0dc1ae51de24da8c051da Mon Sep 17 00:00:00 2001 From: Warren Chu <60903512+vwarchu@users.noreply.github.com> Date: Tue, 29 Jun 2021 00:49:58 -0700 Subject: [PATCH 39/41] Update License.txt to include historical copyright notices --- LICENSE.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/LICENSE.txt b/LICENSE.txt index 85f33650..ac712854 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -2,7 +2,10 @@ Project Name: mssql-django BSD 3-Clause License -Copyright (c) Microsoft Corporation +Copyright (c) 2021, Microsoft Corporation + 2019, ES Solutions AB + 2018, Michiya Takahashi + 2008, 2009 django-pyodbc developers All rights reserved. Redistribution and use in source and binary forms, with or without From aca9d268987076d7c257580ffb501e176c0c0178 Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Wed, 30 Jun 2021 09:35:19 -0700 Subject: [PATCH 40/41] Revert license to BSD --- manage.py | 2 +- mssql/__init__.py | 2 +- mssql/base.py | 2 +- mssql/client.py | 2 +- mssql/compiler.py | 2 +- mssql/creation.py | 2 +- mssql/features.py | 2 +- mssql/functions.py | 2 +- mssql/introspection.py | 2 +- mssql/management/commands/install_regex_clr.py | 2 +- mssql/operations.py | 2 +- mssql/schema.py | 2 +- setup.py | 2 +- testapp/models.py | 2 +- testapp/settings.py | 2 +- testapp/tests/test_constraints.py | 2 +- testapp/tests/test_expressions.py | 2 +- testapp/tests/test_fields.py | 2 +- testapp/tests/test_lookups.py | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/manage.py b/manage.py index e88b9965..4a2e3bfd 100755 --- a/manage.py +++ b/manage.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import os import sys diff --git a/mssql/__init__.py b/mssql/__init__.py index 2c482eac..4c62a7b3 100644 --- a/mssql/__init__.py +++ b/mssql/__init__.py @@ -1,4 +1,4 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import mssql.functions # noqa diff --git a/mssql/base.py b/mssql/base.py index e904785d..5a23dd71 100644 --- a/mssql/base.py +++ b/mssql/base.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. """ MS SQL Server database backend for Django. diff --git a/mssql/client.py b/mssql/client.py index a1acb03d..aaa742b1 100644 --- a/mssql/client.py +++ b/mssql/client.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import re import subprocess diff --git a/mssql/compiler.py b/mssql/compiler.py index 56e53fec..dfa94ce8 100644 --- a/mssql/compiler.py +++ b/mssql/compiler.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import types from itertools import chain diff --git a/mssql/creation.py b/mssql/creation.py index cca18760..18fcfb85 100644 --- a/mssql/creation.py +++ b/mssql/creation.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import binascii import os diff --git a/mssql/features.py b/mssql/features.py index fee7c009..19208580 100644 --- a/mssql/features.py +++ b/mssql/features.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. from django.db.backends.base.features import BaseDatabaseFeatures from django.utils.functional import cached_property diff --git a/mssql/functions.py b/mssql/functions.py index afb3e414..cfb6e033 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import json diff --git a/mssql/introspection.py b/mssql/introspection.py index 110e96c2..65d6cf26 100644 --- a/mssql/introspection.py +++ b/mssql/introspection.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import pyodbc as Database diff --git a/mssql/management/commands/install_regex_clr.py b/mssql/management/commands/install_regex_clr.py index 6cd86cfa..be027ff2 100644 --- a/mssql/management/commands/install_regex_clr.py +++ b/mssql/management/commands/install_regex_clr.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. # Add regex support in SQLServer # Code taken from django-mssql (see https://bitbucket.org/Manfre/django-mssql) diff --git a/mssql/operations.py b/mssql/operations.py index fcab4d6a..1462a094 100644 --- a/mssql/operations.py +++ b/mssql/operations.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import datetime import uuid diff --git a/mssql/schema.py b/mssql/schema.py index d4079fc0..c35e7960 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import binascii import datetime diff --git a/setup.py b/setup.py index 64bdb724..e1346104 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. from os import path from setuptools import find_packages, setup diff --git a/testapp/models.py b/testapp/models.py index 05cbf245..8049309e 100644 --- a/testapp/models.py +++ b/testapp/models.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import uuid diff --git a/testapp/settings.py b/testapp/settings.py index 784b6ef3..207716b0 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. DATABASES = { "default": { diff --git a/testapp/tests/test_constraints.py b/testapp/tests/test_constraints.py index 2c743f11..1030e9f8 100644 --- a/testapp/tests/test_constraints.py +++ b/testapp/tests/test_constraints.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. from django.db import connections, migrations, models from django.db.migrations.state import ProjectState diff --git a/testapp/tests/test_expressions.py b/testapp/tests/test_expressions.py index a23f68b2..c10eb8ca 100644 --- a/testapp/tests/test_expressions.py +++ b/testapp/tests/test_expressions.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. from unittest import skipUnless diff --git a/testapp/tests/test_fields.py b/testapp/tests/test_fields.py index da26132b..9b8f25f2 100644 --- a/testapp/tests/test_fields.py +++ b/testapp/tests/test_fields.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. from django.test import TestCase diff --git a/testapp/tests/test_lookups.py b/testapp/tests/test_lookups.py index 3ca12486..1d972805 100644 --- a/testapp/tests/test_lookups.py +++ b/testapp/tests/test_lookups.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. from django.test import TestCase from ..models import Pizza, Topping From 9e934b0f16f70a066e545a8549f036f205b7e8f6 Mon Sep 17 00:00:00 2001 From: Sicong Jia Date: Wed, 30 Jun 2021 10:36:50 -0700 Subject: [PATCH 41/41] Bump up version to 1.0rc1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e1346104..63793576 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( name='mssql-django', - version='1.0b1', + version='1.0rc1', description='Django backend for Microsoft SQL Server', long_description=long_description, long_description_content_type='text/markdown',