diff --git a/.gitignore b/.gitignore index 0c91415f..1377bf26 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ Thumbs.db *.egg-info - +*.dll tests/local_settings.py # Virtual Env diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a7aa9f0b..568fd400 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,39 @@ ## How to contribute +### Run unit tests +After changes made to the project, it's a good idea to run the unit tests before making a pull request. + + +1. **Run SQL Server** + Make sure you have SQL Server running in your local machine. + Download and install SQL Server [here](https://www.microsoft.com/en-us/sql-server/sql-server-downloads), or you could use docker. Change `testapp/settings.py` to match your SQL Server login username and password. + + ``` + docker run -e 'ACCEPT_EULA=Y' -e 'SA_PASSWORD=MyPassword42' -p 1433:1433 -d mcr.microsoft.com/mssql/server:2019-latest + ``` +2. **Clone Django** + In `mssql-django` folder. + ``` + # Install your local mssql-django + pip install -e . + + # The unit test suite are in `Django` folder, so we need to clone it + git clone https://github.com/django/django.git --depth 1 + ``` + +3. **Install Tox** + ``` + # we use `tox` to run tests and install dependencies + pip install tox + ``` +4. **Run Tox** + ``` + # eg. run django 3.1 tests with Python 3.7 + tox -e py37-django31 + ``` + +--- This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. diff --git a/LICENSE.txt b/LICENSE.txt index bdf74cb2..ac712854 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,11 +1,34 @@ Project Name: mssql-django -MIT License +BSD 3-Clause License -Copyright (c) Microsoft Corporation. +Copyright (c) 2021, Microsoft Corporation + 2019, ES Solutions AB + 2018, Michiya Takahashi + 2008, 2009 django-pyodbc developers +All rights reserved. -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE \ No newline at end of file +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index 5725f879..4ab8739e 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,13 @@ Welcome to the MSSQL-Django 3rd party backend project! *mssql-django* is a fork of [django-mssql-backend](https://pypi.org/project/django-mssql-backend/). This project provides an enterprise database connectivity option for the Django Web Framework, with support for Microsoft SQL Server and Azure SQL Database. -We'd like to give thanks to the community that made this project possible, with particular recognition of the contributors: OskarPersson, michiya, dlo and the original Google Code django-pyodbc team. Moving forward we encourage partipation in this project from both old and new contributors! +We'd like to give thanks to the community that made this project possible, with particular recognition of the contributors: OskarPersson, michiya, dlo and the original Google Code django-pyodbc team. Moving forward we encourage partipation in this project from both old and new contributors! We hope you enjoy using the MSSQL-Django 3rd party backend. ## Features -- Supports Django 2.2, 3.0 +- Supports Django 2.2, 3.0 and 3.1 - Tested on Microsoft SQL Server 2016, 2017, 2019 - Passes most of the tests of the Django test suite - Compatible with @@ -20,12 +20,12 @@ We hope you enjoy using the MSSQL-Django 3rd party backend. ## Dependencies -- Django 2.2 or 3.0 +- Django 2.2, 3.0 or 3.1 - pyodbc 3.0 or newer ## Installation -1. Install pyodbc 3.0 (or newer) and Django 2.2 (or 3.0) +1. Install pyodbc 3.0 (or newer) and Django 2.2, 3.0 or 3.1 2. Install mssql-django: @@ -74,6 +74,10 @@ in DATABASES control the behavior of the backend: Boolean. Set this to `False` if you want to disable Django's transaction management and implement your own. +- Trusted_Connection + + String. Default is `"yes"`. Can be set to `"no"` if required. + and the following entries are also available in the `TEST` dictionary for any given database-level settings dictionary: @@ -81,7 +85,7 @@ for any given database-level settings dictionary: String. The name of database to use when running the test suite. If the default value (`None`) is used, the test database will use - the name `"test\_" + NAME`. + the name `"test_" + NAME`. - COLLATION @@ -135,10 +139,12 @@ Dictionary. Current available keys are: definition present in the ``freetds.conf`` FreeTDS configuration file instead of a hostname or an IP address. - But if this option is present and it's value is ``True``, this - special behavior is turned off. + But if this option is present and its value is ``True``, this + special behavior is turned off. Instead, connections to the database + server will be established using ``HOST`` and ``PORT`` options, without + requiring ``freetds.conf`` to be configured. - See http://www.freetds.org/userguide/dsnless.htm for more information. + See https://www.freetds.org/userguide/dsnless.html for more information. - unicode_results @@ -149,7 +155,7 @@ Dictionary. Current available keys are: - extra_params String. Additional parameters for the ODBC connection. The format is - ``"param=value;param=value"``. + ``"param=value;param=value"``, [Azure AD Authentication](https://github.com/microsoft/mssql-django/wiki/Azure-AD-Authentication) can be added to this field. - collation @@ -216,7 +222,9 @@ Here is an example of the database settings: The following features are currently not supported: - mssql-django does not support SQL-based regex commands -- Altering a model field from or to AutoField at migration +- Altering a model field from or to AutoField at migration + +Certain limitations for JSONField lookups, more details [here](https://github.com/microsoft/mssql-django/wiki/JSONField). ## Future Plans @@ -225,6 +233,8 @@ The following features and additions are planned: ## Contributing +More details on contributing can be found [Here](CONTRIBUTING.md). + This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3a70cd6e..fde089bb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -14,68 +14,28 @@ schedules: always: true jobs: - - job: Windows + - job: Linux pool: - Django-agent-pool + vmImage: ubuntu-18.04 strategy: matrix: - Python 3.8 - Django 3.1: + Python 3.9 - Django 3.2: + python.version: '3.9' + tox.env: 'py39-django32' + Python 3.8 - Django 3.2: python.version: '3.8' - tox.env: 'py38-django31' - Python 3.7 - Django 3.1: + tox.env: 'py38-django32' + Python 3.7 - Django 3.2: python.version: '3.7' - tox.env: 'py37-django31' - Python 3.6 - Django 3.1: + tox.env: 'py37-django32' + Python 3.6 - Django 3.2: python.version: '3.6' - tox.env: 'py36-django31' + tox.env: 'py36-django32' - Python 3.8 - Django 3.0: - python.version: '3.8' - tox.env: 'py38-django30' - Python 3.7 - Django 3.0: - python.version: '3.7' - tox.env: 'py37-django30' - Python 3.6 - Django 3.0: - python.version: '3.6' - tox.env: 'py36-django30' - - Python 3.7 - Django 2.2: - python.version: '3.7' - tox.env: 'py37-django22' - Python 3.6 - Django 2.2: - python.version: '3.6' - tox.env: 'py36-django22' - - steps: - - task: CredScan@2 - inputs: - toolMajorVersion: 'V2' - - task: UsePythonVersion@0 - inputs: - versionSpec: "$(python.version)" - displayName: Use Python $(python.version) - - - script: | - python -m pip install --upgrade pip wheel setuptools - pip install tox - git clone https://github.com/django/django.git - displayName: Install requirements - - - script: tox -e $(tox.env) - displayName: Run tox - - - task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: 'Cobertura' - summaryFileLocation: 'C:\Windows\ServiceProfiles\NetworkService\coverage.xml' - - - job: Linux - pool: - vmImage: ubuntu-18.04 - - strategy: - matrix: + Python 3.9 - Django 3.1: + python.version: '3.9' + tox.env: 'py39-django31' Python 3.8 - Django 3.1: python.version: '3.8' tox.env: 'py38-django31' @@ -86,6 +46,9 @@ jobs: python.version: '3.6' tox.env: 'py36-django31' + Python 3.9 - Django 3.0: + python.version: '3.9' + tox.env: 'py39-django30' Python 3.8 - Django 3.0: python.version: '3.8' tox.env: 'py38-django30' @@ -131,3 +94,10 @@ jobs: inputs: codeCoverageTool: 'Cobertura' summaryFileLocation: '/home/vsts/coverage.xml' + + - task: PublishTestResults@2 + displayName: Publish test results via jUnit + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: '/home/vsts/result.xml' + testRunTitle: 'junit-$(Agent.OS)-$(Agent.OSArchitecture)-$(tox.env)' diff --git a/manage.py b/manage.py index a57ed35e..4a2e3bfd 100755 --- a/manage.py +++ b/manage.py @@ -1,7 +1,8 @@ +#!/usr/bin/env python + # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. -#!/usr/bin/env python import os import sys diff --git a/mssql/__init__.py b/mssql/__init__.py index 2c482eac..4c62a7b3 100644 --- a/mssql/__init__.py +++ b/mssql/__init__.py @@ -1,4 +1,4 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import mssql.functions # noqa diff --git a/mssql/base.py b/mssql/base.py index 17c27da2..5a23dd71 100644 --- a/mssql/base.py +++ b/mssql/base.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. """ MS SQL Server database backend for Django. @@ -75,7 +75,7 @@ class DatabaseWrapper(BaseDatabaseWrapper): 'AutoField': 'int', 'BigAutoField': 'bigint', 'BigIntegerField': 'bigint', - 'BinaryField': 'varbinary(max)', + 'BinaryField': 'varbinary(%(max_length)s)', 'BooleanField': 'bit', 'CharField': 'nvarchar(%(max_length)s)', 'DateField': 'date', @@ -88,10 +88,12 @@ class DatabaseWrapper(BaseDatabaseWrapper): 'IntegerField': 'int', 'IPAddressField': 'nvarchar(15)', 'GenericIPAddressField': 'nvarchar(39)', + 'JSONField': 'nvarchar(max)', 'NullBooleanField': 'bit', 'OneToOneField': 'int', 'PositiveIntegerField': 'int', 'PositiveSmallIntegerField': 'smallint', + 'PositiveBigIntegerField' : 'bigint', 'SlugField': 'nvarchar(%(max_length)s)', 'SmallAutoField': 'smallint', 'SmallIntegerField': 'smallint', @@ -105,8 +107,10 @@ class DatabaseWrapper(BaseDatabaseWrapper): 'SmallAutoField': 'IDENTITY (1, 1)', } data_type_check_constraints = { + 'JSONField': '(ISJSON ("%(column)s") = 1)', 'PositiveIntegerField': '[%(column)s] >= 0', 'PositiveSmallIntegerField': '[%(column)s] >= 0', + 'PositiveBigIntegerField': '[%(column)s] >= 0', } operators = { # Since '=' is used not only for string comparision there is no way @@ -249,9 +253,10 @@ def get_new_connection(self, conn_params): user = conn_params.get('USER', None) password = conn_params.get('PASSWORD', None) port = conn_params.get('PORT', None) + trusted_connection = conn_params.get('Trusted_Connection', 'yes') options = conn_params.get('OPTIONS', {}) - driver = options.get('driver', 'ODBC Driver 13 for SQL Server') + driver = options.get('driver', 'ODBC Driver 17 for SQL Server') dsn = options.get('dsn', None) # Microsoft driver names assumed here are: @@ -286,10 +291,11 @@ def get_new_connection(self, conn_params): if user: cstr_parts['UID'] = user - cstr_parts['PWD'] = password + if 'Authentication=ActiveDirectoryInteractive' not in options.get('extra_params', ''): + cstr_parts['PWD'] = password else: if ms_drivers.match(driver): - cstr_parts['Trusted_Connection'] = 'yes' + cstr_parts['Trusted_Connection'] = trusted_connection else: cstr_parts['Integrated Security'] = 'SSPI' @@ -487,12 +493,12 @@ def check_constraints(self, table_names=None): def disable_constraint_checking(self): if not self.needs_rollback: - self.cursor().execute('EXEC sp_msforeachtable "ALTER TABLE ? NOCHECK CONSTRAINT ALL"') + self._execute_foreach('ALTER TABLE %s NOCHECK CONSTRAINT ALL') return not self.needs_rollback def enable_constraint_checking(self): if not self.needs_rollback: - self.cursor().execute('EXEC sp_msforeachtable "ALTER TABLE ? WITH CHECK CHECK CONSTRAINT ALL"') + self._execute_foreach('ALTER TABLE %s WITH NOCHECK CHECK CONSTRAINT ALL') class CursorWrapper(object): diff --git a/mssql/client.py b/mssql/client.py index a1acb03d..aaa742b1 100644 --- a/mssql/client.py +++ b/mssql/client.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import re import subprocess diff --git a/mssql/compiler.py b/mssql/compiler.py index 28d1fbd2..dfa94ce8 100644 --- a/mssql/compiler.py +++ b/mssql/compiler.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import types from itertools import chain @@ -13,7 +13,8 @@ from django.db.models.sql import compiler from django.db.transaction import TransactionManagementError from django.db.utils import NotSupportedError - +if django.VERSION >= (3, 1): + from django.db.models.fields.json import compile_json_path, KeyTransform as json_KeyTransform def _as_sql_agv(self, compiler, connection): return self.as_sql(compiler, connection, template='%(function)s(CONVERT(float, %(field)s))') @@ -42,6 +43,13 @@ def _as_sql_greatest(self, compiler, connection): template = '(SELECT MAX(value) FROM (VALUES (%(expressions)s)) AS _%(function)s(value))' return self.as_sql(compiler, connection, arg_joiner='), (', template=template) +def _as_sql_json_keytransform(self, compiler, connection): + lhs, params, key_transforms = self.preprocess_lhs(compiler, connection) + json_path = compile_json_path(key_transforms) + return ( + "COALESCE(JSON_QUERY(%s, '%s'), JSON_VALUE(%s, '%s'))" % + ((lhs, json_path) * 2) + ), tuple(params) * 2 def _as_sql_least(self, compiler, connection): # SQL Server does not provide LEAST function, @@ -307,6 +315,12 @@ def as_sql(self, with_limits=True, with_col_aliases=False): params.extend(o_params) result.append('ORDER BY %s' % ', '.join(ordering)) + # For subqueres with an ORDER BY clause, SQL Server also + # requires a TOP or OFFSET clause which is not generated for + # Django 2.x. See https://github.com/microsoft/mssql-django/issues/12 + if django.VERSION < (3, 0, 0) and not (do_offset or do_limit): + result.append("OFFSET 0 ROWS") + # SQL Server requires the backend-specific emulation (2008 or earlier) # or an offset clause (2012 or newer) for offsetting if do_offset: @@ -396,6 +410,9 @@ def _as_microsoft(self, node): as_microsoft = _as_sql_trim elif isinstance(node, Variance): as_microsoft = _as_sql_variance + if django.VERSION >= (3, 1): + if isinstance(node, json_KeyTransform): + as_microsoft = _as_sql_json_keytransform if as_microsoft: node = node.copy() node.as_microsoft = types.MethodType(as_microsoft, node) diff --git a/mssql/creation.py b/mssql/creation.py index cca18760..18fcfb85 100644 --- a/mssql/creation.py +++ b/mssql/creation.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import binascii import os diff --git a/mssql/features.py b/mssql/features.py index 6e840b98..19208580 100644 --- a/mssql/features.py +++ b/mssql/features.py @@ -1,34 +1,43 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. from django.db.backends.base.features import BaseDatabaseFeatures from django.utils.functional import cached_property class DatabaseFeatures(BaseDatabaseFeatures): - has_native_uuid_field = False allow_sliced_subqueries_with_in = False can_introspect_autofield = True + can_introspect_json_field = False can_introspect_small_integer_field = True can_return_columns_from_insert = True can_return_id_from_insert = True can_use_chunked_reads = False for_update_after_from = True greatest_least_ignores_nulls = True + has_json_object_function = False + has_json_operators = False + has_native_json_field = False + has_native_uuid_field = False has_real_datatype = True has_select_for_update = True has_select_for_update_nowait = True has_select_for_update_skip_locked = True - ignores_table_name_case = True ignores_quoted_identifier_case = True + ignores_table_name_case = True + order_by_nulls_first = True requires_literal_defaults = True requires_sqlparse_for_splitting = False supports_boolean_expr_in_select_clause = False supports_deferrable_unique_constraints = False + supports_expression_indexes = False supports_ignore_conflicts = False supports_index_on_text_field = False - supports_json_field = False + supports_json_field_contains = False + supports_order_by_nulls_modifier = False + supports_over_clause = True supports_paramstyle_pyformat = False + supports_primitives_in_json_field = False supports_regex_backreferencing = True supports_sequence_reset = False supports_subqueries_in_group_by = False @@ -63,3 +72,7 @@ def has_zoneinfo_database(self): with self.connection.cursor() as cursor: cursor.execute("SELECT TOP 1 1 FROM sys.time_zone_info") return cursor.fetchone() is not None + + @cached_property + def supports_json_field(self): + return self.connection.sql_server_version >= 2016 diff --git a/mssql/functions.py b/mssql/functions.py index b1a5b99f..cfb6e033 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -1,12 +1,26 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. + +import json from django import VERSION -from django.db.models import BooleanField -from django.db.models.functions import Cast +from django.db import NotSupportedError +from django.db.models import BooleanField, Value +from django.db.models.functions import Cast, NthValue from django.db.models.functions.math import ATan2, Log, Ln, Mod, Round -from django.db.models.expressions import Case, Exists, OrderBy, When +from django.db.models.expressions import Case, Exists, OrderBy, When, Window from django.db.models.lookups import Lookup, In +from django.db.models import lookups +from django.db.models.fields import BinaryField, Field +from django.core import validators + +if VERSION >= (3, 1): + from django.db.models.fields.json import ( + KeyTransform, KeyTransformIn, KeyTransformExact, + HasKeyLookup, compile_json_path) + +if VERSION >= (3, 2): + from django.db.models.functions.math import Random DJANGO3 = VERSION[0] >= 3 @@ -43,9 +57,22 @@ def sqlserver_mod(self, compiler, connection): ) +def sqlserver_nth_value(self, compiler, connection, **extra_content): + raise NotSupportedError('This backend does not support the NthValue function') + + def sqlserver_round(self, compiler, connection, **extra_context): return self.as_sql(compiler, connection, template='%(function)s(%(expressions)s, 0)', **extra_context) +def sqlserver_random(self, compiler, connection, **extra_context): + return self.as_sql(compiler, connection, function='RAND', **extra_context) + +def sqlserver_window(self, compiler, connection, template=None): + # MSSQL window functions require an OVER clause with ORDER BY + if self.order_by is None: + self.order_by = Value('SELECT NULL') + return self.as_sql(compiler, connection, template) + def sqlserver_exists(self, compiler, connection, template=None, **extra_context): # MS SQL doesn't allow EXISTS() in the SELECT list, so wrap it with a @@ -71,24 +98,27 @@ def sqlserver_lookup(self, compiler, connection): def sqlserver_orderby(self, compiler, connection): - # MSSQL doesn't allow ORDER BY EXISTS() unless it's wrapped in - # a CASE WHEN. - template = None if self.nulls_last: template = 'CASE WHEN %(expression)s IS NULL THEN 1 ELSE 0 END, %(expression)s %(ordering)s' if self.nulls_first: template = 'CASE WHEN %(expression)s IS NULL THEN 0 ELSE 1 END, %(expression)s %(ordering)s' + copy = self.copy() + + # Prevent OrderBy.as_sql() from modifying supplied templates + copy.nulls_first = False + copy.nulls_last = False + + # MSSQL doesn't allow ORDER BY EXISTS() unless it's wrapped in a CASE WHEN. if isinstance(self.expression, Exists): - copy = self.copy() copy.expression = Case( When(self.expression, then=True), default=False, output_field=BooleanField(), ) - return copy.as_sql(compiler, connection, template=template) - return self.as_sql(compiler, connection, template=template) + + return copy.as_sql(compiler, connection, template=template) def split_parameter_list_as_sql(self, compiler, connection): @@ -98,7 +128,8 @@ def split_parameter_list_as_sql(self, compiler, connection): with connection.cursor() as cursor: cursor.execute("IF OBJECT_ID('tempdb.dbo.#Temp_params', 'U') IS NOT NULL DROP TABLE #Temp_params; ") - cursor.execute("CREATE TABLE #Temp_params (params nvarchar(32))") + parameter_data_type = self.lhs.field.db_type(connection) + cursor.execute(f"CREATE TABLE #Temp_params (params {parameter_data_type})") for offset in range(0, len(rhs_params), 1000): sqls_params = rhs_params[offset: offset + 1000] sqls_params = ", ".join("('{}')".format(item) for item in sqls_params) @@ -108,12 +139,81 @@ def split_parameter_list_as_sql(self, compiler, connection): return in_clause, () +def unquote_json_rhs(rhs_params): + for value in rhs_params: + value = json.loads(value) + if not isinstance(value, (list, dict)): + rhs_params = [param.replace('"', '') for param in rhs_params] + return rhs_params + +def json_KeyTransformExact_process_rhs(self, compiler, connection): + if isinstance(self.rhs, KeyTransform): + return super(lookups.Exact, self).process_rhs(compiler, connection) + rhs, rhs_params = super(KeyTransformExact, self).process_rhs(compiler, connection) + + return rhs, unquote_json_rhs(rhs_params) + +def json_KeyTransformIn(self, compiler, connection): + lhs, _ = super(KeyTransformIn, self).process_lhs(compiler, connection) + rhs, rhs_params = super(KeyTransformIn, self).process_rhs(compiler, connection) + + return (lhs + ' IN ' + rhs, unquote_json_rhs(rhs_params)) + +def json_HasKeyLookup(self, compiler, connection): + # Process JSON path from the left-hand side. + if isinstance(self.lhs, KeyTransform): + lhs, _, lhs_key_transforms = self.lhs.preprocess_lhs(compiler, connection) + lhs_json_path = compile_json_path(lhs_key_transforms) + else: + lhs, _ = self.process_lhs(compiler, connection) + lhs_json_path = '$' + sql = lhs + ' IN (SELECT ' + lhs + ' FROM ' + self.lhs.output_field.model._meta.db_table + \ + ' CROSS APPLY OPENJSON(' + lhs + ') WITH ( [json_path_value] char(1) \'%s\') WHERE [json_path_value] IS NOT NULL)' + # Process JSON path from the right-hand side. + rhs = self.rhs + rhs_params = [] + if not isinstance(rhs, (list, tuple)): + rhs = [rhs] + for key in rhs: + if isinstance(key, KeyTransform): + *_, rhs_key_transforms = key.preprocess_lhs(compiler, connection) + else: + rhs_key_transforms = [key] + rhs_params.append('%s%s' % ( + lhs_json_path, + compile_json_path(rhs_key_transforms, include_root=False), + )) + # Add condition for each key. + if self.logical_operator: + sql = '(%s)' % self.logical_operator.join([sql] * len(rhs_params)) + + return sql % tuple(rhs_params), [] + +def BinaryField_init(self, *args, **kwargs): + # Add max_length option for BinaryField, default to max + kwargs.setdefault('editable', False) + Field.__init__(self, *args, **kwargs) + if self.max_length is not None: + self.validators.append(validators.MaxLengthValidator(self.max_length)) + else: + self.max_length = 'max' + ATan2.as_microsoft = sqlserver_atan2 -Log.as_microsoft = sqlserver_log +In.split_parameter_list_as_sql = split_parameter_list_as_sql +if VERSION >= (3, 1): + KeyTransformIn.as_microsoft = json_KeyTransformIn + KeyTransformExact.process_rhs = json_KeyTransformExact_process_rhs + HasKeyLookup.as_microsoft = json_HasKeyLookup Ln.as_microsoft = sqlserver_ln +Log.as_microsoft = sqlserver_log Mod.as_microsoft = sqlserver_mod +NthValue.as_microsoft = sqlserver_nth_value Round.as_microsoft = sqlserver_round -In.split_parameter_list_as_sql = split_parameter_list_as_sql +Window.as_microsoft = sqlserver_window +BinaryField.__init__ = BinaryField_init + +if VERSION >= (3, 2): + Random.as_microsoft = sqlserver_random if DJANGO3: Lookup.as_microsoft = sqlserver_lookup @@ -121,3 +221,4 @@ def split_parameter_list_as_sql(self, compiler, connection): Exists.as_microsoft = sqlserver_exists OrderBy.as_microsoft = sqlserver_orderby + diff --git a/mssql/introspection.py b/mssql/introspection.py index 3f794cbd..65d6cf26 100644 --- a/mssql/introspection.py +++ b/mssql/introspection.py @@ -1,8 +1,9 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import pyodbc as Database +from django import VERSION from django.db.backends.base.introspection import ( BaseDatabaseIntrospection, FieldInfo, TableInfo, ) @@ -97,7 +98,11 @@ def get_table_description(self, cursor, table_name, identity_check=True): """ # map pyodbc's cursor.columns to db-api cursor description - columns = [[c[3], c[4], None, c[6], c[6], c[8], c[10], c[12]] for c in cursor.columns(table=table_name)] + if VERSION >= (3, 2): + columns = [[c[3], c[4], None, c[6], c[6], c[8], c[10], c[12], ''] for c in cursor.columns(table=table_name)] + else: + columns = [[c[3], c[4], None, c[6], c[6], c[8], c[10], c[12]] for c in cursor.columns(table=table_name)] + items = [] for column in columns: if identity_check and self._is_auto_field(cursor, table_name, column[0]): diff --git a/mssql/management/commands/install_regex_clr.py b/mssql/management/commands/install_regex_clr.py index 6cd86cfa..be027ff2 100644 --- a/mssql/management/commands/install_regex_clr.py +++ b/mssql/management/commands/install_regex_clr.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. # Add regex support in SQLServer # Code taken from django-mssql (see https://bitbucket.org/Manfre/django-mssql) diff --git a/mssql/operations.py b/mssql/operations.py index 5f8ebb47..1462a094 100644 --- a/mssql/operations.py +++ b/mssql/operations.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import datetime import uuid @@ -49,12 +49,20 @@ def bulk_batch_size(self, fields, objs): are the fields going to be inserted in the batch, the objs contains all the objects to be inserted. """ - objs_len, fields_len, max_row_values = len(objs), len(fields), 1000 - if (objs_len * fields_len) <= max_row_values: - size = objs_len - else: - size = max_row_values // fields_len - return size + max_insert_rows = 1000 + fields_len = len(fields) + if fields_len == 0: + # Required for empty model + # (bulk_create.tests.BulkCreateTests.test_empty_model) + return max_insert_rows + + # MSSQL allows a query to have 2100 parameters but some parameters are + # taken up defining `NVARCHAR` parameters to store the query text and + # query parameters for the `sp_executesql` call. This should only take + # up 2 parameters but I've had this error when sending 2098 parameters. + max_query_params = 2050 + # inserts are capped at 1000 rows regardless of number of query params. + return min(max_insert_rows, max_query_params // fields_len) def bulk_insert_sql(self, fields, placeholder_rows): placeholder_rows_sql = (", ".join(row) for row in placeholder_rows) @@ -135,7 +143,7 @@ def date_interval_sql(self, timedelta): sql = 'DATEADD(microsecond, %d%%s, CAST(%s AS datetime2))' % (timedelta.microseconds, sql) return sql - def date_trunc_sql(self, lookup_type, field_name): + def date_trunc_sql(self, lookup_type, field_name, tzname=''): CONVERT_YEAR = 'CONVERT(varchar, DATEPART(year, %s))' % field_name CONVERT_QUARTER = 'CONVERT(varchar, 1+((DATEPART(quarter, %s)-1)*3))' % field_name CONVERT_MONTH = 'CONVERT(varchar, DATEPART(month, %s))' % field_name @@ -457,7 +465,7 @@ def adapt_datetimefield_value(self, value): value = value.astimezone(self.connection.timezone).replace(tzinfo=None) return value - def time_trunc_sql(self, lookup_type, field_name): + def time_trunc_sql(self, lookup_type, field_name, tzname=''): # if self.connection.sql_server_version >= 2012: # fields = { # 'hour': 'DATEPART(hour, %s)' % field_name, diff --git a/mssql/schema.py b/mssql/schema.py index f85046fb..c35e7960 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import binascii import datetime @@ -17,8 +17,9 @@ Table, ) from django import VERSION as django_version -from django.db.models import Index +from django.db.models import Index, UniqueConstraint from django.db.models.fields import AutoField, BigAutoField +from django.db.models.sql.where import AND from django.db.transaction import TransactionManagementError from django.utils.encoding import force_str @@ -234,6 +235,7 @@ def _db_table_delete_constraint_sql(self, template, db_table, name): template, table=Table(db_table, self.quote_name), name=self.quote_name(name), + include='' ) def alter_db_table(self, model, old_db_table, new_db_table): @@ -688,8 +690,8 @@ def add_field(self, model, field): if self.connection.features.connection_persists_old_columns: self.connection.close() - def _create_unique_sql(self, model, columns, name=None, condition=None, deferrable=None): - if (deferrable and not self.connection.features.supports_deferrable_unique_constraints): + def _create_unique_sql(self, model, columns, name=None, condition=None, deferrable=None, include=None, opclasses=None): + if (deferrable and not getattr(self.connection.features, 'supports_deferrable_unique_constraints', False)): return None def create_unique_name(*args, **kwargs): @@ -712,7 +714,8 @@ def create_unique_name(*args, **kwargs): name=name, columns=columns, condition=' WHERE ' + condition, - **statement_args + **statement_args, + include='', ) if self.connection.features.supports_partial_indexes else None else: return Statement( @@ -720,38 +723,30 @@ def create_unique_name(*args, **kwargs): table=table, name=name, columns=columns, - **statement_args + **statement_args, + include='', ) def _create_index_sql(self, model, fields, *, name=None, suffix='', using='', db_tablespace=None, col_suffixes=(), sql=None, opclasses=(), - condition=None): + condition=None, include=None, expressions=None): """ Return the SQL statement to create the index for one or several fields. `sql` can be specified if the syntax differs from the standard (GIS indexes, ...). """ - tablespace_sql = self._get_index_tablespace_sql(model, fields, db_tablespace=db_tablespace) - columns = [field.column for field in fields] - sql_create_index = sql or self.sql_create_index - table = model._meta.db_table - - def create_index_name(*args, **kwargs): - nonlocal name - if name is None: - name = self._create_index_name(*args, **kwargs) - return self.quote_name(name) - - return Statement( - sql_create_index, - table=Table(table, self.quote_name), - name=IndexName(table, columns, suffix, create_index_name), - using=using, - columns=self._index_columns(table, columns, col_suffixes, opclasses), - extra=tablespace_sql, - condition=(' WHERE ' + condition) if condition else '', + if django_version >= (3, 2): + return super()._create_index_sql( + model, fields=fields, name=name, suffix=suffix, using=using, + db_tablespace=db_tablespace, col_suffixes=col_suffixes, sql=sql, + opclasses=opclasses, condition=condition, include=include, + expressions=expressions, + ) + return super()._create_index_sql( + model, fields=fields, name=name, suffix=suffix, using=using, + db_tablespace=db_tablespace, col_suffixes=col_suffixes, sql=sql, + opclasses=opclasses, condition=condition, ) - def create_model(self, model): """ Takes a model and creates a table for it in the database. @@ -955,3 +950,9 @@ def remove_field(self, model, field): for sql in list(self.deferred_sql): if isinstance(sql, Statement) and sql.references_column(model._meta.db_table, field.column): self.deferred_sql.remove(sql) + + def add_constraint(self, model, constraint): + if isinstance(constraint, UniqueConstraint) and constraint.condition and constraint.condition.connector != AND: + raise NotImplementedError("The backend does not support %s conditions on unique constraint %s." % + (constraint.condition.connector, constraint.name)) + super().add_constraint(model, constraint) diff --git a/setup.py b/setup.py index 55fd3d42..63793576 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. from os import path from setuptools import find_packages, setup @@ -24,7 +24,7 @@ setup( name='mssql-django', - version='1.0b1', + version='1.0rc1', description='Django backend for Microsoft SQL Server', long_description=long_description, long_description_content_type='text/markdown', @@ -34,8 +34,11 @@ license='BSD', packages=find_packages(), install_requires=[ + 'django>=2.2,<3.3', 'pyodbc>=3.0', + 'pytz', ], + package_data={'mssql': ['regex_clr.dll']}, classifiers=CLASSIFIERS, keywords='django', ) diff --git a/test.sh b/test.sh index 79fd40e1..3920b476 100755 --- a/test.sh +++ b/test.sh @@ -10,7 +10,7 @@ DJANGO_VERSION="$(python -m django --version)" cd django git fetch --depth=1 origin +refs/tags/*:refs/tags/* git checkout $DJANGO_VERSION -pip install -r tests/requirements/py3.txt coverage +pip install -r tests/requirements/py3.txt coverage run tests/runtests.py --settings=testapp.settings --noinput \ aggregation \ @@ -110,4 +110,6 @@ coverage run tests/runtests.py --settings=testapp.settings --noinput \ update_only_fields python -m coverage xml --include '*mssql*' --omit '*virtualenvs*' -mv coverage.xml ~/ + +# For Azure Pipelines +cp -f coverage.xml result.xml ~/ diff --git a/testapp/migrations/0011_test_unique_constraints.py b/testapp/migrations/0011_test_unique_constraints.py new file mode 100644 index 00000000..b2a26dad --- /dev/null +++ b/testapp/migrations/0011_test_unique_constraints.py @@ -0,0 +1,68 @@ +# Generated by Django 3.1.5 on 2021-01-18 00:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('testapp', '0010_pizza_topping'), + ] + + operations = [ + migrations.CreateModel( + name='TestUnsupportableUniqueConstraint', + fields=[ + ( + 'id', + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + ), + ), + ('_type', models.CharField(max_length=50)), + ('status', models.CharField(max_length=50)), + ], + options={ + 'managed': False, + }, + ), + migrations.CreateModel( + name='TestSupportableUniqueConstraint', + fields=[ + ( + 'id', + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + ), + ), + ('_type', models.CharField(max_length=50)), + ('status', models.CharField(max_length=50)), + ], + ), + migrations.AddConstraint( + model_name='testsupportableuniqueconstraint', + constraint=models.UniqueConstraint( + condition=models.Q( + ('status', 'in_progress'), + ('status', 'needs_changes'), + ('status', 'published'), + ), + fields=('_type',), + name='and_constraint', + ), + ), + migrations.AddConstraint( + model_name='testsupportableuniqueconstraint', + constraint=models.UniqueConstraint( + condition=models.Q(status__in=['in_progress', 'needs_changes']), + fields=('_type',), + name='in_constraint', + ), + ), + ] diff --git a/testapp/models.py b/testapp/models.py index fb02ebeb..8049309e 100644 --- a/testapp/models.py +++ b/testapp/models.py @@ -1,9 +1,10 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. import uuid from django.db import models +from django.db.models import Q from django.utils import timezone @@ -79,6 +80,7 @@ class TestRemoveOneToOneFieldModel(models.Model): class Topping(models.Model): name = models.UUIDField(primary_key=True, default=uuid.uuid4) + class Pizza(models.Model): name = models.UUIDField(primary_key=True, default=uuid.uuid4) toppings = models.ManyToManyField(Topping) @@ -88,3 +90,39 @@ def __str__(self): self.name, ", ".join(topping.name for topping in self.toppings.all()), ) + + +class TestUnsupportableUniqueConstraint(models.Model): + class Meta: + managed = False + constraints = [ + models.UniqueConstraint( + name='or_constraint', + fields=['_type'], + condition=(Q(status='in_progress') | Q(status='needs_changes')), + ), + ] + + _type = models.CharField(max_length=50) + status = models.CharField(max_length=50) + + +class TestSupportableUniqueConstraint(models.Model): + class Meta: + constraints = [ + models.UniqueConstraint( + name='and_constraint', + fields=['_type'], + condition=( + Q(status='in_progress') & Q(status='needs_changes') & Q(status='published') + ), + ), + models.UniqueConstraint( + name='in_constraint', + fields=['_type'], + condition=(Q(status__in=['in_progress', 'needs_changes'])), + ), + ] + + _type = models.CharField(max_length=50) + status = models.CharField(max_length=50) diff --git a/testapp/runners.py b/testapp/runners.py index a7a032ae..d34f97b8 100644 --- a/testapp/runners.py +++ b/testapp/runners.py @@ -1,22 +1,43 @@ from django.test.runner import DiscoverRunner from django.conf import settings +import xmlrunner + EXCLUDED_TESTS = getattr(settings, 'EXCLUDED_TESTS', []) REGEX_TESTS = getattr(settings, 'REGEX_TESTS', []) ENABLE_REGEX_TESTS = getattr(settings, 'ENABLE_REGEX_TESTS', False) +def MarkexpectedFailure(): + def decorator(test_item): + def wrapper(): + raise "Expected Failure" + wrapper.__unittest_expecting_failure__ = True + return wrapper + return decorator + class ExcludedTestSuiteRunner(DiscoverRunner): def build_suite(self, *args, **kwargs): suite = super().build_suite(*args, **kwargs) tests = [] for case in suite: + test_name = case._testMethodName if ENABLE_REGEX_TESTS: - if not case.id() in EXCLUDED_TESTS: - tests.append(case) + if case.id() in EXCLUDED_TESTS: + test_method = getattr(case, test_name) + setattr(case, test_name, MarkexpectedFailure()(test_method)) else: - if not case.id() in EXCLUDED_TESTS + REGEX_TESTS: - tests.append(case) + if case.id() in EXCLUDED_TESTS + REGEX_TESTS: + test_method = getattr(case, test_name) + setattr(case, test_name, MarkexpectedFailure()(test_method)) + tests.append(case) suite._tests = tests return suite + + def run_suite(self, suite): + kwargs = dict(verbosity=1, descriptions=False) + + with open('./result.xml', 'wb') as xml: + return xmlrunner.XMLTestRunner( + output=xml, **kwargs).run(suite) diff --git a/testapp/settings.py b/testapp/settings.py index ccc57c77..207716b0 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. DATABASES = { "default": { @@ -36,6 +36,8 @@ 'django.contrib.auth.hashers.PBKDF2PasswordHasher', ] +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' + ENABLE_REGEX_TESTS = False TEST_RUNNER = "testapp.runners.ExcludedTestSuiteRunner" @@ -52,6 +54,10 @@ 'expressions.tests.FTimeDeltaTests.test_duration_with_datetime_microseconds', 'expressions.tests.IterableLookupInnerExpressionsTests.test_expressions_in_lookups_join_choice', 'expressions_case.tests.CaseExpressionTests.test_annotate_with_in_clause', + 'expressions_window.tests.WindowFunctionTests.test_nth_returns_null', + 'expressions_window.tests.WindowFunctionTests.test_nthvalue', + 'expressions_window.tests.WindowFunctionTests.test_range_n_preceding_and_following', + 'field_deconstruction.tests.FieldDeconstructionTests.test_binary_field', 'ordering.tests.OrderingTests.test_orders_nulls_first_on_filtered_subquery', 'queries.test_bulk_update.BulkUpdateNoteTests.test_set_field_to_null', 'get_or_create.tests.UpdateOrCreateTransactionTests.test_creation_in_transaction', @@ -101,8 +107,6 @@ 'ordering.tests.OrderingTests.test_deprecated_values_annotate', 'queries.test_qs_combinators.QuerySetSetOperationTests.test_limits', 'backends.tests.BackendTestCase.test_unicode_password', - 'backends.tests.FkConstraintsTests.test_disable_constraint_checks_context_manager', - 'backends.tests.FkConstraintsTests.test_disable_constraint_checks_manually', 'introspection.tests.IntrospectionTests.test_get_table_description_types', 'migrations.test_commands.MigrateTests.test_migrate_syncdb_app_label', 'migrations.test_commands.MigrateTests.test_migrate_syncdb_deferred_sql_executed_with_schemaeditor', @@ -176,10 +180,6 @@ 'expressions.tests.ExpressionOperatorTests.test_lefthand_bitwise_xor_null', 'inspectdb.tests.InspectDBTestCase.test_number_field_types', 'inspectdb.tests.InspectDBTestCase.test_json_field', - 'model_fields.test_integerfield.PositiveBigIntegerFieldTests.test_backend_range_save', - 'model_fields.test_integerfield.PositiveBigIntegerFieldTests.test_coercing', - 'model_fields.test_integerfield.PositiveBigIntegerFieldTests.test_documented_range', - 'model_fields.test_integerfield.PositiveBigIntegerFieldTests.test_types', 'ordering.tests.OrderingTests.test_default_ordering_by_f_expression', 'ordering.tests.OrderingTests.test_order_by_nulls_first', 'ordering.tests.OrderingTests.test_order_by_nulls_last', @@ -188,12 +188,48 @@ 'queries.test_db_returning.ReturningValuesTests.test_insert_returning_multiple', 'dbshell.tests.DbshellCommandTestCase.test_command_missing', 'schema.tests.SchemaTests.test_char_field_pk_to_auto_field', - 'datetimes.tests.DateTimesTests.test_21432' + 'datetimes.tests.DateTimesTests.test_21432', + + # JSONFields + 'model_fields.test_jsonfield.TestQuerying.test_has_key_list', + 'model_fields.test_jsonfield.TestQuerying.test_has_key_null_value', + 'model_fields.test_jsonfield.TestQuerying.test_key_quoted_string', + 'model_fields.test_jsonfield.TestQuerying.test_lookups_with_key_transform', + 'model_fields.test_jsonfield.TestQuerying.test_ordering_grouping_by_count', + 'model_fields.test_jsonfield.TestQuerying.test_isnull_key', + 'model_fields.test_jsonfield.TestQuerying.test_none_key', + 'model_fields.test_jsonfield.TestQuerying.test_none_key_and_exact_lookup', + 'model_fields.test_jsonfield.TestQuerying.test_key_escape', + 'model_fields.test_jsonfield.TestQuerying.test_ordering_by_transform', + 'expressions_window.tests.WindowFunctionTests.test_key_transform', + + # Django 3.2 + 'migrations.test_operations.OperationTests.test_add_covering_unique_constraint', + 'migrations.test_operations.OperationTests.test_remove_covering_unique_constraint', + 'constraints.tests.CheckConstraintTests.test_database_constraint_unicode', + 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_func_with_timezone', + 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_timezone_applied_before_truncation', + 'expressions.tests.ExistsTests.test_optimizations', + 'expressions.tests.FTimeDeltaTests.test_delta_add', + 'expressions.tests.FTimeDeltaTests.test_delta_subtract', + 'expressions.tests.FTimeDeltaTests.test_delta_update', + 'expressions.tests.FTimeDeltaTests.test_exclude', + 'expressions.tests.FTimeDeltaTests.test_mixed_comparisons1', + 'expressions.tests.FTimeDeltaTests.test_negative_timedelta_update', + 'inspectdb.tests.InspectDBTestCase.test_field_types', + 'lookup.tests.LookupTests.test_in_ignore_none', + 'lookup.tests.LookupTests.test_in_ignore_none_with_unhashable_items', + 'queries.test_qs_combinators.QuerySetSetOperationTests.test_exists_union', + 'introspection.tests.IntrospectionTests.test_get_constraints_unique_indexes_orders', + 'schema.tests.SchemaTests.test_ci_cs_db_collation', + 'select_for_update.tests.SelectForUpdateTests.test_unsuported_no_key_raises_error', ] REGEX_TESTS = ['lookup.tests.LookupTests.test_regex', 'lookup.tests.LookupTests.test_regex_backreferencing', 'lookup.tests.LookupTests.test_regex_non_ascii', 'lookup.tests.LookupTests.test_regex_non_string', - 'lookup.tests.LookupTests.test_regex_null' + 'lookup.tests.LookupTests.test_regex_null', + 'model_fields.test_jsonfield.TestQuerying.test_key_iregex', + 'model_fields.test_jsonfield.TestQuerying.test_key_regex', ] diff --git a/testapp/tests/test_constraints.py b/testapp/tests/test_constraints.py index 635fa1b1..1030e9f8 100644 --- a/testapp/tests/test_constraints.py +++ b/testapp/tests/test_constraints.py @@ -1,12 +1,18 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. +from django.db import connections, migrations, models +from django.db.migrations.state import ProjectState from django.db.utils import IntegrityError -from django.test import TestCase, skipUnlessDBFeature +from django.test import TestCase, TransactionTestCase, skipUnlessDBFeature +from mssql.base import DatabaseWrapper from ..models import ( - Author, Editor, Post, - TestUniqueNullableModel, TestNullableUniqueTogetherModel, + Author, + Editor, + Post, + TestUniqueNullableModel, + TestNullableUniqueTogetherModel, ) @@ -55,3 +61,52 @@ def test_after_type_change(self): TestNullableUniqueTogetherModel.objects.create(a='aaa', b='bbb', c='ccc') with self.assertRaises(IntegrityError): TestNullableUniqueTogetherModel.objects.create(a='aaa', b='bbb', c='ccc') + + +class TestUniqueConstraints(TransactionTestCase): + def test_unsupportable_unique_constraint(self): + # Only execute tests when running against SQL Server + connection = connections['default'] + if isinstance(connection, DatabaseWrapper): + + class TestMigration(migrations.Migration): + initial = True + + operations = [ + migrations.CreateModel( + name='TestUnsupportableUniqueConstraint', + fields=[ + ( + 'id', + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + ), + ), + ('_type', models.CharField(max_length=50)), + ('status', models.CharField(max_length=50)), + ], + ), + migrations.AddConstraint( + model_name='testunsupportableuniqueconstraint', + constraint=models.UniqueConstraint( + condition=models.Q( + ('status', 'in_progress'), + ('status', 'needs_changes'), + _connector='OR', + ), + fields=('_type',), + name='or_constraint', + ), + ), + ] + + migration = TestMigration('testapp', 'test_unsupportable_unique_constraint') + + with connection.schema_editor(atomic=True) as editor: + with self.assertRaisesRegex( + NotImplementedError, "does not support OR conditions" + ): + return migration.apply(ProjectState(), editor) diff --git a/testapp/tests/test_expressions.py b/testapp/tests/test_expressions.py index e89a0def..c10eb8ca 100644 --- a/testapp/tests/test_expressions.py +++ b/testapp/tests/test_expressions.py @@ -1,14 +1,15 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. from unittest import skipUnless from django import VERSION -from django.db.models import IntegerField +from django.db.models import IntegerField, F from django.db.models.expressions import Case, Exists, OuterRef, Subquery, Value, When -from django.test import TestCase +from django.test import TestCase, skipUnlessDBFeature -from ..models import Author, Comment, Post + +from ..models import Author, Comment, Post, Editor DJANGO3 = VERSION[0] >= 3 @@ -54,3 +55,25 @@ def test_order_by_exists(self): authors_by_posts = Author.objects.order_by(Exists(Post.objects.filter(author=OuterRef('pk'))).asc()) self.assertSequenceEqual(authors_by_posts, [author_without_posts, self.author]) + + +@skipUnless(DJANGO3, "Django 3 specific tests") +@skipUnlessDBFeature("order_by_nulls_first") +class TestOrderBy(TestCase): + def setUp(self): + self.author = Author.objects.create(name="author") + self.post = Post.objects.create(title="foo", author=self.author) + self.editor = Editor.objects.create(name="editor") + self.post_alt = Post.objects.create(title="Post with editor", author=self.author, alt_editor=self.editor) + + def test_order_by_nulls_last(self): + results = Post.objects.order_by(F("alt_editor").asc(nulls_last=True)).all() + self.assertEqual(len(results), 2) + self.assertIsNotNone(results[0].alt_editor) + self.assertIsNone(results[1].alt_editor) + + def test_order_by_nulls_first(self): + results = Post.objects.order_by(F("alt_editor").desc(nulls_first=True)).all() + self.assertEqual(len(results), 2) + self.assertIsNone(results[0].alt_editor) + self.assertIsNotNone(results[1].alt_editor) diff --git a/testapp/tests/test_fields.py b/testapp/tests/test_fields.py index da26132b..9b8f25f2 100644 --- a/testapp/tests/test_fields.py +++ b/testapp/tests/test_fields.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. from django.test import TestCase diff --git a/testapp/tests/test_lookups.py b/testapp/tests/test_lookups.py index 3ca12486..1d972805 100644 --- a/testapp/tests/test_lookups.py +++ b/testapp/tests/test_lookups.py @@ -1,5 +1,5 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. +# Licensed under the BSD license. from django.test import TestCase from ..models import Pizza, Topping diff --git a/tox.ini b/tox.ini index 19e2d25e..c0693b64 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,14 @@ [tox] envlist = {py36,py37}-django22, - {py36,py37,py38}-django30, - {py36,py37,py38}-django31, + {py36,py37,py38,py39}-django30, + {py36,py37,py38,py39}-django31, + {py36,py37,py38,py39}-django32 [testenv] allowlist_externals = /bin/bash + /usr/bin/bash C:\Program Files\Git\bin\bash.EXE commands = @@ -14,6 +16,10 @@ commands = bash test.sh deps = + coverage + unittest-xml-reporting + django22: django==2.2.* django30: django>=3.0,<3.1 django31: django>=3.1,<3.2 + django32: django==3.2.*