From 113d1e95f90c515cd566893a30879d6ef3366e26 Mon Sep 17 00:00:00 2001 From: WD Date: Mon, 8 Apr 2024 15:06:24 +0800 Subject: [PATCH] support django 5.0 (#61) [Django 5.0](https://docs.djangoproject.com/en/5.0/releases/5.0/#database-computed-default-values) New Feature (db related): 1. Database-computed default values, TiDB has limited support for default value expressions, see [docs](https://docs.pingcap.com/tidb/dev/data-type-default-values#specify-expressions-as-default-values). 2. Database generated model field, TiDB also has some [limitations](https://docs.pingcap.com/tidb/stable/generated-columns). This PR also dropped support for TiDB 4.x as it has already reached its [EOL](https://www.pingcap.com/tidb-release-support-policy/). --- .github/workflows/ci.yml | 18 +- README.md | 7 +- django_test_apps.txt | 2 +- django_test_suite.sh | 1 + django_tidb/__init__.py | 2 +- django_tidb/features.py | 32 +++- django_tidb/introspection.py | 4 +- pyproject.toml | 9 +- tests/tidb_field_defaults/README.md | 3 + tests/tidb_field_defaults/__init__.py | 0 tests/tidb_field_defaults/models.py | 75 ++++++++ tests/tidb_field_defaults/tests.py | 235 ++++++++++++++++++++++++++ tox.ini | 5 +- 13 files changed, 368 insertions(+), 25 deletions(-) create mode 100644 tests/tidb_field_defaults/README.md create mode 100644 tests/tidb_field_defaults/__init__.py create mode 100644 tests/tidb_field_defaults/models.py create mode 100644 tests/tidb_field_defaults/tests.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3370178..35a1990 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,25 +30,23 @@ jobs: fail-fast: false matrix: python-version: - - '3.8' - - '3.9' - '3.10' - '3.11' + # blocked by https://github.com/lericson/pylibmc/issues/288 + # - '3.12' django-version: - - '4.2.5' + - '5.0.4' tidb-version: - - 'v7.1.1' - - 'v6.5.3' + - 'v7.5.1' + - 'v7.1.4' + - 'v6.5.8' - 'v5.4.3' - - 'v4.0.15' exclude: # Django introduced the `debug_transaction` feature in version 4.2.x, # but it does not consider databases that do not support savepoints(TiDB < 6.2.0), # as a result, all `assertNumQueries` in test cases failed. # https://github.com/django/django/commit/798e38c2b9c46ab72e2ee8c33dc822f01b194b1e - - django-version: '4.2.5' - tidb-version: 'v4.0.15' - - django-version: '4.2.5' + - django-version: '5.0.4' tidb-version: 'v5.4.3' name: py${{ matrix.python-version }}_tidb${{ matrix.tidb-version }}_django${{ matrix.django-version }} @@ -88,7 +86,7 @@ jobs: python-version: - '3.11' django-version: - - '4.2.5' + - '5.0.4' name: vector-py${{ matrix.python-version }}_django${{ matrix.django-version }} runs-on: ubuntu-latest diff --git a/README.md b/README.md index 600f64d..0895cd1 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ To install django-tidb, you need to select the version that corresponds with you |django|django-tidb|install command| |:----:|:---------:|:-------------:| +|v5.0.x|v5.0.x|`pip install 'django-tidb>=5.0.0,<5.1.0'`| |v4.2.x|v4.2.x|`pip install 'django-tidb>=4.2.0,<4.3.0'`| |v4.1.x|v4.1.x|`pip install 'django-tidb>=4.1.0,<4.2.0'`| |v3.2.x|v3.2.x|`pip install 'django-tidb>=3.2.0,<3.3.0'`| @@ -188,8 +189,8 @@ Test.objects.alias(distance=CosineDistance('embedding', [3, 1, 2])).filter(dista ## Supported versions -- TiDB 4.0 and newer -- Django 3.2, 4.1 and 4.2 +- TiDB 5.0 and newer +- Django 3.2, 4.1, 4.2 and 5.0 - Python 3.6 and newer(must match Django's Python version requirement) ## Test @@ -213,10 +214,10 @@ $ DJANGO_VERSION=3.2.12 python run_testing_worker.py Releases on PyPi before 3.0.0 are published from repository https://github.com/blacktear23/django_tidb. This repository is a new implementation and released under versions from 3.0.0. No backwards compatibility is ensured. The most significant points are: -- Only Django 3.2 and 4.0 are tested and supported. - Engine name is `django_tidb` instead of `django_tidb.tidb`. ## Known issues - TiDB before v6.6.0 does not support FOREIGN KEY constraints([#18209](https://github.com/pingcap/tidb/issues/18209)). - TiDB before v6.2.0 does not support SAVEPOINT([#6840](https://github.com/pingcap/tidb/issues/6840)). +- TiDB has limited support for default value expressions, please refer to the [documentation](https://docs.pingcap.com/tidb/dev/data-type-default-values#specify-expressions-as-default-values). diff --git a/django_test_apps.txt b/django_test_apps.txt index 321a8c4..dcf3f02 100644 --- a/django_test_apps.txt +++ b/django_test_apps.txt @@ -1,4 +1,5 @@ tidb +tidb_field_defaults admin_changelist admin_custom_urls admin_docs @@ -40,7 +41,6 @@ empty expressions_case expressions_window extra_regress -field_defaults field_subclassing file_storage file_uploads diff --git a/django_test_suite.sh b/django_test_suite.sh index ad89ce9..567da07 100755 --- a/django_test_suite.sh +++ b/django_test_suite.sh @@ -25,6 +25,7 @@ git clone --depth 1 --branch $DJANGO_VERSION https://github.com/django/django.g cp tidb_settings.py $DJANGO_TESTS_DIR/django/tidb_settings.py cp tidb_settings.py $DJANGO_TESTS_DIR/django/tests/tidb_settings.py cp -r ./tests/tidb/ $DJANGO_TESTS_DIR/django/tests/tidb/ +cp -r ./tests/tidb_field_defaults/ $DJANGO_TESTS_DIR/django/tests/tidb_field_defaults/ cd $DJANGO_TESTS_DIR/django && pip3 install -e . && pip3 install -r tests/requirements/py3.txt && pip3 install -r tests/requirements/mysql.txt; cd ../../ cd $DJANGO_TESTS_DIR/django/tests diff --git a/django_tidb/__init__.py b/django_tidb/__init__.py index 65d0451..35bb02e 100644 --- a/django_tidb/__init__.py +++ b/django_tidb/__init__.py @@ -16,7 +16,7 @@ from .patch import monkey_patch -__version__ = "4.2.3" +__version__ = "5.0.0" monkey_patch() diff --git a/django_tidb/features.py b/django_tidb/features.py index 3fe0b1b..8ed71d9 100644 --- a/django_tidb/features.py +++ b/django_tidb/features.py @@ -21,6 +21,7 @@ class DatabaseFeatures(MysqlDatabaseFeatures): has_select_for_update = True + has_native_uuid_field = False atomic_transactions = False supports_atomic_references_rename = False can_clone_databases = False @@ -34,9 +35,10 @@ class DatabaseFeatures(MysqlDatabaseFeatures): test_collations = { "ci": "utf8mb4_general_ci", "non_default": "utf8mb4_bin", + "virtual": "utf8mb4_general_ci", } - minimum_database_version = (4,) + minimum_database_version = (5,) @cached_property def supports_foreign_keys(self): @@ -75,6 +77,8 @@ def django_test_skips(self): "This doesn't work on MySQL.": { "db_functions.comparison.test_greatest.GreatestTests.test_coalesce_workaround", "db_functions.comparison.test_least.LeastTests.test_coalesce_workaround", + # UPDATE ... ORDER BY syntax on MySQL/MariaDB does not support ordering by related fields + "update.tests.AdvancedTests.test_update_ordered_by_m2m_annotation_desc", }, "MySQL doesn't support functional indexes on a function that " "returns JSON": { @@ -135,6 +139,7 @@ def django_test_skips(self): # IntegrityError not raised "constraints.tests.CheckConstraintTests.test_database_constraint", "constraints.tests.CheckConstraintTests.test_database_constraint_unicode", + # Result of function ROUND(x, d) is different from MySQL # https://github.com/pingcap/tidb/issues/26993 "db_functions.math.test_round.RoundTests.test_integer_with_negative_precision", "db_functions.text.test_chr.ChrTests.test_transform", @@ -160,8 +165,27 @@ def django_test_skips(self): "migrations.test_operations.OperationTests.test_alter_field_pk_mti_and_fk_to_base", "migrations.test_operations.OperationTests.test_alter_field_pk_mti_fk", "migrations.test_operations.OperationTests.test_create_model_with_boolean_expression_in_check_constraint", + # Unsupported adding a stored generated column through ALTER TABLE + "migrations.test_operations.OperationTests.test_add_field_after_generated_field", + "migrations.test_operations.OperationTests.test_add_generated_field_stored", + "migrations.test_operations.OperationTests.test_invalid_generated_field_changes_stored", + "migrations.test_operations.OperationTests.test_invalid_generated_field_persistency_change", + "migrations.test_operations.OperationTests.test_remove_generated_field_stored", + "schema.tests.SchemaTests.test_add_generated_field_contains", + # Failed to modify column's default value when has expression index + # https://github.com/pingcap/tidb/issues/52355 + "migrations.test_operations.OperationTests.test_alter_field_with_func_index", + # TiDB has limited support for default value expressions + # https://docs.pingcap.com/tidb/dev/data-type-default-values#specify-expressions-as-default-values + "migrations.test_operations.OperationTests.test_add_field_database_default_function", + "schema.tests.SchemaTests.test_add_text_field_with_db_default", + "schema.tests.SchemaTests.test_db_default_equivalent_sql_noop", + "schema.tests.SchemaTests.test_db_default_output_field_resolving", # about Pessimistic/Optimistic Transaction Model "select_for_update.tests.SelectForUpdateTests.test_raw_lock_not_available", + # Wrong referenced_table_schema in information_schema.key_column_usage + # https://github.com/pingcap/tidb/issues/52350 + "backends.mysql.test_introspection.TestCrossDatabaseRelations.test_omit_cross_database_relations", }, } if self.connection.tidb_version < (5,): @@ -269,6 +293,12 @@ def supports_over_clause(self): def supports_column_check_constraints(self): return True + @cached_property + def supports_expression_defaults(self): + # TiDB has limited support for default value expressions + # https://docs.pingcap.com/tidb/dev/data-type-default-values#specify-expressions-as-default-values + return True + supports_table_check_constraints = property( operator.attrgetter("supports_column_check_constraints") ) diff --git a/django_tidb/introspection.py b/django_tidb/introspection.py index 9fbb477..014462f 100644 --- a/django_tidb/introspection.py +++ b/django_tidb/introspection.py @@ -22,7 +22,8 @@ FieldInfo = namedtuple( "FieldInfo", - BaseFieldInfo._fields + ("extra", "is_unsigned", "has_json_constraint", "comment"), + BaseFieldInfo._fields + + ("extra", "is_unsigned", "has_json_constraint", "comment", "data_type"), ) InfoLine = namedtuple( "InfoLine", @@ -115,6 +116,7 @@ def to_int(i): info.is_unsigned, line[0] in json_constraints, info.comment, + info.data_type, ) ) return fields diff --git a/pyproject.toml b/pyproject.toml index 9e3175f..a175702 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,19 +10,18 @@ authors = [ ] description = "Django backend for TiDB" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.10" classifiers = [ "Development Status :: 5 - Production/Stable", "Framework :: Django", - "Framework :: Django :: 4.2", + "Framework :: Django :: 5.0", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11" + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12" ] dynamic = ["version"] diff --git a/tests/tidb_field_defaults/README.md b/tests/tidb_field_defaults/README.md new file mode 100644 index 0000000..7018cd1 --- /dev/null +++ b/tests/tidb_field_defaults/README.md @@ -0,0 +1,3 @@ +# About + +This test is copied from the Django [field_defaults](https://github.com/django/django/tree/main/tests/field_defaults), as TiDB has some [limitations](https://docs.pingcap.com/tidb/dev/data-type-default-values#specify-expressions-as-default-values) on the default expression of the field, it does not support such many expressions as MySQL. diff --git a/tests/tidb_field_defaults/__init__.py b/tests/tidb_field_defaults/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/tidb_field_defaults/models.py b/tests/tidb_field_defaults/models.py new file mode 100644 index 0000000..a602e95 --- /dev/null +++ b/tests/tidb_field_defaults/models.py @@ -0,0 +1,75 @@ +""" +Callable defaults + +You can pass callable objects as the ``default`` parameter to a field. When +the object is created without an explicit value passed in, Django will call +the method to determine the default value. + +This example uses ``datetime.datetime.now`` as the default for the ``pub_date`` +field. +""" + +from datetime import datetime +from decimal import Decimal + +from django.db import models +from django.db.models.functions import Random, Now + + +class Article(models.Model): + headline = models.CharField(max_length=100, default="Default headline") + pub_date = models.DateTimeField(default=datetime.now) + + def __str__(self): + return self.headline + + +class DBArticle(models.Model): + """ + Values or expressions can be passed as the db_default parameter to a field. + When the object is created without an explicit value passed in, the + database will insert the default value automatically. + """ + + headline = models.CharField(max_length=100, db_default="Default headline") + pub_date = models.DateTimeField(db_default=Now()) + cost = models.DecimalField( + max_digits=3, decimal_places=2, db_default=Decimal("3.33") + ) + + class Meta: + required_db_features = {"supports_expression_defaults"} + + +class DBDefaults(models.Model): + both = models.IntegerField(default=1, db_default=2) + null = models.FloatField(null=True, db_default=1.1) + + +# This model has too many db_default expressions that TiDB does not support +# class DBDefaultsFunction(models.Model): +# number = models.FloatField(db_default=Pi()) +# year = models.IntegerField(db_default=ExtractYear(Now())) +# added = models.FloatField(db_default=Pi() + 4.5) +# multiple_subfunctions = models.FloatField(db_default=Coalesce(4.5, Pi())) +# case_when = models.IntegerField( +# db_default=models.Case(models.When(GreaterThan(2, 1), then=3), default=4) +# ) + +# class Meta: +# required_db_features = {"supports_expression_defaults"} + + +class TiDBDefaultsFunction(models.Model): + number = models.DecimalField(max_digits=3, decimal_places=2, db_default=Random()) + created_at = models.DateTimeField(db_default=Now()) + + +class DBDefaultsPK(models.Model): + language_code = models.CharField(primary_key=True, max_length=2, db_default="en") + + +class DBDefaultsFK(models.Model): + language_code = models.ForeignKey( + DBDefaultsPK, db_default="fr", on_delete=models.CASCADE + ) diff --git a/tests/tidb_field_defaults/tests.py b/tests/tidb_field_defaults/tests.py new file mode 100644 index 0000000..4f46512 --- /dev/null +++ b/tests/tidb_field_defaults/tests.py @@ -0,0 +1,235 @@ +from datetime import datetime +from decimal import Decimal + +from django.core.exceptions import ValidationError +from django.db import connection +from django.db.models import Case, F, FloatField, Value, When +from django.db.models.expressions import ( + Expression, + ExpressionList, + ExpressionWrapper, + Func, + OrderByList, + RawSQL, +) +from django.db.models.functions import Collate +from django.db.models.lookups import GreaterThan +from django.test import SimpleTestCase, TestCase, skipIfDBFeature, skipUnlessDBFeature + +from .models import ( + Article, + DBArticle, + DBDefaults, + DBDefaultsFK, + # DBDefaultsFunction, + TiDBDefaultsFunction, + DBDefaultsPK, +) + + +class DefaultTests(TestCase): + def test_field_defaults(self): + a = Article() + now = datetime.now() + a.save() + + self.assertIsInstance(a.id, int) + self.assertEqual(a.headline, "Default headline") + self.assertLess((now - a.pub_date).seconds, 5) + + @skipUnlessDBFeature( + "can_return_columns_from_insert", "supports_expression_defaults" + ) + def test_field_db_defaults_returning(self): + a = DBArticle() + a.save() + self.assertIsInstance(a.id, int) + self.assertEqual(a.headline, "Default headline") + self.assertIsInstance(a.pub_date, datetime) + self.assertEqual(a.cost, Decimal("3.33")) + + @skipIfDBFeature("can_return_columns_from_insert") + @skipUnlessDBFeature("supports_expression_defaults") + def test_field_db_defaults_refresh(self): + a = DBArticle() + a.save() + a.refresh_from_db() + self.assertIsInstance(a.id, int) + self.assertEqual(a.headline, "Default headline") + self.assertIsInstance(a.pub_date, datetime) + self.assertEqual(a.cost, Decimal("3.33")) + + def test_null_db_default(self): + obj1 = DBDefaults.objects.create() + if not connection.features.can_return_columns_from_insert: + obj1.refresh_from_db() + self.assertEqual(obj1.null, 1.1) + + obj2 = DBDefaults.objects.create(null=None) + self.assertIsNone(obj2.null) + + # @skipUnlessDBFeature("supports_expression_defaults") + # def test_db_default_function(self): + # m = DBDefaultsFunction.objects.create() + # if not connection.features.can_return_columns_from_insert: + # m.refresh_from_db() + # self.assertAlmostEqual(m.number, pi) + # self.assertEqual(m.year, datetime.now().year) + # self.assertAlmostEqual(m.added, pi + 4.5) + # self.assertEqual(m.multiple_subfunctions, 4.5) + + @skipUnlessDBFeature("supports_expression_defaults") + def test_db_default_function_tidb(self): + m = TiDBDefaultsFunction.objects.create() + if not connection.features.can_return_columns_from_insert: + m.refresh_from_db() + self.assertIsInstance(m.number, Decimal) + self.assertTrue(0 <= m.number <= 1) + self.assertIsInstance(m.created_at, datetime) + self.assertEqual(m.created_at.year, datetime.now().year) + + @skipUnlessDBFeature("insert_test_table_with_defaults") + def test_both_default(self): + create_sql = connection.features.insert_test_table_with_defaults + with connection.cursor() as cursor: + cursor.execute(create_sql.format(DBDefaults._meta.db_table)) + obj1 = DBDefaults.objects.get() + self.assertEqual(obj1.both, 2) + + obj2 = DBDefaults.objects.create() + self.assertEqual(obj2.both, 1) + + def test_pk_db_default(self): + obj1 = DBDefaultsPK.objects.create() + if not connection.features.can_return_columns_from_insert: + # refresh_from_db() cannot be used because that needs the pk to + # already be known to Django. + obj1 = DBDefaultsPK.objects.get(pk="en") + self.assertEqual(obj1.pk, "en") + self.assertEqual(obj1.language_code, "en") + + obj2 = DBDefaultsPK.objects.create(language_code="de") + self.assertEqual(obj2.pk, "de") + self.assertEqual(obj2.language_code, "de") + + def test_foreign_key_db_default(self): + parent1 = DBDefaultsPK.objects.create(language_code="fr") + child1 = DBDefaultsFK.objects.create() + if not connection.features.can_return_columns_from_insert: + child1.refresh_from_db() + self.assertEqual(child1.language_code, parent1) + + parent2 = DBDefaultsPK.objects.create() + if not connection.features.can_return_columns_from_insert: + # refresh_from_db() cannot be used because that needs the pk to + # already be known to Django. + parent2 = DBDefaultsPK.objects.get(pk="en") + child2 = DBDefaultsFK.objects.create(language_code=parent2) + self.assertEqual(child2.language_code, parent2) + + # @skipUnlessDBFeature( + # "can_return_columns_from_insert", "supports_expression_defaults" + # ) + # def test_case_when_db_default_returning(self): + # m = DBDefaultsFunction.objects.create() + # self.assertEqual(m.case_when, 3) + + # @skipIfDBFeature("can_return_columns_from_insert") + # @skipUnlessDBFeature("supports_expression_defaults") + # def test_case_when_db_default_no_returning(self): + # m = DBDefaultsFunction.objects.create() + # m.refresh_from_db() + # self.assertEqual(m.case_when, 3) + + @skipUnlessDBFeature("supports_expression_defaults") + def test_bulk_create_all_db_defaults(self): + articles = [DBArticle(), DBArticle()] + DBArticle.objects.bulk_create(articles) + + headlines = DBArticle.objects.values_list("headline", flat=True) + self.assertSequenceEqual(headlines, ["Default headline", "Default headline"]) + + @skipUnlessDBFeature("supports_expression_defaults") + def test_bulk_create_all_db_defaults_one_field(self): + pub_date = datetime.now() + articles = [DBArticle(pub_date=pub_date), DBArticle(pub_date=pub_date)] + DBArticle.objects.bulk_create(articles) + + headlines = DBArticle.objects.values_list("headline", "pub_date", "cost") + self.assertSequenceEqual( + headlines, + [ + ("Default headline", pub_date, Decimal("3.33")), + ("Default headline", pub_date, Decimal("3.33")), + ], + ) + + @skipUnlessDBFeature("supports_expression_defaults") + def test_bulk_create_mixed_db_defaults(self): + articles = [DBArticle(), DBArticle(headline="Something else")] + DBArticle.objects.bulk_create(articles) + + headlines = DBArticle.objects.values_list("headline", flat=True) + self.assertCountEqual(headlines, ["Default headline", "Something else"]) + + # @skipUnlessDBFeature("supports_expression_defaults") + # def test_bulk_create_mixed_db_defaults_function(self): + # instances = [DBDefaultsFunction(), DBDefaultsFunction(year=2000)] + # DBDefaultsFunction.objects.bulk_create(instances) + + # years = DBDefaultsFunction.objects.values_list("year", flat=True) + # self.assertCountEqual(years, [2000, datetime.now().year]) + + def test_full_clean(self): + obj = DBArticle() + obj.full_clean() + obj.save() + obj.refresh_from_db() + self.assertEqual(obj.headline, "Default headline") + + obj = DBArticle(headline="Other title") + obj.full_clean() + obj.save() + obj.refresh_from_db() + self.assertEqual(obj.headline, "Other title") + + obj = DBArticle(headline="") + with self.assertRaises(ValidationError): + obj.full_clean() + + +class AllowedDefaultTests(SimpleTestCase): + def test_allowed(self): + class Max(Func): + function = "MAX" + + tests = [ + Value(10), + Max(1, 2), + RawSQL("Now()", ()), + Value(10) + Value(7), # Combined expression. + ExpressionList(Value(1), Value(2)), + ExpressionWrapper(Value(1), output_field=FloatField()), + Case(When(GreaterThan(2, 1), then=3), default=4), + ] + for expression in tests: + with self.subTest(expression=expression): + self.assertIs(expression.allowed_default, True) + + def test_disallowed(self): + class Max(Func): + function = "MAX" + + tests = [ + Expression(), + F("field"), + Max(F("count"), 1), + Value(10) + F("count"), # Combined expression. + ExpressionList(F("count"), Value(2)), + ExpressionWrapper(F("count"), output_field=FloatField()), + Collate(Value("John"), "nocase"), + OrderByList("field"), + ] + for expression in tests: + with self.subTest(expression=expression): + self.assertIs(expression.allowed_default, False) diff --git a/tox.ini b/tox.ini index c2c7d4e..6833c87 100644 --- a/tox.ini +++ b/tox.ini @@ -13,14 +13,13 @@ [tox] alwayscopy=true -envlist = py311,py310,py39,py38,lint +envlist = py312,py311,py310,lint [gh-actions] python = - 3.8: py38 - 3.9: py39 3.10: py310 3.11: py311 + 3.12: py312 [testenv] passenv = *