[WIP] Generate schema correctly with non-default column/table names. (#…

…166)

* Fixed generation of schema for fields that defined custom ``source_field`` values defined
* Fixed working with Models that have fields with custom ``source_field`` values defined
* Tests for generate schema
* Tests for basic functionality: get/filter/fk/rfk/m2m

CHANGELOG.rst

@@ -28,6 +28,8 @@ Bugfixes:
 - Fixed missing table/column comment generation for ``ForeignKeyField`` and ``ManyToManyField``
 - Fixed comment generation to escape properly for ``SQLite``
 - Fixed comment generation for ``PostgreSQL`` to not duplicate comments
+- Fixed generation of schema for fields that defined custom ``source_field`` values defined
+- Fixed working with Models that have fields with custom ``source_field`` values defined
 
 Docs/examples:
 ^^^^^^^^^^^^^^

tortoise/__init__.py

@@ -27,7 +27,7 @@
 
 logger = logging.getLogger("tortoise")
 
-if sys.version_info < (3, 6):
+if sys.version_info < (3, 6):  # pragma: nocoverage
     warnings.warn("Tortoise-ORM is soon going to require Python 3.6", DeprecationWarning)
 
 
@@ -99,14 +99,19 @@ def split_reference(reference: str) -> Tuple[str, str]:
                     related_model = get_related_model(related_app_name, related_model_name)
 
                     key_field = "{}_id".format(field)
-                    fk_object.source_field = key_field
                     key_fk_object = deepcopy(related_model._meta.pk)
                     key_fk_object.pk = False
                     key_fk_object.index = fk_object.index
                     key_fk_object.default = fk_object.default
                     key_fk_object.null = fk_object.null
                     key_fk_object.generated = fk_object.generated
                     key_fk_object.reference = fk_object
+                    if fk_object.source_field:
+                        key_fk_object.source_field = fk_object.source_field
+                        fk_object.source_field = key_field
+                    else:
+                        fk_object.source_field = key_field
+                        key_fk_object.source_field = key_field
                     model._meta.add_field(key_field, key_fk_object)
 
                     fk_object.type = related_model

tortoise/backends/asyncpg/executor.py

@@ -24,5 +24,6 @@ def _prepare_insert_statement(self, columns: List[str]) -> str:
     async def _process_insert_result(self, instance: Model, results: Optional[asyncpg.Record]):
         if results:
             generated_fields = self.model._meta.generated_db_fields
+            db_projection = instance._meta.fields_db_projection_reverse
             for key, val in zip(generated_fields, results):
-                setattr(instance, key, val)
+                setattr(instance, db_projection[key], val)

tortoise/backends/asyncpg/schema_generator.py

@@ -42,6 +42,8 @@ def _column_comment_generator(self, table: str, column: str, comment: str) -> st
         return ""
 
     def _post_table_hook(self) -> str:
-        val = "\n" + "\n".join(self.comments_array)
+        val = "\n".join(self.comments_array)
         self.comments_array = []
-        return val
+        if val:
+            return "\n" + val
+        return ""

tortoise/backends/base/executor.py

@@ -214,7 +214,7 @@ async def _prefetch_m2m_relation(self, instance_list: list, field: str, related_
             )
             for e in raw_results
         }
-        related_object_list = [related_query.model(_from_db=True, **e) for e in raw_results]
+        related_object_list = [related_query.model._init_from_db(**e) for e in raw_results]
         await self.__class__(
             model=related_query.model, db=self.db, prefetch_map=related_query._prefetch_map
         ).fetch_for_list(related_object_list)

tortoise/backends/base/schema_generator.py

@@ -11,13 +11,13 @@ class BaseSchemaGenerator:
     FIELD_TEMPLATE = '"{name}" {type} {nullable} {unique}{primary}{comment}'
     INDEX_CREATE_TEMPLATE = 'CREATE INDEX {exists}"{index_name}" ON "{table_name}" ({fields});'
     UNIQUE_CONSTRAINT_CREATE_TEMPLATE = "UNIQUE ({fields})"
-    FK_TEMPLATE = ' REFERENCES "{table}" (id) ON DELETE {on_delete}{comment}'
+    FK_TEMPLATE = ' REFERENCES "{table}" ({field}) ON DELETE {on_delete}{comment}'
     M2M_TABLE_TEMPLATE = (
         'CREATE TABLE {exists}"{table_name}" (\n'
-        '    "{backward_key}" {backward_type} NOT NULL REFERENCES "{backward_table}" (id)'
-        " ON DELETE CASCADE,\n"
-        '    "{forward_key}" {forward_type} NOT NULL REFERENCES "{forward_table}" (id)'
-        " ON DELETE CASCADE\n"
+        '    "{backward_key}" {backward_type} NOT NULL REFERENCES "{backward_table}"'
+        " ({backward_field}) ON DELETE CASCADE,\n"
+        '    "{forward_key}" {forward_type} NOT NULL REFERENCES "{forward_table}"'
+        " ({forward_field}) ON DELETE CASCADE\n"
         "){comment};"
     )
 
@@ -145,7 +145,7 @@ def _get_table_sql(self, model, safe=True) -> dict:
                 else ""
             )
             if isinstance(field_object, (fields.IntField, fields.BigIntField)) and field_object.pk:
-                fields_to_create.append(self._get_primary_key_create_string(field_name, comment))
+                fields_to_create.append(self._get_primary_key_create_string(db_field, comment))
                 continue
             nullable = "NOT NULL" if not field_object.null else ""
             unique = "UNIQUE" if field_object.unique else ""
@@ -170,14 +170,15 @@ def _get_table_sql(self, model, safe=True) -> dict:
                 )
                 field_creation_string += self.FK_TEMPLATE.format(
                     table=field_object.reference.type._meta.table,
+                    field=field_object.reference.type._meta.db_pk_field,
                     on_delete=field_object.reference.on_delete,
                     comment=comment,
                 )
                 references.add(field_object.reference.type._meta.table)
             fields_to_create.append(field_creation_string)
 
             if field_object.index:
-                fields_with_index.append(field_name)
+                fields_with_index.append(db_field)
 
         if model._meta.unique_together is not None:
             unique_together_sqls = []
@@ -244,14 +245,16 @@ def _get_table_sql(self, model, safe=True) -> dict:
                 table_name=field_object.through,
                 backward_table=model._meta.table,
                 forward_table=field_object.type._meta.table,
+                backward_field=model._meta.db_pk_field,
+                forward_field=field_object.type._meta.db_pk_field,
                 backward_key=field_object.backward_key,
                 backward_type=self._get_field_type(model._meta.pk),
                 forward_key=field_object.forward_key,
                 forward_type=self._get_field_type(field_object.type._meta.pk),
                 comment=self._table_comment_generator(
                     table=field_object.through, comment=field_object.description
                 )
-                if model._meta.table_description
+                if field_object.description
                 else "",
             )
             m2m_create_string += self._post_table_hook()

tortoise/backends/mysql/schema_generator.py

@@ -6,13 +6,13 @@ class MySQLSchemaGenerator(BaseSchemaGenerator):
     TABLE_CREATE_TEMPLATE = "CREATE TABLE {exists}`{table_name}` ({fields}){comment};"
     INDEX_CREATE_TEMPLATE = "CREATE INDEX `{index_name}` ON `{table_name}` ({fields});"
     FIELD_TEMPLATE = "`{name}` {type} {nullable} {unique}{comment}"
-    FK_TEMPLATE = " REFERENCES `{table}` (`id`) ON DELETE {on_delete}{comment}"
+    FK_TEMPLATE = " REFERENCES `{table}` (`{field}`) ON DELETE {on_delete}{comment}"
     M2M_TABLE_TEMPLATE = (
         "CREATE TABLE `{table_name}` (\n"
-        "    `{backward_key}` {backward_type} NOT NULL REFERENCES `{backward_table}` (`id`)"
-        " ON DELETE CASCADE,\n"
-        "    `{forward_key}` {forward_type} NOT NULL REFERENCES `{forward_table}` (`id`)"
-        " ON DELETE CASCADE\n"
+        "    `{backward_key}` {backward_type} NOT NULL REFERENCES `{backward_table}`"
+        " (`{backward_field}`) ON DELETE CASCADE,\n"
+        "    `{forward_key}` {forward_type} NOT NULL REFERENCES `{forward_table}`"
+        " (`{forward_field}`) ON DELETE CASCADE\n"
         "){comment};"
     )
     FIELD_TYPE_MAP = {

tortoise/filters.py

@@ -158,60 +158,97 @@ def get_filters_for_field(
         return get_m2m_filters(field_name, field)
     if isinstance(field, fields.BackwardFKRelation):
         return get_backward_fk_filters(field_name, field)
+    actual_field_name = field_name
+    if field_name == "pk" and field:
+        actual_field_name = field.model_field_name
     return {
-        field_name: {"field": source_field, "operator": operator.eq},
-        "{}__not".format(field_name): {"field": source_field, "operator": not_equal},
+        field_name: {
+            "field": actual_field_name,
+            "source_field": source_field,
+            "operator": operator.eq,
+        },
+        "{}__not".format(field_name): {
+            "field": actual_field_name,
+            "source_field": source_field,
+            "operator": not_equal,
+        },
         "{}__in".format(field_name): {
-            "field": source_field,
+            "field": actual_field_name,
+            "source_field": source_field,
             "operator": is_in,
             "value_encoder": list_encoder,
         },
         "{}__not_in".format(field_name): {
-            "field": source_field,
+            "field": actual_field_name,
+            "source_field": source_field,
             "operator": not_in,
             "value_encoder": list_encoder,
         },
         "{}__isnull".format(field_name): {
-            "field": source_field,
+            "field": actual_field_name,
+            "source_field": source_field,
             "operator": is_null,
             "value_encoder": bool_encoder,
         },
         "{}__not_isnull".format(field_name): {
-            "field": source_field,
+            "field": actual_field_name,
+            "source_field": source_field,
             "operator": not_null,
             "value_encoder": bool_encoder,
         },
-        "{}__gte".format(field_name): {"field": source_field, "operator": operator.ge},
-        "{}__lte".format(field_name): {"field": source_field, "operator": operator.le},
-        "{}__gt".format(field_name): {"field": source_field, "operator": operator.gt},
-        "{}__lt".format(field_name): {"field": source_field, "operator": operator.lt},
+        "{}__gte".format(field_name): {
+            "field": actual_field_name,
+            "source_field": source_field,
+            "operator": operator.ge,
+        },
+        "{}__lte".format(field_name): {
+            "field": actual_field_name,
+            "source_field": source_field,
+            "operator": operator.le,
+        },
+        "{}__gt".format(field_name): {
+            "field": actual_field_name,
+            "source_field": source_field,
+            "operator": operator.gt,
+        },
+        "{}__lt".format(field_name): {
+            "field": actual_field_name,
+            "source_field": source_field,
+            "operator": operator.lt,
+        },
         "{}__contains".format(field_name): {
-            "field": source_field,
+            "field": actual_field_name,
+            "source_field": source_field,
             "operator": contains,
             "value_encoder": string_encoder,
         },
         "{}__startswith".format(field_name): {
-            "field": source_field,
+            "field": actual_field_name,
+            "source_field": source_field,
             "operator": starts_with,
             "value_encoder": string_encoder,
         },
         "{}__endswith".format(field_name): {
-            "field": source_field,
+            "field": actual_field_name,
+            "source_field": source_field,
             "operator": ends_with,
             "value_encoder": string_encoder,
         },
         "{}__icontains".format(field_name): {
-            "field": source_field,
+            "field": actual_field_name,
+            "source_field": source_field,
             "operator": insensitive_contains,
             "value_encoder": string_encoder,
         },
         "{}__istartswith".format(field_name): {
-            "field": source_field,
+            "field": actual_field_name,
+            "source_field": source_field,
             "operator": insensitive_starts_with,
             "value_encoder": string_encoder,
         },
         "{}__iendswith".format(field_name): {
-            "field": source_field,
+            "field": actual_field_name,
+            "source_field": source_field,
             "operator": insensitive_ends_with,
             "value_encoder": string_encoder,
         },

tortoise/models.py

@@ -283,10 +283,10 @@ class Model(metaclass=ModelMeta):
     # I don' like this here, but it makes autocompletion and static analysis much happier
     _meta = MetaInfo(None)
 
-    def __init__(self, *args, _from_db: bool = False, **kwargs) -> None:
+    def __init__(self, *args, **kwargs) -> None:
         # self._meta is a very common attribute lookup, lets cache it.
         meta = self._meta
-        self._saved_in_db = _from_db or (meta.pk_attr in kwargs and meta.pk.generated)
+        self._saved_in_db = meta.pk_attr in kwargs and meta.pk.generated
         self._init_lazy_fkm2m()
 
         # Assign values and do type conversions
@@ -311,8 +311,9 @@ def _init_from_db(cls, **kwargs) -> MODEL_TYPE:
         meta = self._meta
 
         for key, value in kwargs.items():
-            if key in meta.fields:
-                setattr(self, key, meta.fields_map[key].to_python_value(value))
+            model_field = meta.fields_db_projection_reverse.get(key)
+            if model_field:
+                setattr(self, model_field, meta.fields_map[model_field].to_python_value(value))
 
         return self
 

tortoise/query_utils.py

@@ -34,7 +34,7 @@ def _process_filter_kwarg(model, key, value) -> Tuple[Criterion, Optional[Tuple[
             if param.get("value_encoder")
             else model._meta.db.executor_class._field_to_db(field_object, value, model)
         )
-        criterion = param["operator"](getattr(table, param["field"]), encoded_value)
+        criterion = param["operator"](getattr(table, param["source_field"]), encoded_value)
     return criterion, join
 
 

tortoise/tests/models_schema_create.py

@@ -6,7 +6,7 @@
 
 
 class Tournament(Model):
-    id = fields.IntField(pk=True)
+    tid = fields.IntField(pk=True)
     name = fields.TextField(description="Tournament name", index=True)
     created = fields.DatetimeField(auto_now_add=True, description="Created */'`/* datetime")
 
@@ -23,7 +23,7 @@ class Event(Model):
     participants = fields.ManyToManyField(
         "models.Team",
         related_name="events",
-        through="event_team",
+        through="teamevents",
         description="How participants relate",
     )
     modified = fields.DatetimeField(auto_now=True)
@@ -36,6 +36,26 @@ class Meta:
 
 class Team(Model):
     name = fields.CharField(max_length=50, pk=True, description="The TEAM name (and PK)")
+    manager = fields.ForeignKeyField("models.Team", related_name="team_members", null=True)
+    talks_to = fields.ManyToManyField("models.Team", related_name="gets_talked_to")
 
     class Meta:
         table_description = "The TEAMS!"
+
+
+class SourceFields(Model):
+    id = fields.IntField(pk=True, source_field="sometable_id")
+    chars = fields.CharField(max_length=255, source_field="some_chars_table", index=True)
+    fk = fields.ForeignKeyField(
+        "models.SourceFields", related_name="team_members", null=True, source_field="fk_sometable"
+    )
+    rel_to = fields.ManyToManyField(
+        "models.SourceFields",
+        related_name="rel_from",
+        through="sometable_self",
+        forward_key="sts_forward",
+        backward_key="backward_sts",
+    )
+
+    class Meta:
+        table = "sometable"