Allow partitioning transactions whether they were from the seed file,…

… updated afterwards (a revision) or are in a workbasket.

This allows these all to have seperate orders.

To allow a global order partion, order can be used.

To enable moving away from hardcoding that the first workbasket contains transactions, this PR introduces transactions schemas,
these can be set in settings and on import, with "seed_first" corresponding to the current implementation, "seed" and "revision"
create the corresponding transactions.

Data migrations are added to infer the value of the field.

Tests to verify transaction order are introduced, which in turn means making changes to factories so that transaction creation
and order more closely matches real world operation.

Tests found a pre-existingissue that can occur when certain records are placed into a transaction with each other and end up
out-of-order, this is pushed out to future work.

additional_codes/models.py

@@ -96,6 +96,7 @@ def autocomplete_label(self):
         return f"{self} - {self.get_description().description}"
 
     def in_use(self):
+        # Grab Measure class from measure_set to avoid a circular import.
         return (
             self.measure_set.model.objects.filter(
                 additional_code__sid=self.sid,

common/business_rules.py

@@ -232,7 +232,8 @@ def validate(self, model):
         query = dict(get_field_tuple(model, field) for field in identifying_fields)
 
         if (
-            model.__class__.objects.filter(**query)
+            type(model)
+            .objects.filter(**query)
             .approved_up_to_transaction(self.transaction)
             .exclude(version_group=model.version_group)
             .exists()

common/migrations/0002_transaction_partition_1_of_3.py

@@ -0,0 +1,43 @@
+# Generated by Django 3.2.6 on 2021-08-26 16:21
+
+import django_fsm
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    """
+    Transaction partition field 1 of 3.
+
+    1/3 Create Partition field, with partition defaulting to (1, SEED), as at the time of writing most of
+    Transactions are from the seed file.
+
+    2/3 Data migration to set Transaction partitions to (2, REVISION) and (3, DRAFT). REVISION:  Before this
+    migration was written REVISION transactions are contained in workbaskets after the first workbasket with approved
+    workbasket status. After this migration was written data schemas allow more control over SEED / REVISION
+    transactions. DRAFT:   Draft Transactions are inferred by checking for transactions not in the first workbasket
+    that lack an approved workbasket status..
+
+    3/3
+    Set the default value to (3, DRAFT)
+    """
+
+    dependencies = [
+        ("common", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="transaction",
+            options={"ordering": ("partition", "order")},
+        ),
+        migrations.AddField(
+            model_name="transaction",
+            name="partition",
+            field=django_fsm.FSMIntegerField(
+                choices=[(1, "Seed"), (2, "Revision"), (3, "Draft")],
+                db_index=True,
+                default=1,
+                protected=True,
+            ),
+        ),
+    ]

common/migrations/0003_transaction_partition_2_of_3.py

@@ -0,0 +1,71 @@
+# Generated by Django 3.2.6 on 2021-08-26 17:29
+
+from django.db import migrations
+
+
+def non_seed_transactions(apps):
+    """:return: Queryset of non seed transactions."""
+    Transaction = apps.get_model("common", "Transaction")
+    Workbasket = apps.get_model("workbaskets", "Workbasket")
+
+    return Transaction.objects.select_related("workbasket").exclude(
+        workbasket=Workbasket.objects.first(),
+    )
+
+
+def infer_revision_transaction_partitions(apps, schemaeditor):
+    """Infer the partition of Transactions to REVISION if they are not in the
+    first workbasket and their workbasket status is approved."""
+    from common.models.transactions import TransactionPartition
+    from workbaskets.validators import WorkflowStatus
+
+    apps.get_model("common", "Transaction")
+    apps.get_model("workbaskets", "Workbasket")
+
+    non_seed_transactions(apps).filter(
+        workbasket__status__in=WorkflowStatus.approved_statuses(),
+    ).update(partition=TransactionPartition.REVISION.value)
+
+
+def infer_draft_transaction_partitions(apps, schemaeditor):
+    """Infer the partition of Transactions to DRAFT if they are in an unapproved
+    workbasket that isn't the first (seed) Workbasket."""
+    from common.models.transactions import TransactionPartition
+    from workbaskets.validators import WorkflowStatus
+
+    non_seed_transactions(apps).exclude(
+        workbasket__status__in=WorkflowStatus.approved_statuses(),
+    ).update(partition=TransactionPartition.DRAFT.value)
+
+
+class Migration(migrations.Migration):
+    """
+    Transaction partition field 2 of 3.
+
+    1/3 Create Partition field, with partition defaulting to (1, SEED), as at the time of writing most of
+    Transactions are from the seed file.
+
+    2/3 Data migration to set Transaction partitions to (2, REVISION) and (3, DRAFT). REVISION:  Before this
+    migration was written REVISION transactions are contained in workbaskets after the first workbasket with approved
+    workbasket status. After this migration was written data schemas allow more control over SEED / REVISION
+    transactions. DRAFT:   Draft Transactions are inferred by checking for transactions not in the first workbasket
+    that lack an approved workbasket status..
+
+    3/3
+    Set the default value to (3, DRAFT)
+    """
+
+    dependencies = [
+        ("common", "0002_transaction_partition_1_of_3"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            infer_revision_transaction_partitions,
+            lambda apps, schema: None,
+        ),
+        migrations.RunPython(
+            infer_draft_transaction_partitions,
+            lambda apps, schema: None,
+        ),
+    ]

common/migrations/0004_transaction_partition_3_of_3.py

@@ -0,0 +1,38 @@
+# Generated by Django 3.2.6 on 2021-08-26 19:17
+
+import django_fsm
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    """
+    Transaction partition field 3 of 3.
+
+    1/3 Create Partition field, with partition defaulting to (1, SEED), as at the time of writing most of
+    Transactions are from the seed file.
+
+    2/3 Data migration to set Transaction partitions to (2, REVISION) and (3, DRAFT). REVISION:  Before this
+    migration was written REVISION transactions are contained in workbaskets after the first workbasket with approved
+    workbasket status. After this migration was written data schemas allow more control over SEED / REVISION
+    transactions. DRAFT:   Draft Transactions are inferred by checking for transactions not in the first workbasket
+    that lack an approved workbasket status..
+
+    3/3
+    Set the default value to (3, DRAFT)
+    """
+
+    dependencies = [
+        ("common", "0003_transaction_partition_2_of_3"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="transaction",
+            name="partition",
+            field=django_fsm.FSMIntegerField(
+                choices=[(1, "Seed"), (2, "Revision"), (3, "Draft")],
+                db_index=True,
+                default=3,
+            ),
+        ),
+    ]

common/models/records.py

@@ -65,10 +65,10 @@ def latest_approved(self) -> TrackedModelQuerySet:
 
     def approved_up_to_transaction(self, transaction=None) -> TrackedModelQuerySet:
         """
-        Get the approved versions of the model being queried unless there exists
-        a version of the model in a draft state within a transaction preceding
-        (and including) the given transaction in the workbasket of the given
-        transaction.
+        Get the approved versions of the model being queried, unless there
+        exists a version of the model in a draft state within a transaction
+        preceding (and including) the given transaction in the workbasket of the
+        given transaction.
 
         The generated SQL is equivalent to:
 
@@ -282,10 +282,11 @@ def get_queryset(self):
         return self.annotate_record_codes().order_by("record_code", "subrecord_code")
 
     def approved_query_filter(self, prefix=""):
+        from common.models.transactions import TransactionPartition
+
         return Q(
             **{
-                f"{prefix}transaction__workbasket__status__in": WorkflowStatus.approved_statuses(),
-                f"{prefix}transaction__workbasket__approver__isnull": False,
+                f"{prefix}transaction__partition__in": TransactionPartition.approved_partitions(),
             }
         )