Allow partitioning transactions whether they were from the seed file,…

… updated afterwards (a revision) or are in a workbasket.

This allows these all to have seperate orders.

To allow a global order partion, order can be used.

To enable moving away from hardcoding that the first workbasket contains transactions, this PR introduces transactions schemas,
these can be set in settings and on import, with "seed_first" corresponding to the current implementation, "seed" and "revision"
create the corresponding transactions.

Data migrations are added to infer the value of the field.

Tests to verify transaction order are introduced, which in turn means making changes to factories so that transaction creation
and order more closely matches real world operation.

Tests found a pre-existingissue that can occur when certain records are placed into a transaction with each other and end up
out-of-order, this is pushed out to future work.

additional_codes/models.py

@@ -96,6 +96,7 @@ def autocomplete_label(self):
         return f"{self} - {self.get_description().description}"
 
     def in_use(self):
+        # Grab Measure class from measure_set to avoid a circular import.
         return (
             self.measure_set.model.objects.filter(
                 additional_code__sid=self.sid,

common/business_rules.py

@@ -232,7 +232,8 @@ def validate(self, model):
         query = dict(get_field_tuple(model, field) for field in identifying_fields)
 
         if (
-            model.__class__.objects.filter(**query)
+            type(model)
+            .objects.filter(**query)
             .approved_up_to_transaction(self.transaction)
             .exclude(version_group=model.version_group)
             .exists()

common/migrations/0002_transaction_partition_1_of_3.py

@@ -0,0 +1,42 @@
+# Generated by Django 3.2.6 on 2021-08-26 16:21
+
+import django_fsm
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    """
+    Transaction partition field 1 of 3.
+
+    1/3
+    Create Partition field, with partition defaulting to 1, SEED, as at the time of writing most of Transactions are from the seed file.
+
+    2/3
+    Data migration to set Transaction partitions to REVISION and DRAFT.
+    REVISION:  At the time of writing REVISION transactions are in the first Workbasket, this may not be true in the future.
+    DRAFT:   Draft Transactions are inferred by checking if the Transaction is in a workbasket.
+
+    3/3
+    Set the default value to 3: DRAFT
+    """
+
+    dependencies = [
+        ("common", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="transaction",
+            options={"ordering": ("partition", "order")},
+        ),
+        migrations.AddField(
+            model_name="transaction",
+            name="partition",
+            field=django_fsm.FSMIntegerField(
+                choices=[(1, "Seed"), (2, "Revision"), (3, "Draft")],
+                db_index=True,
+                default=1,
+                protected=True,
+            ),
+        ),
+    ]

common/migrations/0003_transaction_partition_2_of_3.py

@@ -0,0 +1,69 @@
+# Generated by Django 3.2.6 on 2021-08-26 17:29
+
+from django.db import migrations
+
+
+def infer_seed_transactions(apps, schemaeditor):
+    """Infer the partition of Transactions to SEED if they are in the first
+    workbasket, note the assumption is correct now, but may not be in the
+    future."""
+
+    # Transactions that are part of non-approved Workbaskets
+    # will have their finalised flag un-set.
+    # from common.validators import TransactionStatus
+    # from workbaskets.validators import WorkflowStatus
+    #
+    from common.models.transactions import TransactionPartition
+
+    Transaction = apps.get_model("common", "Transaction")
+    Workbasket = apps.get_model("workbaskets", "Workbasket")
+
+    transactions = Transaction.objects.filter(workbasket=Workbasket.objects.first())
+    transactions.update(partition=TransactionPartition.SEED_FILE.value)
+
+
+def infer_draft_transactions(apps, schemaeditor):
+    """Infer the partition of Transactions to DRAFT if they are in an unapproved
+    workbasket that isn't the first (seed) Workbasket."""
+    from common.models.transactions import TransactionPartition
+
+    Transaction = apps.get_model("common", "Transaction")
+    WorkBasket = apps.get_model("workbaskets", "WorkBasket")
+
+    from workbaskets.validators import WorkflowStatus
+
+    transactions = (
+        Transaction.objects.select_related("workbasket")
+        .exclude(workbasket=WorkBasket.objects.first())
+        .exclude(workbasket__status__in=WorkflowStatus.approved_statuses())
+    )
+
+    if transactions.count():
+        transactions.update(partition=TransactionPartition.DRAFT.value)
+
+
+class Migration(migrations.Migration):
+
+    """
+    Transaction partition field 2 of 3.
+
+    1/3
+    Create Partition field, with partition defaulting to 1, SEED, as at the time of writing most of Transactions are from the seed file.
+
+    2/3
+    Data migration to set Transaction partitions to REVISION and DRAFT.
+    REVISION:  At the time of writing REVISION transactions are in the first Workbasket, this may not be true in the future.
+    DRAFT:   Draft Transactions are inferred by checking if the Transaction is in a workbasket.
+
+    3/3
+    Set the default value to 3: DRAFT
+    """
+
+    dependencies = [
+        ("common", "0002_transaction_partition_1_of_3"),
+    ]
+
+    operations = [
+        migrations.RunPython(infer_seed_transactions, lambda apps, schema: None),
+        migrations.RunPython(infer_draft_transactions, lambda apps, schema: None),
+    ]

common/migrations/0004_transaction_partition_3_of_3.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.6 on 2021-08-26 19:17
+
+import django_fsm
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("common", "0003_transaction_partition_2_of_3"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="transaction",
+            name="partition",
+            field=django_fsm.FSMIntegerField(
+                choices=[(1, "Seed"), (2, "Revision"), (3, "Draft")],
+                db_index=True,
+                default=3,
+            ),
+        ),
+    ]

common/models/records.py

@@ -65,10 +65,10 @@ def latest_approved(self) -> TrackedModelQuerySet:
 
     def approved_up_to_transaction(self, transaction=None) -> TrackedModelQuerySet:
         """
-        Get the approved versions of the model being queried unless there exists
-        a version of the model in a draft state within a transaction preceding
-        (and including) the given transaction in the workbasket of the given
-        transaction.
+        Get the approved versions of the model being queried, unless there
+        exists a version of the model in a draft state within a transaction
+        preceding (and including) the given transaction in the workbasket of the
+        given transaction.
 
         The generated SQL is equivalent to:
 
@@ -282,10 +282,11 @@ def get_queryset(self):
         return self.annotate_record_codes().order_by("record_code", "subrecord_code")
 
     def approved_query_filter(self, prefix=""):
+        from common.models.transactions import TransactionPartition
+
         return Q(
             **{
-                f"{prefix}transaction__workbasket__status__in": WorkflowStatus.approved_statuses(),
-                f"{prefix}transaction__workbasket__approver__isnull": False,
+                f"{prefix}transaction__partition__in": TransactionPartition.approved_partitions(),
             }
         )