Allow paritioning transactions whether they were from the seed file, …

…updated afterwards (a revision) or are in a workbasket.

This allows these all to have seperate orders.

To allow a global order partion, order can be used.

Data migrations are added to infer the value of the field.

additional_codes/models.py

@@ -96,6 +96,7 @@ def autocomplete_label(self):
         return f"{self} - {self.get_description().description}"
 
     def in_use(self):
+        # Grab Measure class fom measure_set to avoid a circular import.
         return (
             self.measure_set.model.objects.filter(
                 additional_code__sid=self.sid,

additional_codes/tests/bdd/test_browse_additional_codes.py

@@ -14,7 +14,10 @@
 @pytest.fixture
 @when("I search additional codes with a <search_term>")
 def additional_code_search(search_term, client):
-    return client.get(reverse("additionalcode-list"), {"search": search_term})
+    import ipdb
+
+    with ipdb.launch_ipdb_on_exception():
+        return client.get(reverse("additionalcode-list"), {"search": search_term})
 
 
 @then("the search result should contain the additional code searched for")

common/business_rules.py

@@ -225,7 +225,8 @@ def validate(self, model):
         query = dict(get_field_tuple(model, field) for field in identifying_fields)
 
         if (
-            model.__class__.objects.filter(**query)
+            type(model)
+            .objects.filter(**query)
             .approved_up_to_transaction(self.transaction)
             .exclude(version_group=model.version_group)
             .exists()

common/migrations/0002_transaction_partition_1_of_3.py

@@ -0,0 +1,42 @@
+# Generated by Django 3.2.6 on 2021-08-26 16:21
+
+import django_fsm
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    """
+    Transaction partition field 1 of 3.
+
+    1/3
+    Create Partition field, with partition defaulting to 1, SEED, as at the time of writing most of Transactions are from the seed file.
+
+    2/3
+    Data migration to set Transaction partitions to UPDATE and DRAFT.
+    UPDATE:  At the time of writing UPDATE transactions are in the first Workbasket, this may not be true in the future.
+    DRAFT:   Draft Transactions are inferred by checking if the Transaction is in a workbasket.
+
+    3/3
+    Set the default value to 3: DRAFT
+    """
+
+    dependencies = [
+        ("common", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="transaction",
+            options={"ordering": ("partition", "order")},
+        ),
+        migrations.AddField(
+            model_name="transaction",
+            name="partition",
+            field=django_fsm.FSMIntegerField(
+                choices=[(1, "Seed"), (2, "Revision"), (3, "Draft")],
+                db_index=True,
+                default=1,
+                protected=True,
+            ),
+        ),
+    ]

common/migrations/0003_transaction_partition_2_of_3.py

@@ -0,0 +1,69 @@
+# Generated by Django 3.2.6 on 2021-08-26 17:29
+
+from django.db import migrations
+
+
+def infer_seed_transactions(apps, schemaeditor):
+    """Infer the partition of Transactions to SEED if they are in the first
+    workbasket, note the assumption is correct now, but may not be in the
+    future."""
+
+    # Transactions that are part of non-approved Workbaskets
+    # will have their finalised flag un-set.
+    # from common.validators import TransactionStatus
+    # from workbaskets.validators import WorkflowStatus
+    #
+    from common.models.transactions import TransactionPartition
+
+    Transaction = apps.get_model("common", "Transaction")
+    Workbasket = apps.get_model("workbaskets", "Workbasket")
+
+    transactions = Transaction.objects.filter(workbasket=Workbasket.objects.first())
+    transactions.update(partition=TransactionPartition.SEED_FILE.value)
+
+
+def infer_draft_transactions(apps, schemaeditor):
+    """Infer the partition of Transactions to DRAFT if they are in an unapproved
+    workbasket that isn't the first (seed) Workbasket."""
+    from common.models.transactions import TransactionPartition
+
+    Transaction = apps.get_model("common", "Transaction")
+    WorkBasket = apps.get_model("workbaskets", "WorkBasket")
+
+    from workbaskets.validators import WorkflowStatus
+
+    transactions = (
+        Transaction.objects.select_related("workbasket")
+        .exclude(workbasket=WorkBasket.objects.first())
+        .exclude(workbasket__status__in=WorkflowStatus.approved_statuses())
+    )
+
+    if transactions.count():
+        transactions.update(partition=TransactionPartition.DRAFT.value)
+
+
+class Migration(migrations.Migration):
+
+    """
+    Transaction partition field 2 of 3.
+
+    1/3
+    Create Partition field, with partition defaulting to 1, SEED, as at the time of writing most of Transactions are from the seed file.
+
+    2/3
+    Data migration to set Transaction partitions to UPDATE and DRAFT.
+    UPDATE:  At the time of writing UPDATE transactions are in the first Workbasket, this may not be true in the future.
+    DRAFT:   Draft Transactions are inferred by checking if the Transaction is in a workbasket.
+
+    3/3
+    Set the default value to 3: DRAFT
+    """
+
+    dependencies = [
+        ("common", "0002_transaction_partition_1_of_3"),
+    ]
+
+    operations = [
+        migrations.RunPython(infer_seed_transactions, lambda apps, schema: None),
+        migrations.RunPython(infer_draft_transactions, lambda apps, schema: None),
+    ]

common/migrations/0004_transaction_partition_3_of_3.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.6 on 2021-08-26 19:17
+
+import django_fsm
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("common", "0003_transaction_partition_2_of_3"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="transaction",
+            name="partition",
+            field=django_fsm.FSMIntegerField(
+                choices=[(1, "Seed"), (2, "Revision"), (3, "Draft")],
+                db_index=True,
+                default=3,
+            ),
+        ),
+    ]

common/models/records.py

@@ -121,6 +121,65 @@ def approved_up_to_transaction(self, transaction=None) -> TrackedModelQuerySet:
             .exclude(update_type=UpdateType.DELETE)
         )
 
+    def approved_up_to_transaction_new(self, transaction=None) -> TrackedModelQuerySet:
+        """
+        Get the approved versions of the model being queried unless there exists
+        a version of the model in a draft state within a transaction preceding
+        (and including) the given transaction in the workbasket of the given
+        transaction.
+
+        The generated SQL is equivalent to:
+
+        .. code:: SQL
+
+            SELECT *,
+                   Max(t3."id") filter (
+                       WHERE (
+                           t3."transaction_id" = {TRANSACTION_ID}
+                        OR ("common_transaction"."order" < {TRANSACTION_ORDER} AND "common_transaction"."workbasket_id" = {WORKBASKET_ID})
+                        OR ("workbaskets_workbasket"."approver_id" IS NOT NULL AND "workbaskets_workbasket"."status" IN (APPROVED_STATUSES))
+                       )
+                   ) AS "latest"
+              FROM "common_trackedmodel"
+             INNER JOIN "common_versiongroup"
+                ON "common_trackedmodel"."version_group_id" = "common_versiongroup"."id"
+              LEFT OUTER JOIN "common_trackedmodel" t3
+                ON "common_versiongroup"."id" = t3."version_group_id"
+              LEFT OUTER JOIN "common_transaction"
+                ON t3."transaction_id" = "common_transaction"."id"
+              LEFT OUTER JOIN "workbaskets_workbasket"
+                ON "common_transaction"."workbasket_id" = "workbaskets_workbasket"."id"
+             WHERE NOT "common_trackedmodel"."update_type" = 2
+             GROUP BY "common_trackedmodel"."id"
+            HAVING max(t3."id") filter (
+                       WHERE (
+                           t3."transaction_id" = {TRANSACTION_ID}
+                        OR ("common_transaction"."order" < {TRANSACTION_ORDER} AND "common_transaction"."workbasket_id" = {WORKBASKET_ID})
+                        OR ("workbaskets_workbasket"."approver_id" IS NOT NULL AND "workbaskets_workbasket"."status" IN (APPROVED_STATUSES))
+                       )
+            ) = "common_trackedmodel"."id"
+        """
+        if not transaction:
+            return self.latest_approved()
+
+        return (
+            self.annotate(
+                latest=Max(
+                    "version_group__versions",
+                    filter=(
+                        Q(version_group__versions__transaction=transaction)
+                        | Q(
+                            version_group__versions__transaction__workbasket=transaction.workbasket,
+                            version_group__versions__transaction__order__lt=transaction.order,
+                        )
+                        | self.approved_query_filter_new("version_group__versions__")
+                    ),
+                ),
+            )
+            .filter(latest=F("id"))
+            .exclude(update_type=UpdateType.DELETE)
+        )
+
     def latest_deleted(self) -> TrackedModelQuerySet:
         """
         Get all the latest versions of the model being queried which have been
@@ -288,6 +347,20 @@ def approved_query_filter(self, prefix=""):
             }
         )
 
+    def approved_query_filter_new(self, prefix=""):
+        from common.models.transactions import TransactionPartition
+
+        return Q(
+            **{
+                f"{prefix}transaction__partition__in": [
+                    TransactionPartition.SEED_FILE.value,
+                    TransactionPartition.REVISION.value,
+                    TransactionPartition.DRAFT.value,  ### <!- wrong !
+                ],
+                # f"{prefix}transaction__workbasket__approver__isnull": False,
+            }
+        )
+
     @staticmethod
     def _when_model_record_codes():
         """