TP-841 - Order and partition transactions into partitions: SEED_FILE, REVISION, UPDATE.

additional_codes/models.py

@@ -96,6 +96,7 @@ def autocomplete_label(self):
         return f"{self} - {self.get_description().description}"
 
     def in_use(self):
+        # Grab Measure class from measure_set to avoid a circular import.
         return (
             self.measure_set.model.objects.filter(
                 additional_code__sid=self.sid,

common/business_rules.py

@@ -232,7 +232,8 @@ def validate(self, model):
         query = dict(get_field_tuple(model, field) for field in identifying_fields)
 
         if (
-            model.__class__.objects.filter(**query)
+            type(model)
+            .objects.filter(**query)
             .approved_up_to_transaction(self.transaction)
             .exclude(version_group=model.version_group)
             .exists()

common/migrations/0002_transaction_partition_1_of_3.py

@@ -0,0 +1,45 @@
+# Generated by Django 3.2.6 on 2021-08-26 16:21
+
+import django_fsm
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    """
+    Transaction partition field 1 of 3.
+
+    1/3 Create Partition field, with partition defaulting to (1, SEED), as at the time of writing most of
+    Transactions are from the seed file.
+
+    2/3 Data migration to set Transaction partitions to (2, REVISION) and (3, DRAFT). REVISION:  Before this
+    migration was written REVISION transactions are contained in workbaskets after the first workbasket with approved
+    workbasket status. After this migration was written data schemas allow more control over SEED / REVISION
+    transactions. DRAFT:   Draft Transactions are inferred by checking for transactions not in the first workbasket
+    that lack an approved workbasket status..
+
+    3/3
+    Set the default value to (3, DRAFT)
+    """
+
+    dependencies = [
+        ("common", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="transaction",
+            options={"ordering": ("partition", "order")},
+        ),
+        migrations.AddField(
+            model_name="transaction",
+            name="partition",
+            # Default is SEED here but later set to DRAFT - most data is seed
+            # and this saves a lot of time updating in the database.
+            field=django_fsm.FSMIntegerField(
+                choices=[(1, "Seed"), (2, "Revision"), (3, "Draft")],
+                db_index=True,
+                default=1,
+                protected=True,
+            ),
+        ),
+    ]

common/migrations/0003_transaction_partition_2_of_3.py

@@ -0,0 +1,71 @@
+# Generated by Django 3.2.6 on 2021-08-26 17:29
+
+from django.db import migrations
+
+
+def non_seed_transactions(apps):
+    """:return: Queryset of non seed transactions."""
+    Transaction = apps.get_model("common", "Transaction")
+    Workbasket = apps.get_model("workbaskets", "Workbasket")
+
+    return Transaction.objects.select_related("workbasket").exclude(
+        workbasket=Workbasket.objects.first(),
+    )
+
+
+def infer_revision_transaction_partitions(apps, schemaeditor):
+    """Infer the partition of Transactions to REVISION if they are not in the
+    first workbasket and their workbasket status is approved."""
+    from common.models.transactions import TransactionPartition
+    from workbaskets.validators import WorkflowStatus
+
+    apps.get_model("common", "Transaction")
+    apps.get_model("workbaskets", "Workbasket")
+
+    non_seed_transactions(apps).filter(
+        workbasket__status__in=WorkflowStatus.approved_statuses(),
+    ).update(partition=TransactionPartition.REVISION.value)
+
+
+def infer_draft_transaction_partitions(apps, schemaeditor):
+    """Infer the partition of Transactions to DRAFT if they are in an unapproved
+    workbasket that isn't the first (seed) Workbasket."""
+    from common.models.transactions import TransactionPartition
+    from workbaskets.validators import WorkflowStatus
+
+    non_seed_transactions(apps).exclude(
+        workbasket__status__in=WorkflowStatus.approved_statuses(),
+    ).update(partition=TransactionPartition.DRAFT.value)
+
+
+class Migration(migrations.Migration):
+    """
+    Transaction partition field 2 of 3.
+
+    1/3 Create Partition field, with partition defaulting to (1, SEED), as at the time of writing most of
+    Transactions are from the seed file.
+
+    2/3 Data migration to set Transaction partitions to (2, REVISION) and (3, DRAFT). REVISION:  Before this
+    migration was written REVISION transactions are contained in workbaskets after the first workbasket with approved
+    workbasket status. After this migration was written data schemas allow more control over SEED / REVISION
+    transactions. DRAFT:   Draft Transactions are inferred by checking for transactions not in the first workbasket
+    that lack an approved workbasket status..
+
+    3/3
+    Set the default value to (3, DRAFT)
+    """
+
+    dependencies = [
+        ("common", "0002_transaction_partition_1_of_3"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            infer_revision_transaction_partitions,
+            lambda apps, schema: None,
+        ),
+        migrations.RunPython(
+            infer_draft_transaction_partitions,
+            lambda apps, schema: None,
+        ),
+    ]

common/migrations/0004_transaction_partition_3_of_3.py

@@ -0,0 +1,38 @@
+# Generated by Django 3.2.6 on 2021-08-26 19:17
+
+import django_fsm
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    """
+    Transaction partition field 3 of 3.
+
+    1/3 Create Partition field, with partition defaulting to (1, SEED), as at the time of writing most of
+    Transactions are from the seed file.
+
+    2/3 Data migration to set Transaction partitions to (2, REVISION) and (3, DRAFT). REVISION:  Before this
+    migration was written REVISION transactions are contained in workbaskets after the first workbasket with approved
+    workbasket status. After this migration was written data schemas allow more control over SEED / REVISION
+    transactions. DRAFT:   Draft Transactions are inferred by checking for transactions not in the first workbasket
+    that lack an approved workbasket status..
+
+    3/3
+    Set the default value to (3, DRAFT)
+    """
+
+    dependencies = [
+        ("common", "0003_transaction_partition_2_of_3"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="transaction",
+            name="partition",
+            field=django_fsm.FSMIntegerField(
+                choices=[(1, "Seed"), (2, "Revision"), (3, "Draft")],
+                db_index=True,
+                default=3,
+            ),
+        ),
+    ]

common/models/records.py

@@ -65,10 +65,10 @@ def latest_approved(self) -> TrackedModelQuerySet:
 
     def approved_up_to_transaction(self, transaction=None) -> TrackedModelQuerySet:
         """
-        Get the approved versions of the model being queried unless there exists
-        a version of the model in a draft state within a transaction preceding
-        (and including) the given transaction in the workbasket of the given
-        transaction.
+        Get the approved versions of the model being queried, unless there
+        exists a version of the model in a draft state within a transaction
+        preceding (and including) the given transaction in the workbasket of the
+        given transaction.
 
         The generated SQL is equivalent to:
 
@@ -282,10 +282,11 @@ def get_queryset(self):
         return self.annotate_record_codes().order_by("record_code", "subrecord_code")
 
     def approved_query_filter(self, prefix=""):
+        from common.models.transactions import TransactionPartition
+
         return Q(
             **{
-                f"{prefix}transaction__workbasket__status__in": WorkflowStatus.approved_statuses(),
-                f"{prefix}transaction__workbasket__approver__isnull": False,
+                f"{prefix}transaction__partition__in": TransactionPartition.approved_partitions(),
             }
         )