Add status to transaction so to differenciate between transactions in…

… draft state, where order

scope is the workbasket, and "finalised" transactions where the scope is global.

On setting the Workbasket to approved, finalise the transactions.

common/migrations/0002_transaction_status.py

@@ -0,0 +1,26 @@
+# Generated by Django 3.2.4 on 2021-07-13 23:00
+
+import django_fsm
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("common", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="transaction",
+            name="status",
+            field=django_fsm.FSMField(
+                choices=[
+                    (1, "Draft"),
+                    (2, "Finalised"),
+                ],
+                db_index=True,
+                default=1,
+            ),
+        ),
+    ]

common/migrations/0003_transaction_status_initial_data.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.2.4 on 2021-07-14 01:41
+
+from django.db import migrations
+
+
+def set_initial_transaction_status(apps, schemaeditor):
+    # Transactions that are part of approved Workbaskets
+    # will have their status set to FINALISED.
+    from common.validators import TransactionStatus
+    from workbaskets.validators import WorkflowStatus
+
+    Transaction = apps.get_model("common", "Transaction")
+
+    transactions = Transaction.objects.exclude(
+        status=TransactionStatus.FINALISED.value,
+    ).filter(
+        workbasket__status__in=WorkflowStatus.approved_statuses(),
+    )
+    transactions.update(status=TransactionStatus.FINALISED.value)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("common", "0002_transaction_status"),
+    ]
+
+    operations = [
+        migrations.RunPython(set_initial_transaction_status, lambda apps, schema: None),
+    ]

common/models/transactions.py

@@ -2,15 +2,22 @@
 from __future__ import annotations
 
 import json
+from logging import getLogger
 
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
+from django.db.transaction import atomic
+from django_fsm import FSMField
+from django_fsm import transition
 
 from common.business_rules import BusinessRuleChecker
 from common.business_rules import BusinessRuleViolation
 from common.models.mixins import TimestampedMixin
+from common.validators import TransactionStatus
+
+logger = getLogger(__name__)
 
 
 class TransactionManager(models.Manager):
@@ -41,6 +48,32 @@ def ordered_tracked_models(self):
         )  # order_by record_code, subrecord_code already happened in get_queryset
         return tracked_models
 
+    def last_finalised_transaction(self):
+        return self.filter(status=TransactionStatus.FINALISED).order_by("order").last()
+
+    def finalise_transactions(self):
+        """Set transaction status to FINALISED and ordering to global."""
+        statuses = self.distinct("status").values_list("status", flat=True)
+
+        if statuses == [TransactionStatus.FINALISED.value]:
+            logger.info("Transactions are already finalised.")
+            return
+
+        if statuses != [TransactionStatus.DRAFT.value]:
+            logger.warning("Transactions have non-DRAFT statuses: %s", statuses)
+            return
+
+        with atomic():
+            last_tx = self.model.last_finalised_transaction()
+            last_order = last_tx.order if last_tx is not None else 0
+            for global_id, txn in enumerate(
+                self.order_by("order"),
+                last_order + 1,
+            ):
+                txn.status = TransactionStatus.FINALISED
+                txn.order = global_id
+                txn.save()
+
 
 class Transaction(TimestampedMixin):
     """
@@ -53,20 +86,35 @@ class Transaction(TimestampedMixin):
     deletes. Business rules are also validated at the TrackedModel level for creates.
     """
 
+    status = FSMField(
+        default=TransactionStatus.DRAFT.value,
+        choices=TransactionStatus.choices,
+        db_index=True,
+    )
+
     import_transaction_id = models.IntegerField(null=True, editable=False)
     workbasket = models.ForeignKey(
         "workbaskets.WorkBasket",
         on_delete=models.PROTECT,
         related_name="transactions",
     )
 
-    # The order this transaction appears within the workbasket
+    # The order this transaction appears within the workbasket, once the
+    # transaction has the status Finalised it is also the global order.
     order = models.IntegerField()
 
     composite_key = models.CharField(max_length=16, unique=True)
 
     objects = TransactionManager.from_queryset(TransactionQueryset)()
 
+    @transition(
+        field=status,
+        source=TransactionStatus.DRAFT,
+        target=TransactionStatus.FINALISED,
+    )
+    def finalise_transaction(self):
+        pass
+
     def clean(self):
         """Validate business rules against contained TrackedModels."""
 
@@ -116,16 +164,21 @@ def latest_approved(cls) -> Transaction:
         WorkBasket = cls._meta.get_field("workbasket").related_model
         WorkflowStatus = type(WorkBasket._meta.get_field("status").default)
         return (
-            cls.objects.exclude(
-                workbasket=WorkBasket.objects.first(),
-            )
+            cls.objects.exclude(workbasket=WorkBasket.objects.first())
             .filter(
                 workbasket__status__in=WorkflowStatus.approved_statuses(),
             )
-            .order_by("order")
-            .last()
+            .last_finalised_transaction()
         )
 
+    @classmethod
+    def latest_approved_order(cls) -> int:
+        """
+        :return: order of last approved transaction or 0 on a fresh database.
+        """
+        tx = cls.latest_approved()
+        return 0 if tx is None else tx.order
+
 
 class TransactionGroup(models.Model):
     """

common/tests/factories.py

@@ -14,6 +14,7 @@
 from common.tests.models import TestModelDescription1
 from common.tests.util import Dates
 from common.validators import ApplicabilityCode
+from common.validators import TransactionStatus
 from common.validators import UpdateType
 from geo_areas.validators import AreaCode
 from importer.models import ImporterChunkStatus
@@ -110,7 +111,8 @@ class Meta:
     composite_key = factory.Sequence(str)
 
 
-ApprovedTransactionFactory = TransactionFactory
+class ApprovedTransactionFactory(TransactionFactory):
+    status = TransactionStatus.FINALISED
 
 
 class UnapprovedTransactionFactory(factory.django.DjangoModelFactory):

common/validators.py

@@ -71,3 +71,10 @@ class ApplicabilityCode(models.IntegerChoices):
 
 
 EnvelopeIdValidator = RegexValidator(r"^(?P<year>\d\d)(?P<counter>\d{4})$")
+
+
+class TransactionStatus(models.IntegerChoices):
+    # Newly created, order is relative to other transactions in the Workbasket
+    DRAFT = 1, "Draft"
+    # Finalised, order is global and immutable.
+    FINALISED = 2, "Finalised"

exporter/sqlite/tasks.py

@@ -24,8 +24,8 @@ def export_and_upload_sqlite() -> bool:
     not.
     """
     storage = S3Boto3Storage(bucket_name=settings.SQLITE_STORAGE_BUCKET_NAME)
-    latest_transaction = Transaction.latest_approved()
-    db_name = f"{latest_transaction.order:0>9}.db"
+    latest_transaction_order = Transaction.latest_approved_order()
+    db_name = f"{latest_transaction_order:0>9}.db"
 
     target_filename = Path(settings.SQLITE_STORAGE_DIRECTORY) / db_name
     export_filename = storage.generate_filename(str(target_filename))

workbaskets/models.py

@@ -144,6 +144,9 @@ def approve(self, user):
             version_group.current_version = obj
             version_group.save()
 
+        # Set transactions to be immutable and set their order scope to global.
+        self.transactions.finalise_transactions()
+
         # imported lower down to avoid circular import error
         from exporter.tasks import upload_workbaskets