models: Align with output of inspectdb

Changes: - Upgrade to Django 3.2 (JSONField) - ProgressMonitorDataset.dataset: Change to OneToOneField and update get_state, get_phase, get_size accordingly - Add models: ExchangeRates, FieldLevelCheckExamples, ResourceLevelCheckExamples Fixes: - Fix `unique_together` to reference fields, not columns - Set max_length=255 instead of max_length=-1 Cleanup: - FieldLevelCheck, ResourceLevelCheck: Remove duplicate `data_item` field - Import `models` instead of specific fields - Re-order models, fields and keyword arguments - Remove default `db_column`, `to_field` and `blank` values - Remove optional quotes around class names - Remove extra newlines
open-contracting · Oct 24, 2021 · 0c7448e · 0c7448e
1 parent bf4f120
commit 0c7448e
Show file tree

Hide file tree

Showing 2 changed files with 145 additions and 132 deletions.
diff --git a/README.md b/README.md
@@ -1,5 +1,7 @@
 # Pelican frontend
 
+Pelican frontend has a Django backend that provides an API to the data in [Pelican backend](https://github.com/open-contracting/pelican-backend), and a Vue frontend that reports results to users.
+
 ## Getting started
 
 Set up the git pre-commit hook:
@@ -32,7 +34,7 @@ Pelican backend's database is treated as a [legacy database](https://docs.django
 To create `backend/dqt/models.py`:
 
 -   Run `python backend/manage.py inspectdb > backend/dqt/models.py`
--   Remove comments at top of file
+-   Replace comments at top of file
 -   Replace `models.DO_NOTHING` with `on_delete=models.CASCADE`
 -   `Dataset`: Add methods
 -   `DatasetFilter.dataset_id_original`: Rename to `dataset_original`, add `related_name="dataset_filter_parent"`

diff --git a/backend/dqt/models.py b/backend/dqt/models.py
@@ -1,36 +1,48 @@
-from django.contrib.postgres.fields import JSONField
-from django.db.models import (
-    CASCADE,
-    BigAutoField,
-    BigIntegerField,
-    BooleanField,
-    CharField,
-    DateTimeField,
-    ForeignKey,
-    IntegerField,
-    Model,
-)
-
-
-class Dataset(Model):
-    id = BigAutoField(primary_key=True)
-    name = CharField(max_length=255, blank=False)
-    meta = JSONField()
-    ancestor_id = BigIntegerField(blank=True, null=True)
-    created = DateTimeField(blank=True, null=True)
-    modified = DateTimeField(blank=True, null=True)
+# PostgreSQL allows `character varying` without a length specifier. However, Django requires `max_length` on
+# `CharField`. As such, we set `max_length=255` on:
+#
+# - Report.type: an ENUM with known length strings
+# - FieldLevelCheckExamples.path: with a maximum witnessed length of 75
+# - ResourceLevelCheckExamples.check_name
+# - DatasetLevelCheck.check_name
+# - TimeVarianceLevelCheck.check_name
+#
+# https://code.djangoproject.com/ticket/14094
+# https://www.postgresql.org/docs/current/datatype-character.html
+from django.db import models
+
+
+class DataItem(models.Model):
+    id = models.BigAutoField(primary_key=True)
+    data = models.JSONField()
+    dataset = models.ForeignKey("Dataset", on_delete=models.CASCADE)
+    created = models.DateTimeField(blank=True, null=True)
+    modified = models.DateTimeField(blank=True, null=True)
+
+    class Meta:
+        managed = False
+        db_table = "data_item"
+
+
+class Dataset(models.Model):
+    id = models.BigAutoField(primary_key=True)
+    name = models.CharField(max_length=255)
+    meta = models.JSONField()
+    ancestor_id = models.BigIntegerField(blank=True, null=True)
+    created = models.DateTimeField(blank=True, null=True)
+    modified = models.DateTimeField(blank=True, null=True)
 
     def get_state(self):
-        for item in self.progress.all():
-            return item.state
+        if self.progress:
+            return self.progress.state
 
     def get_phase(self):
-        for item in self.progress.all():
-            return item.phase
+        if self.progress:
+            return self.progress.phase
 
     def get_size(self):
-        for item in self.progress.all():
-            return item.size
+        if self.progress:
+            return self.progress.size
 
     def get_filtered_children_ids(self):
         return [item.dataset_filtered.id for item in self.dataset_filter_parent.all()]
@@ -48,160 +60,159 @@ def get_filter_message(self):
         return items[0].filter_message if items else None
 
     class Meta:
-        app_label = "data"
         managed = False
         db_table = "dataset"
 
 
-class DatasetFilter(Model):
-    id = BigAutoField(primary_key=True)
-    dataset_original = ForeignKey(
-        "Dataset",
-        related_name="dataset_filter_parent",
-        db_column="dataset_id_original",
-        to_field="id",
-        on_delete=CASCADE,
+class DatasetFilter(models.Model):
+    id = models.BigAutoField(primary_key=True)
+    dataset_original = models.ForeignKey(
+        Dataset, on_delete=models.CASCADE, related_name="dataset_filter_parent", db_column="dataset_id_original"
     )
-    dataset_filtered = ForeignKey(
-        "Dataset",
-        related_name="dataset_filter_child",
-        db_column="dataset_id_filtered",
-        to_field="id",
-        on_delete=CASCADE,
+    dataset_filtered = models.ForeignKey(
+        Dataset, on_delete=models.CASCADE, related_name="dataset_filter_child", db_column="dataset_id_filtered"
     )
-    filter_message = JSONField()
-    created = DateTimeField(blank=True, null=True)
-    modified = DateTimeField(blank=True, null=True)
+    filter_message = models.JSONField()
+    created = models.DateTimeField(blank=True, null=True)
+    modified = models.DateTimeField(blank=True, null=True)
 
     class Meta:
-        app_label = "data"
         managed = False
         db_table = "dataset_filter"
 
 
-class Report(Model):
-    id = BigAutoField(primary_key=True)
-    dataset = ForeignKey("Dataset", db_column="dataset_id", to_field="id", on_delete=CASCADE)
-    type = CharField(max_length=-1, blank=True, null=True)
-    data = JSONField()
-    created = DateTimeField(blank=True, null=True)
-    modified = DateTimeField(blank=True, null=True)
+class DatasetLevelCheck(models.Model):
+    id = models.BigAutoField(primary_key=True)
+    check_name = models.CharField(max_length=255, blank=True, null=True)
+    result = models.BooleanField(blank=True, null=True)
+    value = models.IntegerField(blank=True, null=True)
+    meta = models.JSONField()
+    dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE)
+    created = models.DateTimeField(blank=True, null=True)
+    modified = models.DateTimeField(blank=True, null=True)
 
     class Meta:
-        app_label = "data"
         managed = False
-        db_table = "report"
+        db_table = "dataset_level_check"
 
 
-class DataItem(Model):
-    id = BigAutoField(primary_key=True)
-    data = JSONField()
-    dataset = ForeignKey("Dataset", db_column="dataset_id", to_field="id", on_delete=CASCADE)
-    created = DateTimeField(blank=True, null=True)
-    modified = DateTimeField(blank=True, null=True)
+class ExchangeRates(models.Model):
+    id = models.BigAutoField(primary_key=True)
+    valid_on = models.DateField(unique=True)
+    rates = models.JSONField()
+    created = models.DateTimeField(blank=True, null=True)
+    modified = models.DateTimeField(blank=True, null=True)
 
     class Meta:
-        app_label = "data"
         managed = False
-        db_table = "data_item"
-
+        db_table = "exchange_rates"
 
-class DatasetLevelCheck(Model):
-    id = BigAutoField(primary_key=True)
-    check_name = CharField(max_length=-1, blank=True, null=True)
-    result = BooleanField(blank=True, null=True)
-    value = IntegerField(blank=True, null=True)
-    meta = JSONField()
-    dataset = ForeignKey("Dataset", db_column="dataset_id", to_field="id", on_delete=CASCADE)
 
-    created = DateTimeField(blank=True, null=True)
-    modified = DateTimeField(blank=True, null=True)
+class FieldLevelCheck(models.Model):
+    id = models.BigAutoField(primary_key=True)
+    data_item = models.ForeignKey(DataItem, on_delete=models.CASCADE)
+    dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE)
+    result = models.JSONField()
+    created = models.DateTimeField(blank=True, null=True)
+    modified = models.DateTimeField(blank=True, null=True)
 
     class Meta:
-        app_label = "data"
         managed = False
-        db_table = "dataset_level_check"
-
+        db_table = "field_level_check"
 
-class FieldLevelCheck(Model):
-    id = BigAutoField(primary_key=True)
-    data_item = ForeignKey("DataItem", db_column="data_item_id", on_delete=CASCADE)
-    dataset = ForeignKey("Dataset", db_column="dataset_id", to_field="id", on_delete=CASCADE)
-    result = JSONField()
-    data_item = ForeignKey("DataItem", db_column="data_item_id", on_delete=CASCADE)
 
-    created = DateTimeField(blank=True, null=True)
-    modified = DateTimeField(blank=True, null=True)
+class FieldLevelCheckExamples(models.Model):
+    id = models.BigAutoField(primary_key=True)
+    dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE)
+    data = models.JSONField(blank=True, null=True)
+    path = models.CharField(max_length=255, blank=True, null=True)
+    created = models.DateTimeField(blank=True, null=True)
+    modified = models.DateTimeField(blank=True, null=True)
 
     class Meta:
-        app_label = "data"
         managed = False
-        db_table = "field_level_check"
+        db_table = "field_level_check_examples"
 
 
-class TimeVarianceLevelCheck(Model):
-    id = BigAutoField(primary_key=True)
-    dataset = ForeignKey("Dataset", db_column="dataset_id", to_field="id", on_delete=CASCADE)
+class ProgressMonitorDataset(models.Model):
+    id = models.BigAutoField(primary_key=True)
+    dataset = models.OneToOneField(Dataset, on_delete=models.CASCADE, related_name="progress")
+    state = models.CharField(max_length=255, blank=True, null=True)
+    phase = models.CharField(max_length=255, blank=True, null=True)
+    size = models.IntegerField(blank=True, null=True)
+    created = models.DateTimeField(blank=True, null=True)
+    modified = models.DateTimeField(blank=True, null=True)
+
+    class Meta:
+        managed = False
+        db_table = "progress_monitor_dataset"
 
-    check_name = CharField(max_length=-1, blank=True, null=True)
-    coverage_result = BooleanField(blank=True, null=True)
-    coverage_value = IntegerField(blank=True, null=True)
-    check_result = BooleanField(blank=True, null=True)
-    check_value = IntegerField(blank=True, null=True)
-    meta = JSONField()
 
-    created = DateTimeField(blank=True, null=True)
-    modified = DateTimeField(blank=True, null=True)
+class ProgressMonitorItem(models.Model):
+    id = models.BigAutoField(primary_key=True)
+    dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE)
+    data_item = models.ForeignKey(DataItem, on_delete=models.CASCADE)
+    state = models.CharField(max_length=255, blank=True, null=True)
+    created = models.DateTimeField(blank=True, null=True)
+    modified = models.DateTimeField(blank=True, null=True)
 
     class Meta:
-        app_label = "data"
         managed = False
-        db_table = "time_variance_level_check"
-
+        db_table = "progress_monitor_item"
+        unique_together = (("dataset", "data_item"),)
 
-class ProgressMonitorDataset(Model):
-    id = BigAutoField(primary_key=True)
-    dataset = ForeignKey("Dataset", related_name="progress", db_column="dataset_id", to_field="id", on_delete=CASCADE)
-    state = CharField(max_length=255, blank=True, null=True)
-    phase = CharField(max_length=255, blank=True, null=True)
-    size = IntegerField(blank=True, null=True)
 
-    created = DateTimeField(blank=True, null=True)
-    modified = DateTimeField(blank=True, null=True)
+class Report(models.Model):
+    id = models.BigAutoField(primary_key=True)
+    dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE)
+    type = models.CharField(max_length=255, blank=True, null=True)
+    data = models.JSONField()
+    created = models.DateTimeField(blank=True, null=True)
+    modified = models.DateTimeField(blank=True, null=True)
 
     class Meta:
-        app_label = "data"
         managed = False
-        db_table = "progress_monitor_dataset"
-
+        db_table = "report"
 
-class ProgressMonitorItem(Model):
-    id = BigAutoField(primary_key=True)
-    dataset = ForeignKey("Dataset", db_column="dataset_id", to_field="id", on_delete=CASCADE)
-    data_item = ForeignKey("DataItem", db_column="item_id", on_delete=CASCADE)
-    state = CharField(max_length=255, blank=True, null=True)
 
-    created = DateTimeField(blank=True, null=True)
-    modified = DateTimeField(blank=True, null=True)
+class ResourceLevelCheck(models.Model):
+    id = models.BigAutoField(primary_key=True)
+    data_item = models.ForeignKey(DataItem, on_delete=models.CASCADE)
+    dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE)
+    result = models.JSONField()
+    created = models.DateTimeField(blank=True, null=True)
+    modified = models.DateTimeField(blank=True, null=True)
 
     class Meta:
-        app_label = "data"
         managed = False
-        db_table = "progress_monitor_item"
-        unique_together = (("dataset_id", "item_id"),)
+        db_table = "resource_level_check"
+
+
+class ResourceLevelCheckExamples(models.Model):
+    id = models.BigAutoField(primary_key=True)
+    dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE)
+    data = models.JSONField(blank=True, null=True)
+    check_name = models.CharField(max_length=255, blank=True, null=True)
+    created = models.DateTimeField(blank=True, null=True)
+    modified = models.DateTimeField(blank=True, null=True)
 
+    class Meta:
+        managed = False
+        db_table = "resource_level_check_examples"
 
-class ResourceLevelCheck(Model):
-    id = BigAutoField(primary_key=True)
-    data_item = ForeignKey("DataItem", db_column="data_item_id", on_delete=CASCADE)
-    dataset = ForeignKey("Dataset", db_column="dataset_id", to_field="id", on_delete=CASCADE)
-    result = JSONField()
-    data_item = ForeignKey("DataItem", db_column="data_item_id", on_delete=CASCADE)
 
-    created = DateTimeField(blank=True, null=True)
-    modified = DateTimeField(blank=True, null=True)
+class TimeVarianceLevelCheck(models.Model):
+    id = models.BigAutoField(primary_key=True)
+    check_name = models.CharField(max_length=255, blank=True, null=True)
+    coverage_result = models.BooleanField(blank=True, null=True)
+    coverage_value = models.IntegerField(blank=True, null=True)
+    check_result = models.BooleanField(blank=True, null=True)
+    check_value = models.IntegerField(blank=True, null=True)
+    meta = models.JSONField()
+    dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE)
+    created = models.DateTimeField(blank=True, null=True)
+    modified = models.DateTimeField(blank=True, null=True)
 
     class Meta:
-        app_label = "data"
         managed = False
-        db_table = "resource_level_check"
+        db_table = "time_variance_level_check"