diff --git a/README.md b/README.md index 947e46be..2a1f93ee 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Pelican frontend +Pelican frontend has a Django backend that provides an API to the data in [Pelican backend](https://github.com/open-contracting/pelican-backend), and a Vue frontend that reports results to users. + ## Getting started Set up the git pre-commit hook: @@ -32,7 +34,7 @@ Pelican backend's database is treated as a [legacy database](https://docs.django To create `backend/dqt/models.py`: - Run `python backend/manage.py inspectdb > backend/dqt/models.py` -- Remove comments at top of file +- Replace comments at top of file - Replace `models.DO_NOTHING` with `on_delete=models.CASCADE` - `Dataset`: Add methods - `DatasetFilter.dataset_id_original`: Rename to `dataset_original`, add `related_name="dataset_filter_parent"` diff --git a/backend/dqt/models.py b/backend/dqt/models.py index 3c7ea634..63f953a2 100644 --- a/backend/dqt/models.py +++ b/backend/dqt/models.py @@ -1,36 +1,48 @@ -from django.contrib.postgres.fields import JSONField -from django.db.models import ( - CASCADE, - BigAutoField, - BigIntegerField, - BooleanField, - CharField, - DateTimeField, - ForeignKey, - IntegerField, - Model, -) - - -class Dataset(Model): - id = BigAutoField(primary_key=True) - name = CharField(max_length=255, blank=False) - meta = JSONField() - ancestor_id = BigIntegerField(blank=True, null=True) - created = DateTimeField(blank=True, null=True) - modified = DateTimeField(blank=True, null=True) +# PostgreSQL allows `character varying` without a length specifier. However, Django requires `max_length` on +# `CharField`. As such, we set `max_length=255` on: +# +# - Report.type: an ENUM with known length strings +# - FieldLevelCheckExamples.path: with a maximum witnessed length of 75 +# - ResourceLevelCheckExamples.check_name +# - DatasetLevelCheck.check_name +# - TimeVarianceLevelCheck.check_name +# +# https://code.djangoproject.com/ticket/14094 +# https://www.postgresql.org/docs/current/datatype-character.html +from django.db import models + + +class DataItem(models.Model): + id = models.BigAutoField(primary_key=True) + data = models.JSONField() + dataset = models.ForeignKey("Dataset", on_delete=models.CASCADE) + created = models.DateTimeField(blank=True, null=True) + modified = models.DateTimeField(blank=True, null=True) + + class Meta: + managed = False + db_table = "data_item" + + +class Dataset(models.Model): + id = models.BigAutoField(primary_key=True) + name = models.CharField(max_length=255) + meta = models.JSONField() + ancestor_id = models.BigIntegerField(blank=True, null=True) + created = models.DateTimeField(blank=True, null=True) + modified = models.DateTimeField(blank=True, null=True) def get_state(self): - for item in self.progress.all(): - return item.state + if self.progress: + return self.progress.state def get_phase(self): - for item in self.progress.all(): - return item.phase + if self.progress: + return self.progress.phase def get_size(self): - for item in self.progress.all(): - return item.size + if self.progress: + return self.progress.size def get_filtered_children_ids(self): return [item.dataset_filtered.id for item in self.dataset_filter_parent.all()] @@ -48,160 +60,159 @@ def get_filter_message(self): return items[0].filter_message if items else None class Meta: - app_label = "data" managed = False db_table = "dataset" -class DatasetFilter(Model): - id = BigAutoField(primary_key=True) - dataset_original = ForeignKey( - "Dataset", - related_name="dataset_filter_parent", - db_column="dataset_id_original", - to_field="id", - on_delete=CASCADE, +class DatasetFilter(models.Model): + id = models.BigAutoField(primary_key=True) + dataset_original = models.ForeignKey( + Dataset, on_delete=models.CASCADE, related_name="dataset_filter_parent", db_column="dataset_id_original" ) - dataset_filtered = ForeignKey( - "Dataset", - related_name="dataset_filter_child", - db_column="dataset_id_filtered", - to_field="id", - on_delete=CASCADE, + dataset_filtered = models.ForeignKey( + Dataset, on_delete=models.CASCADE, related_name="dataset_filter_child", db_column="dataset_id_filtered" ) - filter_message = JSONField() - created = DateTimeField(blank=True, null=True) - modified = DateTimeField(blank=True, null=True) + filter_message = models.JSONField() + created = models.DateTimeField(blank=True, null=True) + modified = models.DateTimeField(blank=True, null=True) class Meta: - app_label = "data" managed = False db_table = "dataset_filter" -class Report(Model): - id = BigAutoField(primary_key=True) - dataset = ForeignKey("Dataset", db_column="dataset_id", to_field="id", on_delete=CASCADE) - type = CharField(max_length=-1, blank=True, null=True) - data = JSONField() - created = DateTimeField(blank=True, null=True) - modified = DateTimeField(blank=True, null=True) +class DatasetLevelCheck(models.Model): + id = models.BigAutoField(primary_key=True) + check_name = models.CharField(max_length=255, blank=True, null=True) + result = models.BooleanField(blank=True, null=True) + value = models.IntegerField(blank=True, null=True) + meta = models.JSONField() + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE) + created = models.DateTimeField(blank=True, null=True) + modified = models.DateTimeField(blank=True, null=True) class Meta: - app_label = "data" managed = False - db_table = "report" + db_table = "dataset_level_check" -class DataItem(Model): - id = BigAutoField(primary_key=True) - data = JSONField() - dataset = ForeignKey("Dataset", db_column="dataset_id", to_field="id", on_delete=CASCADE) - created = DateTimeField(blank=True, null=True) - modified = DateTimeField(blank=True, null=True) +class ExchangeRates(models.Model): + id = models.BigAutoField(primary_key=True) + valid_on = models.DateField(unique=True) + rates = models.JSONField() + created = models.DateTimeField(blank=True, null=True) + modified = models.DateTimeField(blank=True, null=True) class Meta: - app_label = "data" managed = False - db_table = "data_item" - + db_table = "exchange_rates" -class DatasetLevelCheck(Model): - id = BigAutoField(primary_key=True) - check_name = CharField(max_length=-1, blank=True, null=True) - result = BooleanField(blank=True, null=True) - value = IntegerField(blank=True, null=True) - meta = JSONField() - dataset = ForeignKey("Dataset", db_column="dataset_id", to_field="id", on_delete=CASCADE) - created = DateTimeField(blank=True, null=True) - modified = DateTimeField(blank=True, null=True) +class FieldLevelCheck(models.Model): + id = models.BigAutoField(primary_key=True) + data_item = models.ForeignKey(DataItem, on_delete=models.CASCADE) + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE) + result = models.JSONField() + created = models.DateTimeField(blank=True, null=True) + modified = models.DateTimeField(blank=True, null=True) class Meta: - app_label = "data" managed = False - db_table = "dataset_level_check" - + db_table = "field_level_check" -class FieldLevelCheck(Model): - id = BigAutoField(primary_key=True) - data_item = ForeignKey("DataItem", db_column="data_item_id", on_delete=CASCADE) - dataset = ForeignKey("Dataset", db_column="dataset_id", to_field="id", on_delete=CASCADE) - result = JSONField() - data_item = ForeignKey("DataItem", db_column="data_item_id", on_delete=CASCADE) - created = DateTimeField(blank=True, null=True) - modified = DateTimeField(blank=True, null=True) +class FieldLevelCheckExamples(models.Model): + id = models.BigAutoField(primary_key=True) + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE) + data = models.JSONField(blank=True, null=True) + path = models.CharField(max_length=255, blank=True, null=True) + created = models.DateTimeField(blank=True, null=True) + modified = models.DateTimeField(blank=True, null=True) class Meta: - app_label = "data" managed = False - db_table = "field_level_check" + db_table = "field_level_check_examples" -class TimeVarianceLevelCheck(Model): - id = BigAutoField(primary_key=True) - dataset = ForeignKey("Dataset", db_column="dataset_id", to_field="id", on_delete=CASCADE) +class ProgressMonitorDataset(models.Model): + id = models.BigAutoField(primary_key=True) + dataset = models.OneToOneField(Dataset, on_delete=models.CASCADE, related_name="progress") + state = models.CharField(max_length=255, blank=True, null=True) + phase = models.CharField(max_length=255, blank=True, null=True) + size = models.IntegerField(blank=True, null=True) + created = models.DateTimeField(blank=True, null=True) + modified = models.DateTimeField(blank=True, null=True) + + class Meta: + managed = False + db_table = "progress_monitor_dataset" - check_name = CharField(max_length=-1, blank=True, null=True) - coverage_result = BooleanField(blank=True, null=True) - coverage_value = IntegerField(blank=True, null=True) - check_result = BooleanField(blank=True, null=True) - check_value = IntegerField(blank=True, null=True) - meta = JSONField() - created = DateTimeField(blank=True, null=True) - modified = DateTimeField(blank=True, null=True) +class ProgressMonitorItem(models.Model): + id = models.BigAutoField(primary_key=True) + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE) + data_item = models.ForeignKey(DataItem, on_delete=models.CASCADE) + state = models.CharField(max_length=255, blank=True, null=True) + created = models.DateTimeField(blank=True, null=True) + modified = models.DateTimeField(blank=True, null=True) class Meta: - app_label = "data" managed = False - db_table = "time_variance_level_check" - + db_table = "progress_monitor_item" + unique_together = (("dataset", "data_item"),) -class ProgressMonitorDataset(Model): - id = BigAutoField(primary_key=True) - dataset = ForeignKey("Dataset", related_name="progress", db_column="dataset_id", to_field="id", on_delete=CASCADE) - state = CharField(max_length=255, blank=True, null=True) - phase = CharField(max_length=255, blank=True, null=True) - size = IntegerField(blank=True, null=True) - created = DateTimeField(blank=True, null=True) - modified = DateTimeField(blank=True, null=True) +class Report(models.Model): + id = models.BigAutoField(primary_key=True) + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE) + type = models.CharField(max_length=255, blank=True, null=True) + data = models.JSONField() + created = models.DateTimeField(blank=True, null=True) + modified = models.DateTimeField(blank=True, null=True) class Meta: - app_label = "data" managed = False - db_table = "progress_monitor_dataset" - + db_table = "report" -class ProgressMonitorItem(Model): - id = BigAutoField(primary_key=True) - dataset = ForeignKey("Dataset", db_column="dataset_id", to_field="id", on_delete=CASCADE) - data_item = ForeignKey("DataItem", db_column="item_id", on_delete=CASCADE) - state = CharField(max_length=255, blank=True, null=True) - created = DateTimeField(blank=True, null=True) - modified = DateTimeField(blank=True, null=True) +class ResourceLevelCheck(models.Model): + id = models.BigAutoField(primary_key=True) + data_item = models.ForeignKey(DataItem, on_delete=models.CASCADE) + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE) + result = models.JSONField() + created = models.DateTimeField(blank=True, null=True) + modified = models.DateTimeField(blank=True, null=True) class Meta: - app_label = "data" managed = False - db_table = "progress_monitor_item" - unique_together = (("dataset_id", "item_id"),) + db_table = "resource_level_check" + + +class ResourceLevelCheckExamples(models.Model): + id = models.BigAutoField(primary_key=True) + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE) + data = models.JSONField(blank=True, null=True) + check_name = models.CharField(max_length=255, blank=True, null=True) + created = models.DateTimeField(blank=True, null=True) + modified = models.DateTimeField(blank=True, null=True) + class Meta: + managed = False + db_table = "resource_level_check_examples" -class ResourceLevelCheck(Model): - id = BigAutoField(primary_key=True) - data_item = ForeignKey("DataItem", db_column="data_item_id", on_delete=CASCADE) - dataset = ForeignKey("Dataset", db_column="dataset_id", to_field="id", on_delete=CASCADE) - result = JSONField() - data_item = ForeignKey("DataItem", db_column="data_item_id", on_delete=CASCADE) - created = DateTimeField(blank=True, null=True) - modified = DateTimeField(blank=True, null=True) +class TimeVarianceLevelCheck(models.Model): + id = models.BigAutoField(primary_key=True) + check_name = models.CharField(max_length=255, blank=True, null=True) + coverage_result = models.BooleanField(blank=True, null=True) + coverage_value = models.IntegerField(blank=True, null=True) + check_result = models.BooleanField(blank=True, null=True) + check_value = models.IntegerField(blank=True, null=True) + meta = models.JSONField() + dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE) + created = models.DateTimeField(blank=True, null=True) + modified = models.DateTimeField(blank=True, null=True) class Meta: - app_label = "data" managed = False - db_table = "resource_level_check" + db_table = "time_variance_level_check"