Merge pull request #369 from data-mermaid/M1839

[M1839] Update summary models in batches
data-mermaid · Mar 24, 2023 · b36c890 · b36c890
2 parents 565cb17 + f0c4104
commit b36c890
Show file tree

Hide file tree

Showing 12 changed files with 340 additions and 64 deletions.
diff --git a/src/api/migrations/0013_auto_20230316_1608.py b/src/api/migrations/0013_auto_20230316_1608.py
@@ -0,0 +1,128 @@
+# Generated by Django 3.2.18 on 2023-03-16 16:08
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0012_auto_20230216_1925'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='beltfishobsmodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='beltfishsemodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='beltfishsumodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='benthiclitobsmodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='benthiclitsemodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='benthiclitsumodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='benthicphotoquadrattransectobsmodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='benthicphotoquadrattransectsemodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='benthicphotoquadrattransectsumodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='benthicpitobsmodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='benthicpitsemodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='benthicpitsumodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='bleachingqccoloniesbleachedobsmodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='bleachingqcquadratbenthicpercentobsmodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='bleachingqcsemodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='bleachingqcsumodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='habitatcomplexityobsmodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='habitatcomplexitysemodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='habitatcomplexitysumodel',
+            name='created_on',
+            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
+            preserve_default=False,
+        ),
+    ]
diff --git a/src/api/models/sql_models/base.py b/src/api/models/sql_models/base.py
@@ -6,7 +6,8 @@
 from api.models import Project
 
 
-project_where = """    WHERE\n        project.id = '%(project_id)s' :: uuid\n"""
+project_where = """project.id = '%(project_id)s'::uuid"""
+sample_event_where = """se.id::uuid = ANY(ARRAY[%(sample_event_ids)s]::uuid[])"""
 
 sample_event_sql_template = f"""
     WITH tags AS MATERIALIZED (
@@ -186,7 +187,8 @@
         LEFT JOIN management_compliance mc ON m.compliance_id = mc.id
         LEFT JOIN parties ON m.id = parties.management_id
         LEFT JOIN site_covariates ON site.id = site_covariates.site_id
-    {project_where}
+    WHERE
+        <<__sql_table_args__>>
 """
 
 

diff --git a/src/api/models/sql_models/beltfish.py b/src/api/models/sql_models/beltfish.py
@@ -2,7 +2,13 @@
 from django.utils.translation import gettext_lazy as _
 
 from sqltables import SQLTableArg, SQLTableManager
-from .base import BaseSQLModel, BaseSUSQLModel, sample_event_sql_template
+from .base import (
+    BaseSQLModel,
+    BaseSUSQLModel,
+    project_where,
+    sample_event_sql_template,
+    sample_event_where
+)
 
 
 class BeltFishObsSQLModel(BaseSUSQLModel):
@@ -103,7 +109,10 @@ class BeltFishObsSQLModel(BaseSUSQLModel):
             LEFT JOIN api_reefslope rs ON su.reef_slope_id = rs.id
         """
 
-    sql_args = dict(project_id=SQLTableArg(required=True))
+    sql_args = dict(
+        project_id=SQLTableArg(sql=project_where, required=True),
+        sample_event_ids=SQLTableArg(sql=sample_event_where, required=False),
+    )
 
     objects = SQLTableManager()
 
@@ -327,7 +336,10 @@ class BeltFishSUSQLModel(BaseSUSQLModel):
         ON (beltfish_su.pseudosu_id = beltfish_observers.pseudosu_id)
     """
 
-    sql_args = dict(project_id=SQLTableArg(required=True))
+    sql_args = dict(
+        project_id=SQLTableArg(sql=project_where, required=True),
+        sample_event_ids=SQLTableArg(sql=sample_event_where, required=False),
+    )
 
     objects = SQLTableManager()
 
@@ -443,7 +455,10 @@ class BeltFishSESQLModel(BaseSQLModel):
         biomass_kgha_by_fish_family_avg
     """
 
-    sql_args = dict(project_id=SQLTableArg(required=True))
+    sql_args = dict(
+        project_id=SQLTableArg(sql=project_where, required=True),
+        sample_event_ids=SQLTableArg(sql=sample_event_where, required=False),
+    )
 
     objects = SQLTableManager()
 

diff --git a/src/api/models/sql_models/benthiclit.py b/src/api/models/sql_models/benthiclit.py
@@ -2,7 +2,13 @@
 from django.utils.translation import gettext_lazy as _
 
 from sqltables import SQLTableArg, SQLTableManager
-from .base import BaseSQLModel, BaseSUSQLModel, sample_event_sql_template
+from .base import (
+    BaseSQLModel,
+    BaseSUSQLModel,
+    project_where,
+    sample_event_sql_template,
+    sample_event_where
+)
 
 
 # Unique combination of these fields defines a single (pseudo) sample unit. All other fields are aggregated.
@@ -118,7 +124,10 @@ class BenthicLITObsSQLModel(BaseSUSQLModel):
             ) benthiclit_su ON (benthiclit_obs.pseudosu_id = benthiclit_su.pseudosu_id)
     """
 
-    sql_args = dict(project_id=SQLTableArg(required=True))
+    sql_args = dict(
+        project_id=SQLTableArg(sql=project_where, required=True),
+        sample_event_ids=SQLTableArg(sql=sample_event_where, required=False),
+    )
 
     objects = SQLTableManager()
 
@@ -242,7 +251,10 @@ class BenthicLITSUSQLModel(BaseSUSQLModel):
         ON (benthiclit_su.pseudosu_id = benthiclit_observers.pseudosu_id)
     """
 
-    sql_args = dict(project_id=SQLTableArg(required=True))
+    sql_args = dict(
+        project_id=SQLTableArg(sql=project_where, required=True),
+        sample_event_ids=SQLTableArg(sql=sample_event_where, required=False),
+    )
 
     objects = SQLTableManager()
 
@@ -315,7 +327,10 @@ class BenthicLITSESQLModel(BaseSQLModel):
             data_policy_benthiclit,
             percent_cover_by_benthic_category_avg
     """
-    sql_args = dict(project_id=SQLTableArg(required=True))
+    sql_args = dict(
+        project_id=SQLTableArg(sql=project_where, required=True),
+        sample_event_ids=SQLTableArg(sql=sample_event_where, required=False),
+    )
 
     objects = SQLTableManager()
 

diff --git a/src/api/models/sql_models/benthicpit.py b/src/api/models/sql_models/benthicpit.py
@@ -2,7 +2,13 @@
 from django.utils.translation import gettext_lazy as _
 
 from sqltables import SQLTableArg, SQLTableManager
-from .base import BaseSQLModel, BaseSUSQLModel, sample_event_sql_template
+from .base import (
+    BaseSQLModel,
+    BaseSUSQLModel,
+    project_where,
+    sample_event_sql_template,
+    sample_event_where
+)
 
 
 class BenthicPITObsSQLModel(BaseSUSQLModel):
@@ -87,7 +93,10 @@ class BenthicPITObsSQLModel(BaseSUSQLModel):
             LEFT JOIN api_reefslope rs ON su.reef_slope_id = rs.id
     """
 
-    sql_args = dict(project_id=SQLTableArg(required=True))
+    sql_args = dict(
+        project_id=SQLTableArg(sql=project_where, required=True),
+        sample_event_ids=SQLTableArg(sql=sample_event_where, required=False),
+    )
 
     objects = SQLTableManager()
 
@@ -225,7 +234,10 @@ class BenthicPITSUSQLModel(BaseSUSQLModel):
         ON (benthicpit_su.pseudosu_id = benthicpit_observers.pseudosu_id)
     """
 
-    sql_args = dict(project_id=SQLTableArg(required=True))
+    sql_args = dict(
+        project_id=SQLTableArg(sql=project_where, required=True),
+        sample_event_ids=SQLTableArg(sql=sample_event_where, required=False),
+    )
 
     objects = SQLTableManager()
 
@@ -300,7 +312,10 @@ class BenthicPITSESQLModel(BaseSQLModel):
         percent_cover_by_benthic_category_avg
     """
 
-    sql_args = dict(project_id=SQLTableArg(required=True))
+    sql_args = dict(
+        project_id=SQLTableArg(sql=project_where, required=True),
+        sample_event_ids=SQLTableArg(sql=sample_event_where, required=False),
+    )
 
     objects = SQLTableManager()
 

diff --git a/src/api/models/sql_models/benthicpqt.py b/src/api/models/sql_models/benthicpqt.py
@@ -2,7 +2,13 @@
 from django.utils.translation import gettext_lazy as _
 
 from sqltables import SQLTableArg, SQLTableManager
-from .base import BaseSQLModel, BaseSUSQLModel, sample_event_sql_template
+from .base import (
+    BaseSQLModel,
+    BaseSUSQLModel,
+    project_where,
+    sample_event_sql_template,
+    sample_event_where
+)
 
 
 class BenthicPhotoQuadratTransectObsSQLModel(BaseSUSQLModel):
@@ -101,7 +107,10 @@ class BenthicPhotoQuadratTransectObsSQLModel(BaseSUSQLModel):
             LEFT JOIN api_reefslope rs ON su.reef_slope_id = rs.id
     """
 
-    sql_args = dict(project_id=SQLTableArg(required=True))
+    sql_args = dict(
+        project_id=SQLTableArg(sql=project_where, required=True),
+        sample_event_ids=SQLTableArg(sql=sample_event_where, required=False),
+    )
 
     objects = SQLTableManager()
 
@@ -233,7 +242,10 @@ class BenthicPhotoQuadratTransectSUSQLModel(BaseSUSQLModel):
         ON (benthicpqt_su.pseudosu_id = benthicpqt_observers.pseudosu_id)
     """
 
-    sql_args = dict(project_id=SQLTableArg(required=True))
+    sql_args = dict(
+        project_id=SQLTableArg(sql=project_where, required=True),
+        sample_event_ids=SQLTableArg(sql=sample_event_where, required=False),
+    )
 
     objects = SQLTableManager()
 
@@ -299,7 +311,10 @@ class BenthicPhotoQuadratTransectSESQLModel(BaseSQLModel):
         percent_cover_by_benthic_category_avg
     """
 
-    sql_args = dict(project_id=SQLTableArg(required=True))
+    sql_args = dict(
+        project_id=SQLTableArg(sql=project_where, required=True),
+        sample_event_ids=SQLTableArg(sql=sample_event_where, required=False),
+    )
 
     objects = SQLTableManager()