codecov · joseph-sentry · Mar 4, 2025 · Jan 31, 2025
diff --git a/shared/django_apps/timeseries/migrations/0015_testrun_model.py b/shared/django_apps/timeseries/migrations/0015_testrun_model.py
@@ -0,0 +1,111 @@
+# Generated by Django 4.2.16 on 2025-02-06 15:02
+
+import django.contrib.postgres.fields
+import django_prometheus.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        (
+            "timeseries",
+            "0014_remove_measurement_timeseries_measurement_flag_unique_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Testrun",
+            fields=[
+                ("timestamp", models.DateTimeField(primary_key=True, serialize=False)),
+                ("repo_id", models.BigIntegerField()),
+                ("test_id", models.BinaryField()),
+                ("testsuite", models.TextField(null=True)),
+                ("classname", models.TextField(null=True)),
+                ("name", models.TextField(null=True)),
+                ("computed_name", models.TextField(null=True)),
+                ("outcome", models.TextField()),
+                ("duration_seconds", models.FloatField(null=True)),
+                ("failure_message", models.TextField(null=True)),
+                ("framework", models.TextField(null=True)),
+                ("filename", models.TextField(null=True)),
+                ("commit_sha", models.TextField(null=True)),
+                ("branch", models.TextField(null=True)),
+                (
+                    "flags",
+                    django.contrib.postgres.fields.ArrayField(
+                        base_field=models.TextField(), null=True, size=None
+                    ),
+                ),
+                ("upload_id", models.BigIntegerField(null=True)),
+            ],
+            bases=(
+                django_prometheus.models.ExportModelOperationsMixin(
+                    "timeseries.testrun"
+                ),
+                models.Model,
+            ),
+        ),
+        migrations.RunSQL(
+            "ALTER TABLE timeseries_testrun DROP CONSTRAINT timeseries_testrun_pkey;",
+            reverse_sql="",
+        ),
+        migrations.RunSQL(
+            "SELECT create_hypertable('timeseries_testrun', 'timestamp');",
+            reverse_sql="",
+        ),
+        migrations.AddIndex(
+            model_name="testrun",
+            index=models.Index(
+                fields=[
+                    "repo_id",
+                    "branch",
+                    "timestamp",
+                ],
+                name="ts__repo_branch_time_i",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="testrun",
+            index=models.Index(
+                fields=["repo_id", "branch", "test_id", "timestamp"],
+                name="ts__repo_branch_test_time_i",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="testrun",
+            index=models.Index(
+                fields=["repo_id", "test_id", "timestamp"],
+                name="ts__repo_test_time_i",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="testrun",
+            index=models.Index(
+                fields=["repo_id", "commit_sha", "timestamp"],
+                name="ts__repo_commit_time_i",
+            ),
+        ),
+        migrations.RunSQL(
+            """
+            CREATE OR REPLACE FUNCTION array_merge_dedup(anyarray, anyarray)
+            RETURNS anyarray LANGUAGE sql IMMUTABLE AS $$
+                SELECT array_agg(DISTINCT x)
+                FROM (
+                    SELECT unnest($1) as x
+                    UNION
+                    SELECT unnest($2)
+                ) s;
+            $$;
+            CREATE AGGREGATE array_merge_dedup_agg(anyarray) (
+                SFUNC = array_merge_dedup,
+                STYPE = anyarray,
+                INITCOND = '{}'
+            );
+            """,
+            reverse_sql="""
+            DROP AGGREGATE array_merge_dedup_agg(anyarray);
+            DROP FUNCTION array_merge_dedup(anyarray, anyarray);
+            """,
+        ),
+    ]
diff --git a/shared/django_apps/timeseries/migrations/0016_testrun_summary_1day.py b/shared/django_apps/timeseries/migrations/0016_testrun_summary_1day.py
@@ -0,0 +1,71 @@
+# Generated by Django 4.2.16 on 2025-02-06 16:56
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    # cant create this views in a transaction
+    atomic = False
+
+    dependencies = [
+        ("timeseries", "0015_testrun_model"),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            """
+                create materialized view timeseries_testrun_summary_1day
+                with (timescaledb.continuous) as
+                select
+                    repo_id,
+                    testsuite,
+                    classname,
+                    name,
+                    time_bucket(interval '1 days', timestamp) as timestamp_bin,
+
+                    min(computed_name) as computed_name,
+                    COUNT(DISTINCT CASE WHEN outcome = 'failure' OR outcome = 'flaky_fail' THEN commit_sha ELSE NULL END) AS failing_commits,
+                    last(duration_seconds, timestamp) as last_duration_seconds,
+                    avg(duration_seconds) as avg_duration_seconds,
+                    COUNT(*) FILTER (WHERE outcome = 'pass') AS pass_count,
+                    COUNT(*) FILTER (WHERE outcome = 'failure') AS fail_count,
+                    COUNT(*) FILTER (WHERE outcome = 'skip') AS skip_count,
+                    COUNT(*) FILTER (WHERE outcome = 'flaky_fail') AS flaky_fail_count,
+                    MAX(timestamp) AS updated_at,
+                    array_merge_dedup_agg(flags) as flags
+                from timeseries_testrun
+                group by
+                    repo_id, testsuite, classname, name, timestamp_bin;
+            """,
+            reverse_sql="drop materialized view timeseries_testrun_summary_1day;",
+        ),
+        migrations.RunSQL(
+            """
+                create materialized view timeseries_testrun_branch_summary_1day
+                with (timescaledb.continuous) as
+                select
+                    repo_id,
+                    branch,
+                    testsuite,
+                    classname,
+                    name,
+                    time_bucket(interval '1 days', timestamp) as timestamp_bin,
+
+                    min(computed_name) as computed_name,
+                    COUNT(DISTINCT CASE WHEN outcome = 'failure' OR outcome = 'flaky_fail' THEN commit_sha ELSE NULL END) AS failing_commits,
+                    last(duration_seconds, timestamp) as last_duration_seconds,
+                    avg(duration_seconds) as avg_duration_seconds,
+                    COUNT(*) FILTER (WHERE outcome = 'pass') AS pass_count,
+                    COUNT(*) FILTER (WHERE outcome = 'failure') AS fail_count,
+                    COUNT(*) FILTER (WHERE outcome = 'skip') AS skip_count,
+                    COUNT(*) FILTER (WHERE outcome = 'flaky_fail') AS flaky_fail_count,
+                    MAX(timestamp) AS updated_at,
+                    array_merge_dedup_agg(flags) as flags
+                from timeseries_testrun
+                where branch in ('main', 'master', 'develop')
+                group by
+                    repo_id, branch, testsuite, classname, name, timestamp_bin;
+            """,
+            reverse_sql="drop materialized view timeseries_testrun_branch_summary_1day;",
+        ),
+    ]
diff --git a/shared/django_apps/timeseries/migrations/0017_testrun_cagg_policy.py b/shared/django_apps/timeseries/migrations/0017_testrun_cagg_policy.py
@@ -0,0 +1,34 @@
+# Generated by Django 4.2.16 on 2025-02-06 16:57
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("timeseries", "0016_testrun_summary_1day"),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            """
+            select add_continuous_aggregate_policy(
+                'timeseries_testrun_summary_1day',
+                start_offset => '7 days',
+                end_offset => '1 days',
+                schedule_interval => INTERVAL '1 days'
+            );
+            """,
+            reverse_sql="select remove_continuous_aggregate_policy('timeseries_testrun_summary_1day');",
+        ),
+        migrations.RunSQL(
+            """
+            select add_continuous_aggregate_policy(
+                'timeseries_testrun_branch_summary_1day',
+                start_offset => '7 days',
+                end_offset => '1 days',
+                schedule_interval => INTERVAL '1 days'
+            );
+            """,
+            reverse_sql="select remove_continuous_aggregate_policy('timeseries_testrun_branch_summary_1day');",
+        ),
+    ]
diff --git a/shared/django_apps/timeseries/migrations/0018_testrun_summary_model.py b/shared/django_apps/timeseries/migrations/0018_testrun_summary_model.py
@@ -0,0 +1,90 @@
+# Generated by Django 4.2.16 on 2025-02-06 16:57
+import django.contrib.postgres.fields
+import django_prometheus.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("timeseries", "0017_testrun_cagg_policy"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="TestrunSummary",
+            fields=[
+                (
+                    "timestamp_bin",
+                    models.DateTimeField(primary_key=True, serialize=False),
+                ),
+                ("repo_id", models.IntegerField()),
+                ("name", models.TextField()),
+                ("classname", models.TextField()),
+                ("testsuite", models.TextField()),
+                ("computed_name", models.TextField()),
+                ("failing_commits", models.IntegerField()),
+                ("avg_duration_seconds", models.FloatField()),
+                ("last_duration_seconds", models.FloatField()),
+                ("pass_count", models.IntegerField()),
+                ("fail_count", models.IntegerField()),
+                ("skip_count", models.IntegerField()),
+                ("flaky_fail_count", models.IntegerField()),
+                ("updated_at", models.DateTimeField()),
+                (
+                    "flags",
+                    django.contrib.postgres.fields.ArrayField(
+                        base_field=models.TextField(), null=True, size=None
+                    ),
+                ),
+            ],
+            options={
+                "db_table": "timeseries_testrun_summary_1day",
+                "managed": False,
+            },
+            bases=(
+                django_prometheus.models.ExportModelOperationsMixin(
+                    "timeseries.testrun_continuous_aggregate"
+                ),
+                models.Model,
+            ),
+        ),
+        migrations.CreateModel(
+            name="TestrunBranchSummary",
+            fields=[
+                (
+                    "timestamp_bin",
+                    models.DateTimeField(primary_key=True, serialize=False),
+                ),
+                ("repo_id", models.IntegerField()),
+                ("branch", models.TextField()),
+                ("name", models.TextField()),
+                ("classname", models.TextField()),
+                ("testsuite", models.TextField()),
+                ("computed_name", models.TextField()),
+                ("failing_commits", models.IntegerField()),
+                ("avg_duration_seconds", models.FloatField()),
+                ("last_duration_seconds", models.FloatField()),
+                ("pass_count", models.IntegerField()),
+                ("fail_count", models.IntegerField()),
+                ("skip_count", models.IntegerField()),
+                ("flaky_fail_count", models.IntegerField()),
+                ("updated_at", models.DateTimeField()),
+                (
+                    "flags",
+                    django.contrib.postgres.fields.ArrayField(
+                        base_field=models.TextField(), null=True, size=None
+                    ),
+                ),
+            ],
+            options={
+                "db_table": "timeseries_testrun_summary_1day",
+                "managed": False,
+            },
+            bases=(
+                django_prometheus.models.ExportModelOperationsMixin(
+                    "timeseries.testrun_continuous_aggregate"
+                ),
+                models.Model,
+            ),
+        ),
+    ]
diff --git a/shared/django_apps/timeseries/migrations/0019_testrun_real_time_agg.py b/shared/django_apps/timeseries/migrations/0019_testrun_real_time_agg.py
@@ -0,0 +1,35 @@
+# Generated by Django 4.2.16 on 2025-02-12 15:58
+
+from django.conf import settings
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("timeseries", "0018_testrun_summary_model"),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            """
+                alter materialized view timeseries_testrun_summary_1day set (timescaledb.materialized_only = true);
+                """
+        )
+        if not settings.TIMESERIES_REAL_TIME_AGGREGATES
+        else migrations.RunSQL(
+            """
+                alter materialized view timeseries_testrun_summary_1day set (timescaledb.materialized_only = false);
+                """
+        ),
+        migrations.RunSQL(
+            """
+                alter materialized view timeseries_testrun_branch_summary_1day set (timescaledb.materialized_only = true);
+                """
+        )
+        if not settings.TIMESERIES_REAL_TIME_AGGREGATES
+        else migrations.RunSQL(
+            """
+                alter materialized view timeseries_testrun_branch_summary_1day set (timescaledb.materialized_only = false);
+                """
+        ),
+    ]