Skip to content

Commit

Permalink
feat: create testruns model in timeseries app
Browse files Browse the repository at this point in the history
  • Loading branch information
@joseph-sentry
joseph-sentry committed Feb 13, 2025
1 parent 3aea532 commit aadbae1
Showing 6 changed files with 439 additions and 0 deletions.
111 changes: 111 additions & 0 deletions shared/django_apps/timeseries/migrations/0015_testrun_model.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
# Generated by Django 4.2.16 on 2025-02-06 15:02

import django.contrib.postgres.fields
import django_prometheus.models
from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
(
"timeseries",
"0014_remove_measurement_timeseries_measurement_flag_unique_and_more",
),
]

operations = [
migrations.CreateModel(
name="Testrun",
fields=[
("timestamp", models.DateTimeField(primary_key=True, serialize=False)),
("repo_id", models.BigIntegerField()),
("test_id", models.BinaryField()),
("testsuite", models.TextField(null=True)),
("classname", models.TextField(null=True)),
("name", models.TextField(null=True)),
("computed_name", models.TextField(null=True)),
("outcome", models.TextField()),
("duration_seconds", models.FloatField(null=True)),
("failure_message", models.TextField(null=True)),
("framework", models.TextField(null=True)),
("filename", models.TextField(null=True)),
("commit_sha", models.TextField(null=True)),
("branch", models.TextField(null=True)),
(
"flags",
django.contrib.postgres.fields.ArrayField(
base_field=models.TextField(), null=True, size=None
),
),
("upload_id", models.BigIntegerField(null=True)),
],
bases=(
django_prometheus.models.ExportModelOperationsMixin(
"timeseries.testrun"
),
models.Model,
),
),
migrations.RunSQL(
"ALTER TABLE timeseries_testrun DROP CONSTRAINT timeseries_testrun_pkey;",
reverse_sql="",
),
migrations.RunSQL(
"SELECT create_hypertable('timeseries_testrun', 'timestamp');",
reverse_sql="",
),
migrations.AddIndex(
model_name="testrun",
index=models.Index(
fields=[
"repo_id",
"branch",
"timestamp",
],
name="ts__repo_branch_time_i",
),
),
migrations.AddIndex(
model_name="testrun",
index=models.Index(
fields=["repo_id", "branch", "test_id", "timestamp"],
name="ts__repo_branch_test_time_i",
),
),
migrations.AddIndex(
model_name="testrun",
index=models.Index(
fields=["repo_id", "test_id", "timestamp"],
name="ts__repo_test_time_i",
),
),
migrations.AddIndex(
model_name="testrun",
index=models.Index(
fields=["repo_id", "commit_sha", "timestamp"],
name="ts__repo_commit_time_i",
),
),
migrations.RunSQL(
"""
CREATE OR REPLACE FUNCTION array_merge_dedup(anyarray, anyarray)
RETURNS anyarray LANGUAGE sql IMMUTABLE AS $$
SELECT array_agg(DISTINCT x)
FROM (
SELECT unnest($1) as x
UNION
SELECT unnest($2)
) s;
$$;
CREATE AGGREGATE array_merge_dedup_agg(anyarray) (
SFUNC = array_merge_dedup,
STYPE = anyarray,
INITCOND = '{}'
);
""",
reverse_sql="""
DROP AGGREGATE array_merge_dedup_agg(anyarray);
DROP FUNCTION array_merge_dedup(anyarray, anyarray);
""",
),
]
71 changes: 71 additions & 0 deletions shared/django_apps/timeseries/migrations/0016_testrun_summary_1day.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
# Generated by Django 4.2.16 on 2025-02-06 16:56

from django.db import migrations


class Migration(migrations.Migration):
# cant create this views in a transaction
atomic = False

dependencies = [
("timeseries", "0015_testrun_model"),
]

operations = [
migrations.RunSQL(
"""
create materialized view timeseries_testrun_summary_1day
with (timescaledb.continuous) as
select
repo_id,
testsuite,
classname,
name,
time_bucket(interval '1 days', timestamp) as timestamp_bin,
min(computed_name) as computed_name,
COUNT(DISTINCT CASE WHEN outcome = 'failure' OR outcome = 'flaky_fail' THEN commit_sha ELSE NULL END) AS failing_commits,
last(duration_seconds, timestamp) as last_duration_seconds,
avg(duration_seconds) as avg_duration_seconds,
COUNT(*) FILTER (WHERE outcome = 'pass') AS pass_count,
COUNT(*) FILTER (WHERE outcome = 'failure') AS fail_count,
COUNT(*) FILTER (WHERE outcome = 'skip') AS skip_count,
COUNT(*) FILTER (WHERE outcome = 'flaky_fail') AS flaky_fail_count,
MAX(timestamp) AS updated_at,
array_merge_dedup_agg(flags) as flags
from timeseries_testrun
group by
repo_id, testsuite, classname, name, timestamp_bin;
""",
reverse_sql="drop materialized view timeseries_testrun_summary_1day;",
),
migrations.RunSQL(
"""
create materialized view timeseries_testrun_branch_summary_1day
with (timescaledb.continuous) as
select
repo_id,
branch,
testsuite,
classname,
name,
time_bucket(interval '1 days', timestamp) as timestamp_bin,
min(computed_name) as computed_name,
COUNT(DISTINCT CASE WHEN outcome = 'failure' OR outcome = 'flaky_fail' THEN commit_sha ELSE NULL END) AS failing_commits,
last(duration_seconds, timestamp) as last_duration_seconds,
avg(duration_seconds) as avg_duration_seconds,
COUNT(*) FILTER (WHERE outcome = 'pass') AS pass_count,
COUNT(*) FILTER (WHERE outcome = 'failure') AS fail_count,
COUNT(*) FILTER (WHERE outcome = 'skip') AS skip_count,
COUNT(*) FILTER (WHERE outcome = 'flaky_fail') AS flaky_fail_count,
MAX(timestamp) AS updated_at,
array_merge_dedup_agg(flags) as flags
from timeseries_testrun
where branch in ('main', 'master', 'develop')
group by
repo_id, branch, testsuite, classname, name, timestamp_bin;
""",
reverse_sql="drop materialized view timeseries_testrun_branch_summary_1day;",
),
]
34 changes: 34 additions & 0 deletions shared/django_apps/timeseries/migrations/0017_testrun_cagg_policy.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Generated by Django 4.2.16 on 2025-02-06 16:57

from django.db import migrations


class Migration(migrations.Migration):
dependencies = [
("timeseries", "0016_testrun_summary_1day"),
]

operations = [
migrations.RunSQL(
"""
select add_continuous_aggregate_policy(
'timeseries_testrun_summary_1day',
start_offset => '7 days',
end_offset => '1 days',
schedule_interval => INTERVAL '1 days'
);
""",
reverse_sql="select remove_continuous_aggregate_policy('timeseries_testrun_summary_1day');",
),
migrations.RunSQL(
"""
select add_continuous_aggregate_policy(
'timeseries_testrun_branch_summary_1day',
start_offset => '7 days',
end_offset => '1 days',
schedule_interval => INTERVAL '1 days'
);
""",
reverse_sql="select remove_continuous_aggregate_policy('timeseries_testrun_branch_summary_1day');",
),
]
90 changes: 90 additions & 0 deletions shared/django_apps/timeseries/migrations/0018_testrun_summary_model.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
# Generated by Django 4.2.16 on 2025-02-06 16:57
import django.contrib.postgres.fields
import django_prometheus.models
from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("timeseries", "0017_testrun_cagg_policy"),
]

operations = [
migrations.CreateModel(
name="TestrunSummary",
fields=[
(
"timestamp_bin",
models.DateTimeField(primary_key=True, serialize=False),
),
("repo_id", models.IntegerField()),
("name", models.TextField()),
("classname", models.TextField()),
("testsuite", models.TextField()),
("computed_name", models.TextField()),
("failing_commits", models.IntegerField()),
("avg_duration_seconds", models.FloatField()),
("last_duration_seconds", models.FloatField()),
("pass_count", models.IntegerField()),
("fail_count", models.IntegerField()),
("skip_count", models.IntegerField()),
("flaky_fail_count", models.IntegerField()),
("updated_at", models.DateTimeField()),
(
"flags",
django.contrib.postgres.fields.ArrayField(
base_field=models.TextField(), null=True, size=None
),
),
],
options={
"db_table": "timeseries_testrun_summary_1day",
"managed": False,
},
bases=(
django_prometheus.models.ExportModelOperationsMixin(
"timeseries.testrun_continuous_aggregate"
),
models.Model,
),
),
migrations.CreateModel(
name="TestrunBranchSummary",
fields=[
(
"timestamp_bin",
models.DateTimeField(primary_key=True, serialize=False),
),
("repo_id", models.IntegerField()),
("branch", models.TextField()),
("name", models.TextField()),
("classname", models.TextField()),
("testsuite", models.TextField()),
("computed_name", models.TextField()),
("failing_commits", models.IntegerField()),
("avg_duration_seconds", models.FloatField()),
("last_duration_seconds", models.FloatField()),
("pass_count", models.IntegerField()),
("fail_count", models.IntegerField()),
("skip_count", models.IntegerField()),
("flaky_fail_count", models.IntegerField()),
("updated_at", models.DateTimeField()),
(
"flags",
django.contrib.postgres.fields.ArrayField(
base_field=models.TextField(), null=True, size=None
),
),
],
options={
"db_table": "timeseries_testrun_summary_1day",
"managed": False,
},
bases=(
django_prometheus.models.ExportModelOperationsMixin(
"timeseries.testrun_continuous_aggregate"
),
models.Model,
),
),
]
35 changes: 35 additions & 0 deletions shared/django_apps/timeseries/migrations/0019_testrun_real_time_agg.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# Generated by Django 4.2.16 on 2025-02-12 15:58

from django.conf import settings
from django.db import migrations


class Migration(migrations.Migration):
dependencies = [
("timeseries", "0018_testrun_summary_model"),
]

operations = [
migrations.RunSQL(
"""
alter materialized view timeseries_testrun_summary_1day set (timescaledb.materialized_only = true);
"""
)
if not settings.TIMESERIES_REAL_TIME_AGGREGATES
else migrations.RunSQL(
"""
alter materialized view timeseries_testrun_summary_1day set (timescaledb.materialized_only = false);
"""
),
migrations.RunSQL(
"""
alter materialized view timeseries_testrun_branch_summary_1day set (timescaledb.materialized_only = true);
"""
)
if not settings.TIMESERIES_REAL_TIME_AGGREGATES
else migrations.RunSQL(
"""
alter materialized view timeseries_testrun_branch_summary_1day set (timescaledb.materialized_only = false);
"""
),
]
Loading

0 comments on commit aadbae1

Please sign in to comment.