Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: create testruns model in timeseries app #508

Merged
merged 1 commit into from
Mar 4, 2025
Merged

feat: create testruns model in timeseries app #508

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 111 additions & 0 deletions shared/django_apps/timeseries/migrations/0015_testrun_model.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
# Generated by Django 4.2.16 on 2025-02-06 15:02

import django.contrib.postgres.fields
import django_prometheus.models
from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
(
"timeseries",
"0014_remove_measurement_timeseries_measurement_flag_unique_and_more",
),
]

operations = [
migrations.CreateModel(
name="Testrun",
fields=[
("timestamp", models.DateTimeField(primary_key=True, serialize=False)),
("repo_id", models.BigIntegerField()),
("test_id", models.BinaryField()),
("testsuite", models.TextField(null=True)),
("classname", models.TextField(null=True)),
("name", models.TextField(null=True)),
("computed_name", models.TextField(null=True)),
("outcome", models.TextField()),
("duration_seconds", models.FloatField(null=True)),
("failure_message", models.TextField(null=True)),
("framework", models.TextField(null=True)),
("filename", models.TextField(null=True)),
("commit_sha", models.TextField(null=True)),
("branch", models.TextField(null=True)),
(
"flags",
django.contrib.postgres.fields.ArrayField(
base_field=models.TextField(), null=True, size=None
),
),
("upload_id", models.BigIntegerField(null=True)),
],
bases=(
django_prometheus.models.ExportModelOperationsMixin(
"timeseries.testrun"
),
models.Model,
),
),
migrations.RunSQL(
"ALTER TABLE timeseries_testrun DROP CONSTRAINT timeseries_testrun_pkey;",
reverse_sql="",
),
migrations.RunSQL(
"SELECT create_hypertable('timeseries_testrun', 'timestamp');",
reverse_sql="",
),
migrations.AddIndex(
model_name="testrun",
index=models.Index(
fields=[
"repo_id",
"branch",
"timestamp",
],
name="ts__repo_branch_time_i",
),
),
migrations.AddIndex(
model_name="testrun",
index=models.Index(
fields=["repo_id", "branch", "test_id", "timestamp"],
name="ts__repo_branch_test_time_i",
),
),
migrations.AddIndex(
model_name="testrun",
index=models.Index(
fields=["repo_id", "test_id", "timestamp"],
name="ts__repo_test_time_i",
),
),
migrations.AddIndex(
model_name="testrun",
index=models.Index(
fields=["repo_id", "commit_sha", "timestamp"],
name="ts__repo_commit_time_i",
),
),
migrations.RunSQL(
"""
CREATE OR REPLACE FUNCTION array_merge_dedup(anyarray, anyarray)
RETURNS anyarray LANGUAGE sql IMMUTABLE AS $$
SELECT array_agg(DISTINCT x)
FROM (
SELECT unnest($1) as x
UNION
SELECT unnest($2)
) s;
$$;
CREATE AGGREGATE array_merge_dedup_agg(anyarray) (
SFUNC = array_merge_dedup,
STYPE = anyarray,
INITCOND = '{}'
);
""",
reverse_sql="""
DROP AGGREGATE array_merge_dedup_agg(anyarray);
DROP FUNCTION array_merge_dedup(anyarray, anyarray);
""",
),
]
71 changes: 71 additions & 0 deletions shared/django_apps/timeseries/migrations/0016_testrun_summary_1day.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
# Generated by Django 4.2.16 on 2025-02-06 16:56

from django.db import migrations


class Migration(migrations.Migration):
# cant create this views in a transaction
atomic = False

dependencies = [
("timeseries", "0015_testrun_model"),
]

operations = [
migrations.RunSQL(
"""
create materialized view timeseries_testrun_summary_1day
with (timescaledb.continuous) as
select
repo_id,
testsuite,
classname,
name,
time_bucket(interval '1 days', timestamp) as timestamp_bin,

min(computed_name) as computed_name,
COUNT(DISTINCT CASE WHEN outcome = 'failure' OR outcome = 'flaky_fail' THEN commit_sha ELSE NULL END) AS failing_commits,
last(duration_seconds, timestamp) as last_duration_seconds,
avg(duration_seconds) as avg_duration_seconds,
COUNT(*) FILTER (WHERE outcome = 'pass') AS pass_count,
COUNT(*) FILTER (WHERE outcome = 'failure') AS fail_count,
COUNT(*) FILTER (WHERE outcome = 'skip') AS skip_count,
COUNT(*) FILTER (WHERE outcome = 'flaky_fail') AS flaky_fail_count,
MAX(timestamp) AS updated_at,
array_merge_dedup_agg(flags) as flags
from timeseries_testrun
group by
repo_id, testsuite, classname, name, timestamp_bin;
""",
reverse_sql="drop materialized view timeseries_testrun_summary_1day;",
),
migrations.RunSQL(
"""
create materialized view timeseries_testrun_branch_summary_1day
with (timescaledb.continuous) as
select
repo_id,
branch,
testsuite,
classname,
name,
time_bucket(interval '1 days', timestamp) as timestamp_bin,

min(computed_name) as computed_name,
COUNT(DISTINCT CASE WHEN outcome = 'failure' OR outcome = 'flaky_fail' THEN commit_sha ELSE NULL END) AS failing_commits,
last(duration_seconds, timestamp) as last_duration_seconds,
avg(duration_seconds) as avg_duration_seconds,
COUNT(*) FILTER (WHERE outcome = 'pass') AS pass_count,
COUNT(*) FILTER (WHERE outcome = 'failure') AS fail_count,
COUNT(*) FILTER (WHERE outcome = 'skip') AS skip_count,
COUNT(*) FILTER (WHERE outcome = 'flaky_fail') AS flaky_fail_count,
MAX(timestamp) AS updated_at,
array_merge_dedup_agg(flags) as flags
from timeseries_testrun
where branch in ('main', 'master', 'develop')
group by
repo_id, branch, testsuite, classname, name, timestamp_bin;
""",
reverse_sql="drop materialized view timeseries_testrun_branch_summary_1day;",
),
]
34 changes: 34 additions & 0 deletions shared/django_apps/timeseries/migrations/0017_testrun_cagg_policy.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Generated by Django 4.2.16 on 2025-02-06 16:57

from django.db import migrations


class Migration(migrations.Migration):
dependencies = [
("timeseries", "0016_testrun_summary_1day"),
]

operations = [
migrations.RunSQL(
"""
select add_continuous_aggregate_policy(
'timeseries_testrun_summary_1day',
start_offset => '7 days',
end_offset => '1 days',
schedule_interval => INTERVAL '1 days'
);
""",
reverse_sql="select remove_continuous_aggregate_policy('timeseries_testrun_summary_1day');",
),
migrations.RunSQL(
"""
select add_continuous_aggregate_policy(
'timeseries_testrun_branch_summary_1day',
start_offset => '7 days',
end_offset => '1 days',
schedule_interval => INTERVAL '1 days'
);
""",
reverse_sql="select remove_continuous_aggregate_policy('timeseries_testrun_branch_summary_1day');",
),
]
90 changes: 90 additions & 0 deletions shared/django_apps/timeseries/migrations/0018_testrun_summary_model.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
# Generated by Django 4.2.16 on 2025-02-06 16:57
import django.contrib.postgres.fields
import django_prometheus.models
from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("timeseries", "0017_testrun_cagg_policy"),
]

operations = [
migrations.CreateModel(
name="TestrunSummary",
fields=[
(
"timestamp_bin",
models.DateTimeField(primary_key=True, serialize=False),
),
("repo_id", models.IntegerField()),
("name", models.TextField()),
("classname", models.TextField()),
("testsuite", models.TextField()),
("computed_name", models.TextField()),
("failing_commits", models.IntegerField()),
("avg_duration_seconds", models.FloatField()),
("last_duration_seconds", models.FloatField()),
("pass_count", models.IntegerField()),
("fail_count", models.IntegerField()),
("skip_count", models.IntegerField()),
("flaky_fail_count", models.IntegerField()),
("updated_at", models.DateTimeField()),
(
"flags",
django.contrib.postgres.fields.ArrayField(
base_field=models.TextField(), null=True, size=None
),
),
],
options={
"db_table": "timeseries_testrun_summary_1day",
"managed": False,
},
bases=(
django_prometheus.models.ExportModelOperationsMixin(
"timeseries.testrun_continuous_aggregate"
),
models.Model,
),
),
migrations.CreateModel(
name="TestrunBranchSummary",
fields=[
(
"timestamp_bin",
models.DateTimeField(primary_key=True, serialize=False),
),
("repo_id", models.IntegerField()),
("branch", models.TextField()),
("name", models.TextField()),
("classname", models.TextField()),
("testsuite", models.TextField()),
("computed_name", models.TextField()),
("failing_commits", models.IntegerField()),
("avg_duration_seconds", models.FloatField()),
("last_duration_seconds", models.FloatField()),
("pass_count", models.IntegerField()),
("fail_count", models.IntegerField()),
("skip_count", models.IntegerField()),
("flaky_fail_count", models.IntegerField()),
("updated_at", models.DateTimeField()),
(
"flags",
django.contrib.postgres.fields.ArrayField(
base_field=models.TextField(), null=True, size=None
),
),
],
options={
"db_table": "timeseries_testrun_summary_1day",
"managed": False,
},
bases=(
django_prometheus.models.ExportModelOperationsMixin(
"timeseries.testrun_continuous_aggregate"
),
models.Model,
),
),
]
35 changes: 35 additions & 0 deletions shared/django_apps/timeseries/migrations/0019_testrun_real_time_agg.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# Generated by Django 4.2.16 on 2025-02-12 15:58

from django.conf import settings
from django.db import migrations


class Migration(migrations.Migration):
dependencies = [
("timeseries", "0018_testrun_summary_model"),
]

operations = [
migrations.RunSQL(
"""
alter materialized view timeseries_testrun_summary_1day set (timescaledb.materialized_only = true);
"""
)
if not settings.TIMESERIES_REAL_TIME_AGGREGATES
else migrations.RunSQL(
"""
alter materialized view timeseries_testrun_summary_1day set (timescaledb.materialized_only = false);
"""
),
migrations.RunSQL(
"""
alter materialized view timeseries_testrun_branch_summary_1day set (timescaledb.materialized_only = true);
"""
)
if not settings.TIMESERIES_REAL_TIME_AGGREGATES
else migrations.RunSQL(
"""
alter materialized view timeseries_testrun_branch_summary_1day set (timescaledb.materialized_only = false);
"""
),
]
Loading
Oops, something went wrong.