From 5d74f2f3e1114cc98433bbd94fc55d7311bcc93d Mon Sep 17 00:00:00 2001 From: devsjc <47188100+devsjc@users.noreply.github.com> Date: Mon, 18 May 2026 15:07:51 +0100 Subject: [PATCH 1/4] fix(sql): Remove retention policy from partman Partman previously had the retention policy set to one month, but we want to be able to query data that is older than that. This resets partman to have no retention policy, and reattaches and detached partitions from the affected tables. --- .../00007_partman_yearly_retention.sql | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql diff --git a/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql b/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql new file mode 100644 index 0000000..c39ea0f --- /dev/null +++ b/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql @@ -0,0 +1,98 @@ +-- +goose Up + +/* + * Removes the month-long retention policy on partman-managed partitions. + * + * This allows for querying of all historical data without needing custom queries. However, some + * partitions may have already been detached by the previous retention policy, so they must also + * be re-attached to the parent table. This is done by iterating through all existing partitions, + * extracting the date from their name, and attaching them with the appropriate range values. + */ + +DO $$ +DECLARE + partition_record RECORD; + start_time TIMESTAMP; + end_time TIMESTAMP; + attach_sql TEXT; +BEGIN + UPDATE partman.part_config + SET retention = NULL + WHERE parent_table = 'obs.observed_generation_values'; + + FOR partition_record IN + SELECT + table_schema || '.' || table_name AS full_table_name, + SUBSTRING(table_name FROM 'p(\d{8})$') AS date_str + FROM information_schema.tables + WHERE table_schema = 'obs' + AND table_name LIKE 'observed_generation_values_p________' + AND NOT EXISTS ( + SELECT 1 FROM pg_inherits + WHERE inhrelid = (table_schema || '.' || table_name)::regclass + AND inhparent = 'obs.observed_generation_values'::regclass + ) + LOOP + start_time := to_timestamp(partition_record.date_str, 'YYYYMMDD'); + end_time := start_time + INTERVAL '1 week'; + attach_sql := format( + 'ALTER TABLE obs.observed_generation_values ATTACH PARTITION %s FOR VALUES FROM (%L) TO (%L);', + partition_record.full_table_name, + start_time, + end_time + ); + RAISE NOTICE 'Executing: %', attach_sql; + EXECUTE attach_sql; + END LOOP; +END $$; + +DO $$ +DECLARE + target_table TEXT; + parent_table_name TEXT; + partition_pattern TEXT; + partition_record RECORD; + start_time TIMESTAMP; + end_time TIMESTAMP; + attach_sql TEXT; +BEGIN + FOREACH target_table IN ARRAY ARRAY['forecasts', 'predicted_generation_values'] + LOOP + parent_table_name := 'pred.' || target_table; + partition_pattern := target_table || '_p________'; + + UPDATE partman.part_config + SET retention = NULL + WHERE parent_table = parent_table_name; + + FOR partition_record IN + SELECT + table_schema || '.' || table_name AS full_table_name, + SUBSTRING(table_name FROM 'p(\d{8})$') AS date_str + FROM information_schema.tables + WHERE table_schema = 'pred' + AND table_name LIKE partition_pattern + AND NOT EXISTS ( + SELECT 1 FROM pg_inherits + WHERE inhrelid = (table_schema || '.' || table_name)::regclass + AND inhparent = parent_table_name::regclass + ) + LOOP + start_time := to_timestamp(partition_record.date_str, 'YYYYMMDD'); + end_time := start_time + INTERVAL '1 week'; + + attach_sql := format( + 'ALTER TABLE %s ATTACH PARTITION %s FOR VALUES FROM (UUIDV7_BOUNDARY(%L::TIMESTAMP)) TO (UUIDV7_BOUNDARY(%L::TIMESTAMP));', + parent_table_name, + partition_record.full_table_name, + start_time, + end_time + ); + + RAISE NOTICE 'Executing: %', attach_sql; + EXECUTE attach_sql; + END LOOP; + END LOOP; +END $$; + +-- From df3a4ad01ee7fc91af58a2bfbbfc6a469efecca2 Mon Sep 17 00:00:00 2001 From: devsjc <47188100+devsjc@users.noreply.github.com> Date: Mon, 18 May 2026 15:11:15 +0100 Subject: [PATCH 2/4] chore(repo): Linting --- .../sql/migrations/00007_partman_yearly_retention.sql | 10 +++++++++- internal/server/postgres/sql/queries/iam.sql | 8 ++++---- internal/server/postgres/sql/queries/locations.sql | 2 +- internal/server/postgres/sql/queries/observations.sql | 4 ++-- internal/server/postgres/sql/queries/predictions.sql | 6 +++--- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql b/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql index c39ea0f..989b452 100644 --- a/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql +++ b/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql @@ -95,4 +95,12 @@ BEGIN END LOOP; END $$; --- +-- +goose Down + +UPDATE partman.part_config +SET retention = '1 month' +WHERE parent_table = 'obs.observed_generation_values'; + +UPDATE partman.part_config +SET retention = '1 month' +WHERE parent_table IN ('pred.forecasts', 'pred.predicted_generation_values'); diff --git a/internal/server/postgres/sql/queries/iam.sql b/internal/server/postgres/sql/queries/iam.sql index 224126c..3d56a1f 100644 --- a/internal/server/postgres/sql/queries/iam.sql +++ b/internal/server/postgres/sql/queries/iam.sql @@ -35,8 +35,8 @@ WHERE org_name = LOWER(sqlc.arg(org_name)::TEXT); SELECT org_uuid, org_name, - UUIDV7_EXTRACT_TIMESTAMP(org_uuid)::TIMESTAMP AS created_at_utc, - metadata + metadata, + UUIDV7_EXTRACT_TIMESTAMP(org_uuid)::TIMESTAMP AS created_at_utc FROM iam.orgs ORDER BY org_name; @@ -79,9 +79,9 @@ SELECT u.user_uuid, u.org_uuid, o.org_name, - UUIDV7_EXTRACT_TIMESTAMP(u.user_uuid)::TIMESTAMP AS created_at_utc, u.oauth_id, - u.metadata + u.metadata, + UUIDV7_EXTRACT_TIMESTAMP(u.user_uuid)::TIMESTAMP AS created_at_utc FROM iam.orgs AS o INNER JOIN iam.users AS u USING (org_uuid) WHERE u.oauth_id = $1; diff --git a/internal/server/postgres/sql/queries/locations.sql b/internal/server/postgres/sql/queries/locations.sql index 22c26ef..b93397e 100644 --- a/internal/server/postgres/sql/queries/locations.sql +++ b/internal/server/postgres/sql/queries/locations.sql @@ -96,9 +96,9 @@ new_state AS ( SELECT $1::UUID AS geometry_uuid, $2::SMALLINT AS source_type_id, + $3::TIMESTAMP AS valid_from_utc, sqlc.arg(capacity_watts)::BIGINT AS capacity_watts, sqlc.narg(capacity_limit_sip)::SMALLINT AS capacity_limit_sip, - $3::TIMESTAMP AS valid_from_utc, CASE WHEN sqlc.arg(metadata)::JSONB = '{}'::JSONB THEN NULL ELSE sqlc.arg(metadata)::JSONB END AS metadata ) INSERT INTO loc.sources_history ( diff --git a/internal/server/postgres/sql/queries/observations.sql b/internal/server/postgres/sql/queries/observations.sql index 93a6a0e..0dc816f 100644 --- a/internal/server/postgres/sql/queries/observations.sql +++ b/internal/server/postgres/sql/queries/observations.sql @@ -95,11 +95,11 @@ target_observer AS ( ) SELECT tl.geometry_uuid::UUID AS geometry_uuid, -- SQLC complains without this - sqlc.arg(source_type_id)::SMALLINT AS source_type_id, latest_obs.observation_timestamp_utc, latest_obs.value_sip, sh.capacity_limit_sip, - sh.capacity_watts + sh.capacity_watts, + sqlc.arg(source_type_id)::SMALLINT AS source_type_id FROM target_locations AS tl CROSS JOIN target_observer AS tobs CROSS JOIN diff --git a/internal/server/postgres/sql/queries/predictions.sql b/internal/server/postgres/sql/queries/predictions.sql index 35de0bc..bf51d47 100644 --- a/internal/server/postgres/sql/queries/predictions.sql +++ b/internal/server/postgres/sql/queries/predictions.sql @@ -149,7 +149,6 @@ matched_forecasters AS ( AND f.forecaster_version = LOWER(rf.fversion) ) SELECT - UUIDV7_EXTRACT_TIMESTAMP(f.forecast_uuid)::TIMESTAMP AS init_time_utc, mf.forecaster_name, mf.forecaster_version, f.created_at_utc, @@ -157,6 +156,7 @@ SELECT pg.p50_sip, pg.other_stats_fractions, sv.capacity_watts, + UUIDV7_EXTRACT_TIMESTAMP(f.forecast_uuid)::TIMESTAMP AS init_time_utc, COALESCE(pg.metadata || f.metadata, pg.metadata, f.metadata) AS metadata FROM pred.forecasts AS f INNER JOIN matched_forecasters AS mf USING (forecaster_id) @@ -310,8 +310,8 @@ latest_allowed_forecast_per_location AS ( tl.geometry_uuid::UUID AS geometry_uuid, -- again, SQLC complains without this lf.source_type_id, lf.created_at_utc, - UUIDV7_EXTRACT_TIMESTAMP(lf.forecast_uuid)::TIMESTAMP AS init_time_utc, - lf.metadata + lf.metadata, + UUIDV7_EXTRACT_TIMESTAMP(lf.forecast_uuid)::TIMESTAMP AS init_time_utc FROM target_locations AS tl CROSS JOIN LATERAL ( SELECT From 491648996a9f9b0a40ac8f0aad3683bcc2d8af69 Mon Sep 17 00:00:00 2001 From: devsjc <47188100+devsjc@users.noreply.github.com> Date: Mon, 18 May 2026 15:21:03 +0100 Subject: [PATCH 3/4] fix(sql): Wrap statements --- .../sql/migrations/00007_partman_yearly_retention.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql b/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql index 989b452..9d4687d 100644 --- a/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql +++ b/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql @@ -9,6 +9,7 @@ * extracting the date from their name, and attaching them with the appropriate range values. */ +-- +goose StatementBegin DO $$ DECLARE partition_record RECORD; @@ -45,7 +46,9 @@ BEGIN EXECUTE attach_sql; END LOOP; END $$; +-- +goose StatementEnd +-- +goose StatementBegin DO $$ DECLARE target_table TEXT; @@ -94,8 +97,9 @@ BEGIN END LOOP; END LOOP; END $$; +-- +goose StatementEnd --- +goose Down +-- +goose Down UPDATE partman.part_config SET retention = '1 month' From a3c05b58b10df98145bcefa4822df2d1166672b9 Mon Sep 17 00:00:00 2001 From: devsjc <47188100+devsjc@users.noreply.github.com> Date: Mon, 18 May 2026 18:23:27 +0100 Subject: [PATCH 4/4] fix(impl): Correct row scan order --- internal/server/postgres/dataserverimpl.go | 2 +- .../migrations/00007_partman_yearly_retention.sql | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/server/postgres/dataserverimpl.go b/internal/server/postgres/dataserverimpl.go index a4982ae..f9db644 100644 --- a/internal/server/postgres/dataserverimpl.go +++ b/internal/server/postgres/dataserverimpl.go @@ -544,7 +544,6 @@ func (s *DataPlatformDataServiceServerImpl) StreamForecastData( var row db.ListPredictionsForForecastsRow err := rows.Scan( - &row.InitTimeUtc, &row.ForecasterName, &row.ForecasterVersion, &row.CreatedAtUtc, @@ -552,6 +551,7 @@ func (s *DataPlatformDataServiceServerImpl) StreamForecastData( &row.P50Sip, &row.OtherStatsFractions, &row.CapacityWatts, + &row.InitTimeUtc, &row.Metadata, ) if err != nil { diff --git a/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql b/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql index 9d4687d..0f708ab 100644 --- a/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql +++ b/internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql @@ -35,9 +35,9 @@ BEGIN ) LOOP start_time := to_timestamp(partition_record.date_str, 'YYYYMMDD'); - end_time := start_time + INTERVAL '1 week'; + end_time := start_time + INTERVAL '7 days'; attach_sql := format( - 'ALTER TABLE obs.observed_generation_values ATTACH PARTITION %s FOR VALUES FROM (%L) TO (%L);', + 'ALTER TABLE obs.observed_generation_values ATTACH PARTITION %s FOR VALUES FROM (%L::TIMESTAMP) TO (%L::TIMESTAMP);', partition_record.full_table_name, start_time, end_time @@ -55,8 +55,8 @@ DECLARE parent_table_name TEXT; partition_pattern TEXT; partition_record RECORD; - start_time TIMESTAMP; - end_time TIMESTAMP; + start_time TIMESTAMPTZ; + end_time TIMESTAMPTZ; attach_sql TEXT; BEGIN FOREACH target_table IN ARRAY ARRAY['forecasts', 'predicted_generation_values'] @@ -81,11 +81,11 @@ BEGIN AND inhparent = parent_table_name::regclass ) LOOP - start_time := to_timestamp(partition_record.date_str, 'YYYYMMDD'); - end_time := start_time + INTERVAL '1 week'; + start_time := to_date(partition_record.date_str, 'YYYYMMDD')::TIMESTAMP AT TIME ZONE 'UTC'; + end_time := start_time + INTERVAL '7 days'; attach_sql := format( - 'ALTER TABLE %s ATTACH PARTITION %s FOR VALUES FROM (UUIDV7_BOUNDARY(%L::TIMESTAMP)) TO (UUIDV7_BOUNDARY(%L::TIMESTAMP));', + 'ALTER TABLE %s ATTACH PARTITION %s FOR VALUES FROM (partman.uuid7_time_encoder(%L::TIMESTAMPTZ)) TO (partman.uuid7_time_encoder(%L::TIMESTAMPTZ));', parent_table_name, partition_record.full_table_name, start_time,