fix(sql): Remove retention policy from partman

internal/server/postgres/dataserverimpl.go

@@ -544,14 +544,14 @@ func (s *DataPlatformDataServiceServerImpl) StreamForecastData(
 					var row db.ListPredictionsForForecastsRow
 
 					err := rows.Scan(
-						&row.InitTimeUtc,
 						&row.ForecasterName,
 						&row.ForecasterVersion,
 						&row.CreatedAtUtc,
 						&row.HorizonMins,
 						&row.P50Sip,
 						&row.OtherStatsFractions,
 						&row.CapacityWatts,
+						&row.InitTimeUtc,
 						&row.Metadata,
 					)
 					if err != nil {

internal/server/postgres/sql/migrations/00007_partman_yearly_retention.sql

@@ -0,0 +1,110 @@
+-- +goose Up
+
+/*
+ * Removes the month-long retention policy on partman-managed partitions.
+ *
+ * This allows for querying of all historical data without needing custom queries. However, some
+ * partitions may have already been detached by the previous retention policy, so they must also
+ * be re-attached to the parent table. This is done by iterating through all existing partitions,
+ * extracting the date from their name, and attaching them with the appropriate range values.
+ */
+
+-- +goose StatementBegin
+DO $$
+DECLARE
+    partition_record RECORD;
+    start_time TIMESTAMP;
+    end_time TIMESTAMP;
+    attach_sql TEXT;
+BEGIN
+    UPDATE partman.part_config
+    SET retention = NULL
+    WHERE parent_table = 'obs.observed_generation_values';
+
+    FOR partition_record IN 
+        SELECT 
+            table_schema || '.' || table_name AS full_table_name,
+            SUBSTRING(table_name FROM 'p(\d{8})$') AS date_str
+        FROM information_schema.tables
+        WHERE table_schema = 'obs'
+          AND table_name LIKE 'observed_generation_values_p________'
+          AND NOT EXISTS (
+              SELECT 1 FROM pg_inherits
+              WHERE inhrelid = (table_schema || '.' || table_name)::regclass
+                AND inhparent = 'obs.observed_generation_values'::regclass
+          )
+    LOOP
+        start_time := to_timestamp(partition_record.date_str, 'YYYYMMDD');
+        end_time := start_time + INTERVAL '7 days';
+        attach_sql := format(
+	    'ALTER TABLE obs.observed_generation_values ATTACH PARTITION %s FOR VALUES FROM (%L::TIMESTAMP) TO (%L::TIMESTAMP);',
+            partition_record.full_table_name,
+            start_time,
+            end_time
+        );
+        RAISE NOTICE 'Executing: %', attach_sql;
+        EXECUTE attach_sql;
+    END LOOP;
+END $$;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+DO $$
+DECLARE
+    target_table TEXT;
+    parent_table_name TEXT;
+    partition_pattern TEXT;
+    partition_record RECORD;
+    start_time TIMESTAMPTZ;
+    end_time TIMESTAMPTZ;
+    attach_sql TEXT;
+BEGIN
+    FOREACH target_table IN ARRAY ARRAY['forecasts', 'predicted_generation_values']
+    LOOP
+	parent_table_name := 'pred.' || target_table;
+	partition_pattern := target_table || '_p________';
+
+        UPDATE partman.part_config
+    	SET retention = NULL
+   	WHERE parent_table = parent_table_name;
+
+	FOR partition_record IN 
+            SELECT 
+                table_schema || '.' || table_name AS full_table_name,
+                SUBSTRING(table_name FROM 'p(\d{8})$') AS date_str
+            FROM information_schema.tables
+            WHERE table_schema = 'pred'
+              AND table_name LIKE partition_pattern
+              AND NOT EXISTS (
+                  SELECT 1 FROM pg_inherits
+                  WHERE inhrelid = (table_schema || '.' || table_name)::regclass
+                    AND inhparent = parent_table_name::regclass
+              )
+        LOOP
+	    start_time := to_date(partition_record.date_str, 'YYYYMMDD')::TIMESTAMP AT TIME ZONE 'UTC';
+	    end_time := start_time + INTERVAL '7 days';
+
+            attach_sql := format(
+	    'ALTER TABLE %s ATTACH PARTITION %s FOR VALUES FROM (partman.uuid7_time_encoder(%L::TIMESTAMPTZ)) TO (partman.uuid7_time_encoder(%L::TIMESTAMPTZ));',
+                parent_table_name,
+                partition_record.full_table_name,
+                start_time,
+                end_time
+            );
+
+            RAISE NOTICE 'Executing: %', attach_sql;
+            EXECUTE attach_sql;
+        END LOOP;
+    END LOOP;
+END $$;
+-- +goose StatementEnd
+
+-- +goose Down
+
+UPDATE partman.part_config
+SET retention = '1 month'
+WHERE parent_table = 'obs.observed_generation_values';
+
+UPDATE partman.part_config
+SET retention = '1 month'
+WHERE parent_table IN ('pred.forecasts', 'pred.predicted_generation_values');

internal/server/postgres/sql/queries/iam.sql

@@ -35,8 +35,8 @@ WHERE org_name = LOWER(sqlc.arg(org_name)::TEXT);
 SELECT
     org_uuid,
     org_name,
-    UUIDV7_EXTRACT_TIMESTAMP(org_uuid)::TIMESTAMP AS created_at_utc,
-    metadata
+    metadata,
+    UUIDV7_EXTRACT_TIMESTAMP(org_uuid)::TIMESTAMP AS created_at_utc
 FROM iam.orgs
 ORDER BY org_name;
 
@@ -79,9 +79,9 @@ SELECT
     u.user_uuid,
     u.org_uuid,
     o.org_name,
-    UUIDV7_EXTRACT_TIMESTAMP(u.user_uuid)::TIMESTAMP AS created_at_utc,
     u.oauth_id,
-    u.metadata
+    u.metadata,
+    UUIDV7_EXTRACT_TIMESTAMP(u.user_uuid)::TIMESTAMP AS created_at_utc
 FROM iam.orgs AS o
     INNER JOIN iam.users AS u USING (org_uuid)
 WHERE u.oauth_id = $1;

internal/server/postgres/sql/queries/locations.sql

@@ -96,9 +96,9 @@ new_state AS (
     SELECT
         $1::UUID AS geometry_uuid,
         $2::SMALLINT AS source_type_id,
+        $3::TIMESTAMP AS valid_from_utc,
         sqlc.arg(capacity_watts)::BIGINT AS capacity_watts,
         sqlc.narg(capacity_limit_sip)::SMALLINT AS capacity_limit_sip,
-        $3::TIMESTAMP AS valid_from_utc,
         CASE WHEN sqlc.arg(metadata)::JSONB = '{}'::JSONB THEN NULL ELSE sqlc.arg(metadata)::JSONB END AS metadata
 )
 INSERT INTO loc.sources_history (

internal/server/postgres/sql/queries/observations.sql

@@ -95,11 +95,11 @@ target_observer AS (
 )
 SELECT
     tl.geometry_uuid::UUID AS geometry_uuid, -- SQLC complains without this
-    sqlc.arg(source_type_id)::SMALLINT AS source_type_id,
     latest_obs.observation_timestamp_utc,
     latest_obs.value_sip,
     sh.capacity_limit_sip,
-    sh.capacity_watts
+    sh.capacity_watts,
+    sqlc.arg(source_type_id)::SMALLINT AS source_type_id
 FROM target_locations AS tl
     CROSS JOIN target_observer AS tobs
     CROSS JOIN

internal/server/postgres/sql/queries/predictions.sql

@@ -149,14 +149,14 @@ matched_forecasters AS (
             AND f.forecaster_version = LOWER(rf.fversion)
 )
 SELECT
-    UUIDV7_EXTRACT_TIMESTAMP(f.forecast_uuid)::TIMESTAMP AS init_time_utc,
     mf.forecaster_name,
     mf.forecaster_version,
     f.created_at_utc,
     pg.horizon_mins,
     pg.p50_sip,
     pg.other_stats_fractions,
     sv.capacity_watts,
+    UUIDV7_EXTRACT_TIMESTAMP(f.forecast_uuid)::TIMESTAMP AS init_time_utc,
     COALESCE(pg.metadata || f.metadata, pg.metadata, f.metadata) AS metadata
 FROM pred.forecasts AS f
     INNER JOIN matched_forecasters AS mf USING (forecaster_id)
@@ -310,8 +310,8 @@ latest_allowed_forecast_per_location AS (
         tl.geometry_uuid::UUID AS geometry_uuid, -- again, SQLC complains without this
         lf.source_type_id,
         lf.created_at_utc,
-        UUIDV7_EXTRACT_TIMESTAMP(lf.forecast_uuid)::TIMESTAMP AS init_time_utc,
-        lf.metadata
+        lf.metadata,
+        UUIDV7_EXTRACT_TIMESTAMP(lf.forecast_uuid)::TIMESTAMP AS init_time_utc
     FROM target_locations AS tl
         CROSS JOIN LATERAL (
             SELECT