From 5925d0ad2f812343d2f76165863853963c068652 Mon Sep 17 00:00:00 2001 From: Benjamin Knofe-Vider Date: Thu, 21 May 2026 13:25:16 +0200 Subject: [PATCH] chore(observe): drop dead pg_stat_activity scrape path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The postgres_query()-based pg_stat_activity scrape added in #598 silently no-ops in our DuckDB build and never populates the duckgres_ducklake_metadata_connections gauge. Remove it (and the gauge) rather than leave dead code that masks a real "no data" signal. Aurora-side connection / query activity is already attributable to duckgres via the application_name we tag at ATTACH time, so Performance Insights and pg_stat_activity are the better tools for that view — no worker-side scrape needed. Keeps the pg_pool_max_connections probe via duckdb_settings(), which works, and the per-query postgres_scan_seconds histogram and postgres_scan_ms query_log column from #598. --- duckdbservice/service.go | 48 ++++++--------------------------------- server/observe/metrics.go | 11 --------- 2 files changed, 7 insertions(+), 52 deletions(-) diff --git a/duckdbservice/service.go b/duckdbservice/service.go index fc502e3..ba26a1d 100644 --- a/duckdbservice/service.go +++ b/duckdbservice/service.go @@ -274,13 +274,18 @@ func (p *SessionPool) scrapeMetadataMetrics() { if v, ok := scrapeMetadataPoolMax(ctx, scrapeDB); ok { observe.MetadataPoolMaxConnections.WithLabelValues(orgID).Set(v) } - - scrapeMetadataConnectionsByState(ctx, scrapeDB, orgID) } // scrapeMetadataPoolMax reads DuckDB's pg_pool_max_connections setting via // duckdb_settings(). A value of 0 means the in-process pool is disabled (we // set this when ViaPgBouncer is true) — still useful to emit as a gauge. +// +// Aurora-side load (active/idle connections, slow queries) is observed via +// pg_stat_activity / Performance Insights, attributed back to duckgres by +// the application_name we tag at ATTACH time (ducklake.Config.ApplicationName). +// We deliberately don't try to scrape pg_stat_activity from the worker via +// postgres_query — an earlier attempt silently no-op'd in our DuckDB build +// and Performance Insights is the better tool for that view anyway. func scrapeMetadataPoolMax(ctx context.Context, db *sql.DB) (float64, bool) { var raw string err := db.QueryRowContext(ctx, @@ -297,45 +302,6 @@ func scrapeMetadataPoolMax(ctx context.Context, db *sql.DB) (float64, bool) { return v, true } -// scrapeMetadataConnectionsByState issues a single pg_stat_activity query -// against the ATTACHed DuckLake metadata DB via postgres_query() and emits a -// gauge per Postgres connection state. Connections are filtered by the -// application_name we injected at attach time so we report only duckgres' -// own connections, not whatever else is sharing the metadata Postgres. -// -// We deliberately do not log on failure: if postgres_query is unavailable -// (extension load order, version skew) or the metadata user lacks -// pg_stat_activity visibility, the gauge simply stops updating, and the -// per-org missing-data signal in Grafana is more useful than a recurring -// warn on every worker. -func scrapeMetadataConnectionsByState(ctx context.Context, db *sql.DB, orgID string) { - const q = `SELECT state, n FROM postgres_query( - '__ducklake_metadata_ducklake', - 'SELECT COALESCE(state, ''unknown'') AS state, count(*) AS n FROM pg_stat_activity WHERE application_name LIKE ''duckgres/%'' GROUP BY state' - )` - rows, err := db.QueryContext(ctx, q) - if err != nil { - return - } - defer func() { _ = rows.Close() }() - - seen := map[string]float64{} - for rows.Next() { - var state string - var n int64 - if err := rows.Scan(&state, &n); err != nil { - return - } - seen[state] = float64(n) - } - if err := rows.Err(); err != nil { - return - } - for state, n := range seen { - observe.MetadataConnectionsByState.WithLabelValues(orgID, state).Set(n) - } -} - // Run starts the DuckDB service, blocking until shutdown. func Run(cfg ServiceConfig) { svc := NewDuckDBService(cfg) diff --git a/server/observe/metrics.go b/server/observe/metrics.go index 1219467..40a16d9 100644 --- a/server/observe/metrics.go +++ b/server/observe/metrics.go @@ -63,14 +63,3 @@ var MetadataPoolMaxConnections = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "duckgres_ducklake_metadata_pool_max_connections", Help: "Configured postgres_scanner pool ceiling for DuckLake metadata (pg_pool_max_connections).", }, []string{"org"}) - -// MetadataConnectionsByState is the live count of metadata DB connections -// from pg_stat_activity on the DuckLake metadata DB, grouped by Postgres -// connection state (active, idle, idle in transaction, etc.). Scraped via -// postgres_query against the ATTACHed metadata catalog every metadata -// metrics tick. Only populated when the metadata DB tags duckgres -// connections via application_name (see ducklake.Config.ApplicationName). -var MetadataConnectionsByState = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "duckgres_ducklake_metadata_connections", - Help: "DuckLake metadata DB connection count by state, scraped from pg_stat_activity (filtered by application_name).", -}, []string{"org", "state"})