From 6ce193c8c2836f8be631fd29ebb88a493992987f Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Tue, 11 Nov 2025 14:02:03 +0100 Subject: [PATCH 1/6] Added more pgstac config options. --- CHANGELOG.md | 6 +++ .../initdb-data/settings/pgstac-settings.sql | 12 ----- .../settings/pgstac-settings.sql.tpl | 26 +++++++++++ .../templates/pgstacbootstrap/configmap.yaml | 2 +- charts/eoapi/values.yaml | 16 +++++++ docs/configuration.md | 44 +++++++++++++++++++ 6 files changed, 93 insertions(+), 13 deletions(-) delete mode 100644 charts/eoapi/initdb-data/settings/pgstac-settings.sql create mode 100644 charts/eoapi/initdb-data/settings/pgstac-settings.sql.tpl diff --git a/CHANGELOG.md b/CHANGELOG.md index f37b059d..cf8e4d9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Expose PgSTAC configuration options in Helm chart values (`pgstacBootstrap.settings.pgstacSettings`). These are being dynamically applied via templated SQL during bootstrap. + - Added `queue_timeout`, `use_queue`, and `update_collection_extent` settings for performance tuning + - Added context settings (`context`, `context_estimated_count`, `context_estimated_cost`, `context_stats_ttl`) for search result optimization + ## [0.7.13] - 2025-11-04 ### Added diff --git a/charts/eoapi/initdb-data/settings/pgstac-settings.sql b/charts/eoapi/initdb-data/settings/pgstac-settings.sql deleted file mode 100644 index 2827508a..00000000 --- a/charts/eoapi/initdb-data/settings/pgstac-settings.sql +++ /dev/null @@ -1,12 +0,0 @@ --- Apply pgstac settings -DELETE FROM pgstac.pgstac_settings WHERE name = 'context'; -INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context', 'auto'); - -DELETE FROM pgstac.pgstac_settings WHERE name = 'context_estimated_count'; -INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_estimated_count', '100000'); - -DELETE FROM pgstac.pgstac_settings WHERE name = 'context_estimated_cost'; -INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_estimated_cost', '100000'); - -DELETE FROM pgstac.pgstac_settings WHERE name = 'context_stats_ttl'; -INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_stats_ttl', '1 day'); diff --git a/charts/eoapi/initdb-data/settings/pgstac-settings.sql.tpl b/charts/eoapi/initdb-data/settings/pgstac-settings.sql.tpl new file mode 100644 index 00000000..52eb1d9e --- /dev/null +++ b/charts/eoapi/initdb-data/settings/pgstac-settings.sql.tpl @@ -0,0 +1,26 @@ +-- Apply pgstac settings +-- These settings are configured via Helm values at pgstacBootstrap.settings.pgstacSettings + +-- Queue settings +DELETE FROM pgstac.pgstac_settings WHERE name = 'queue_timeout'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('queue_timeout', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.queue_timeout }}'); + +DELETE FROM pgstac.pgstac_settings WHERE name = 'use_queue'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('use_queue', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.use_queue }}'); + +-- Collection extent management +DELETE FROM pgstac.pgstac_settings WHERE name = 'update_collection_extent'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('update_collection_extent', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.update_collection_extent }}'); + +-- Context settings +DELETE FROM pgstac.pgstac_settings WHERE name = 'context'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.context }}'); + +DELETE FROM pgstac.pgstac_settings WHERE name = 'context_estimated_count'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_estimated_count', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.context_estimated_count }}'); + +DELETE FROM pgstac.pgstac_settings WHERE name = 'context_estimated_cost'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_estimated_cost', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.context_estimated_cost }}'); + +DELETE FROM pgstac.pgstac_settings WHERE name = 'context_stats_ttl'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_stats_ttl', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.context_stats_ttl }}'); diff --git a/charts/eoapi/templates/pgstacbootstrap/configmap.yaml b/charts/eoapi/templates/pgstacbootstrap/configmap.yaml index b00c0d47..ce5b77d0 100644 --- a/charts/eoapi/templates/pgstacbootstrap/configmap.yaml +++ b/charts/eoapi/templates/pgstacbootstrap/configmap.yaml @@ -13,7 +13,7 @@ metadata: helm.sh/hook-delete-policy: "before-hook-creation,hook-succeeded" data: pgstac-settings.sql: | - {{ $.Files.Get "initdb-data/settings/pgstac-settings.sql" | nindent 4 }} + {{- tpl ($.Files.Get "initdb-data/settings/pgstac-settings.sql.tpl") $ | nindent 4 }} {{- if (index .Values "eoapi-notifier").enabled }} {{ $.Files.Get "initdb-data/settings/pgstac-notification-triggers.sql" | nindent 4 }} {{- end }} diff --git a/charts/eoapi/values.yaml b/charts/eoapi/values.yaml index 2fc60ae0..9cacbe5f 100644 --- a/charts/eoapi/values.yaml +++ b/charts/eoapi/values.yaml @@ -177,6 +177,22 @@ pgstacBootstrap: # deleteMissing: true queryables: [] + # PgSTAC settings configuration + # These settings control key PgSTAC behaviors and performance characteristics + pgstacSettings: + # Queue configuration for handling complex queries + queue_timeout: "10 minutes" # Timeout for queued queries (PostgreSQL interval format) + use_queue: "true" # Enable/disable query queue mechanism + + # Collection extent management + update_collection_extent: "false" # Auto-update collection extents when items are added/modified + + # Context configuration for search results + context: "auto" # Context mode: "on", "off", or "auto" + context_estimated_count: "100000" # Row threshold for using estimates instead of exact counts + context_estimated_cost: "100000" # Query cost threshold for using estimates + context_stats_ttl: "1 day" # Cache duration for context statistics + # Wait configuration for init containers waiting for pgstac jobs # These parameters control how long services wait for pgstac migration jobs to complete waitConfig: diff --git a/docs/configuration.md b/docs/configuration.md index 9a036433..e0a7b511 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -48,6 +48,7 @@ Using Crunchydata's PostgreSQL Operator (`postgresql.type: "postgrescluster"`): For external databases, set `postgresql.type` to either: + 1. Using plaintext credentials (`external-plaintext`): ```yaml postgresql: @@ -76,6 +77,49 @@ postgresql: database: "eoapi" # can also be in secret ``` +## PgSTAC Configuration + +Control PgSTAC database behavior and performance tuning: + +### Core Settings + +Configure via `pgstacBootstrap.settings.pgstacSettings`: + +| **Values Key** | **Description** | **Default** | **Format** | +|:--------------|:----------------|:------------|:-----------| +| `queue_timeout` | Timeout for queued queries | "10 minutes" | PostgreSQL interval | +| `use_queue` | Enable query queue mechanism | "true" | boolean string | +| `update_collection_extent` | Auto-update collection extents | "false" | boolean string | + +### Context Settings + +Control search result count calculations: + +| **Values Key** | **Description** | **Default** | **Format** | +|:--------------|:----------------|:------------|:-----------| +| `context` | Context mode | "auto" | "on", "off", "auto" | +| `context_estimated_count` | Row threshold for estimates | "100000" | integer string | +| `context_estimated_cost` | Cost threshold for estimates | "100000" | integer string | +| `context_stats_ttl` | Stats cache duration | "1 day" | PostgreSQL interval | + +Example configuration: + +```yaml +pgstacBootstrap: + settings: + pgstacSettings: + # Performance tuning for large datasets + queue_timeout: "20 minutes" + use_queue: "true" + update_collection_extent: "false" + + # Optimize context for performance + context: "auto" + context_estimated_count: "50000" + context_estimated_cost: "75000" + context_stats_ttl: "12 hours" +``` + ## Ingress Configuration Unified ingress configuration supporting both NGINX and Traefik: From 2b9a948c828cc30f42864a15c36cf8fd29d49bf4 Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Tue, 11 Nov 2025 16:34:55 +0100 Subject: [PATCH 2/6] Added queueprocessor cronjob. --- CHANGELOG.md | 5 +- .../pgstacbootstrap/queue-processor.yaml | 46 +++++++++++++++++++ charts/eoapi/values.yaml | 5 ++ docs/configuration.md | 11 +++++ 4 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 charts/eoapi/templates/pgstacbootstrap/queue-processor.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index cf8e4d9c..7181ea46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Expose PgSTAC configuration options in Helm chart values (`pgstacBootstrap.settings.pgstacSettings`). These are being dynamically applied via templated SQL during bootstrap. - - Added `queue_timeout`, `use_queue`, and `update_collection_extent` settings for performance tuning - - Added context settings (`context`, `context_estimated_count`, `context_estimated_cost`, `context_stats_ttl`) for search result optimization + - Added `queue_timeout`, `use_queue`, and `update_collection_extent` settings for database performance tuning + - Made existing context settings configurable (`context`, `context_estimated_count`, `context_estimated_cost`, `context_stats_ttl`) + - Automatic queue processor CronJob created when `use_queue` is "true" (configurable schedule via `queueProcessor.schedule`) ## [0.7.13] - 2025-11-04 diff --git a/charts/eoapi/templates/pgstacbootstrap/queue-processor.yaml b/charts/eoapi/templates/pgstacbootstrap/queue-processor.yaml new file mode 100644 index 00000000..a3becdee --- /dev/null +++ b/charts/eoapi/templates/pgstacbootstrap/queue-processor.yaml @@ -0,0 +1,46 @@ +{{- if .Values.pgstacBootstrap.enabled }} +{{- if .Values.pgstacBootstrap.settings.pgstacSettings }} +{{- if ne (default "true" .Values.pgstacBootstrap.settings.pgstacSettings.use_queue | toString) "false" }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-pgstac-queue-processor + labels: + {{- include "eoapi.labels" . | nindent 4 }} + app.kubernetes.io/component: pgstac-queue +spec: + schedule: {{ .Values.pgstacBootstrap.settings.queueProcessor.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + metadata: + labels: + {{- include "eoapi.labels" . | nindent 12 }} + app.kubernetes.io/component: pgstac-queue + spec: + restartPolicy: OnFailure + containers: + - name: queue-processor + image: {{ .Values.pgstacBootstrap.image.name }}:{{ .Values.pgstacBootstrap.image.tag }} + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-c" + - | + psql -c "{{ .Values.pgstacBootstrap.settings.queueProcessor.command }}" + env: + {{- include "eoapi.postgresqlEnv" . | nindent 14 }} + resources: + limits: + cpu: "256m" + memory: "512Mi" + requests: + cpu: "128m" + memory: "256Mi" +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/eoapi/values.yaml b/charts/eoapi/values.yaml index 9cacbe5f..d61f02e8 100644 --- a/charts/eoapi/values.yaml +++ b/charts/eoapi/values.yaml @@ -193,6 +193,11 @@ pgstacBootstrap: context_estimated_cost: "100000" # Query cost threshold for using estimates context_stats_ttl: "1 day" # Cache duration for context statistics + # Queue processing configuration (only used when use_queue is "true") + queueProcessor: + schedule: "0 * * * *" # Run every hour (cron format) + command: "SELECT run_queued_queries();" + # Wait configuration for init containers waiting for pgstac jobs # These parameters control how long services wait for pgstac migration jobs to complete waitConfig: diff --git a/docs/configuration.md b/docs/configuration.md index e0a7b511..f14fc078 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -102,6 +102,17 @@ Control search result count calculations: | `context_estimated_cost` | Cost threshold for estimates | "100000" | integer string | | `context_stats_ttl` | Stats cache duration | "1 day" | PostgreSQL interval | +### Queue Processing + +When `use_queue` is set to "true", a CronJob is automatically created to process queued queries: + +| **Values Key** | **Description** | **Default** | **Format** | +|:--------------|:----------------|:------------|:-----------| +| `queueProcessor.schedule` | Cron schedule for processing | "0 * * * *" | Cron format | +| `queueProcessor.command` | SQL command to run | "SELECT run_queued_queries();" | SQL | + +The queue processor runs hourly by default but can be customized for your workload. + Example configuration: ```yaml From 1bee190b61d68c06fb84a0a13a4ab55e3487bb46 Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Tue, 11 Nov 2025 16:39:07 +0100 Subject: [PATCH 3/6] Added extendUpdater cronjob. --- CHANGELOG.md | 1 + .../pgstacbootstrap/extent-updater.yaml | 46 +++++++++++++++++++ .../pgstacbootstrap/queue-processor.yaml | 2 +- charts/eoapi/values.yaml | 5 ++ docs/configuration.md | 17 ++++--- 5 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 charts/eoapi/templates/pgstacbootstrap/extent-updater.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 7181ea46..68131123 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `queue_timeout`, `use_queue`, and `update_collection_extent` settings for database performance tuning - Made existing context settings configurable (`context`, `context_estimated_count`, `context_estimated_cost`, `context_stats_ttl`) - Automatic queue processor CronJob created when `use_queue` is "true" (configurable schedule via `queueProcessor.schedule`) + - Automatic extent updater CronJob created when `update_collection_extent` is "false" (configurable schedule via `extentUpdater.schedule`) ## [0.7.13] - 2025-11-04 diff --git a/charts/eoapi/templates/pgstacbootstrap/extent-updater.yaml b/charts/eoapi/templates/pgstacbootstrap/extent-updater.yaml new file mode 100644 index 00000000..7ef410b6 --- /dev/null +++ b/charts/eoapi/templates/pgstacbootstrap/extent-updater.yaml @@ -0,0 +1,46 @@ +{{- if .Values.pgstacBootstrap.enabled }} +{{- if .Values.pgstacBootstrap.settings.pgstacSettings }} +{{- if eq (default "false" .Values.pgstacBootstrap.settings.pgstacSettings.update_collection_extent) "false" }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-pgstac-extent-updater + labels: + {{- include "eoapi.labels" . | nindent 4 }} + app.kubernetes.io/component: pgstac-extents +spec: + schedule: {{ .Values.pgstacBootstrap.settings.extentUpdater.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + metadata: + labels: + {{- include "eoapi.labels" . | nindent 12 }} + app.kubernetes.io/component: pgstac-extents + spec: + restartPolicy: OnFailure + containers: + - name: extent-updater + image: {{ .Values.pgstacBootstrap.image.name }}:{{ .Values.pgstacBootstrap.image.tag }} + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-c" + - | + psql -c "{{ .Values.pgstacBootstrap.settings.extentUpdater.command }}" + env: + {{- include "eoapi.postgresqlEnv" . | nindent 14 }} + resources: + limits: + cpu: "512m" + memory: "1024Mi" + requests: + cpu: "256m" + memory: "512Mi" +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/eoapi/templates/pgstacbootstrap/queue-processor.yaml b/charts/eoapi/templates/pgstacbootstrap/queue-processor.yaml index a3becdee..4de8a8d0 100644 --- a/charts/eoapi/templates/pgstacbootstrap/queue-processor.yaml +++ b/charts/eoapi/templates/pgstacbootstrap/queue-processor.yaml @@ -1,6 +1,6 @@ {{- if .Values.pgstacBootstrap.enabled }} {{- if .Values.pgstacBootstrap.settings.pgstacSettings }} -{{- if ne (default "true" .Values.pgstacBootstrap.settings.pgstacSettings.use_queue | toString) "false" }} +{{- if eq (.Values.pgstacBootstrap.settings.pgstacSettings.use_queue | default "true") "true" }} --- apiVersion: batch/v1 kind: CronJob diff --git a/charts/eoapi/values.yaml b/charts/eoapi/values.yaml index d61f02e8..36d2f5a3 100644 --- a/charts/eoapi/values.yaml +++ b/charts/eoapi/values.yaml @@ -198,6 +198,11 @@ pgstacBootstrap: schedule: "0 * * * *" # Run every hour (cron format) command: "SELECT run_queued_queries();" + # Extent updater configuration (only used when update_collection_extent is "false") + extentUpdater: + schedule: "0 2 * * *" # Run daily at 2 AM (cron format) + command: "SELECT update_collection_extents();" + # Wait configuration for init containers waiting for pgstac jobs # These parameters control how long services wait for pgstac migration jobs to complete waitConfig: diff --git a/docs/configuration.md b/docs/configuration.md index f14fc078..d9c84dc0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -102,16 +102,19 @@ Control search result count calculations: | `context_estimated_cost` | Cost threshold for estimates | "100000" | integer string | | `context_stats_ttl` | Stats cache duration | "1 day" | PostgreSQL interval | -### Queue Processing +### Automatic Maintenance Jobs -When `use_queue` is set to "true", a CronJob is automatically created to process queued queries: +Two CronJobs are conditionally created based on PgSTAC settings: -| **Values Key** | **Description** | **Default** | **Format** | -|:--------------|:----------------|:------------|:-----------| -| `queueProcessor.schedule` | Cron schedule for processing | "0 * * * *" | Cron format | -| `queueProcessor.command` | SQL command to run | "SELECT run_queued_queries();" | SQL | +**Queue Processor** (when `use_queue: "true"`): +- `queueProcessor.schedule`: "0 * * * *" (hourly) +- `queueProcessor.command`: "SELECT run_queued_queries();" + +**Extent Updater** (when `update_collection_extent: "false"`): +- `extentUpdater.schedule`: "0 2 * * *" (daily at 2 AM) +- `extentUpdater.command`: "SELECT update_collection_extents();" -The queue processor runs hourly by default but can be customized for your workload. +Both schedules are customizable using standard cron format. Example configuration: From 543276a788cb75d7b01eb95415d822d5606ca00a Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Tue, 11 Nov 2025 16:43:18 +0100 Subject: [PATCH 4/6] Added helm tests. --- charts/eoapi/tests/pgstac_config_tests.yaml | 216 ++++++++++++++++++++ charts/eoapi/values.yaml | 8 +- 2 files changed, 220 insertions(+), 4 deletions(-) create mode 100644 charts/eoapi/tests/pgstac_config_tests.yaml diff --git a/charts/eoapi/tests/pgstac_config_tests.yaml b/charts/eoapi/tests/pgstac_config_tests.yaml new file mode 100644 index 00000000..9f653ded --- /dev/null +++ b/charts/eoapi/tests/pgstac_config_tests.yaml @@ -0,0 +1,216 @@ +suite: pgstac configuration tests +templates: + - templates/pgstacbootstrap/configmap.yaml + - templates/pgstacbootstrap/queue-processor.yaml + - templates/pgstacbootstrap/extent-updater.yaml +tests: + # PgSTAC Settings Tests + - it: should apply custom pgstac settings + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + queue_timeout: "30 minutes" + use_queue: "false" + update_collection_extent: "true" + context: "on" + context_estimated_count: "50000" + template: templates/pgstacbootstrap/configmap.yaml + documentIndex: 0 + asserts: + - matchRegex: + path: data["pgstac-settings.sql"] + pattern: "VALUES \\('queue_timeout', '30 minutes'\\)" + - matchRegex: + path: data["pgstac-settings.sql"] + pattern: "VALUES \\('use_queue', 'false'\\)" + - matchRegex: + path: data["pgstac-settings.sql"] + pattern: "VALUES \\('update_collection_extent', 'true'\\)" + - matchRegex: + path: data["pgstac-settings.sql"] + pattern: "VALUES \\('context_estimated_count', '50000'\\)" + + - it: should use default pgstac settings + set: + pgstacBootstrap: + enabled: true + template: templates/pgstacbootstrap/configmap.yaml + documentIndex: 0 + asserts: + - matchRegex: + path: data["pgstac-settings.sql"] + pattern: "VALUES \\('queue_timeout', '10 minutes'\\)" + - matchRegex: + path: data["pgstac-settings.sql"] + pattern: "VALUES \\('use_queue', 'true'\\)" + - matchRegex: + path: data["pgstac-settings.sql"] + pattern: "VALUES \\('update_collection_extent', 'false'\\)" + + # Queue Processor Tests + - it: should create queue processor when use_queue is true + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + use_queue: "true" + template: templates/pgstacbootstrap/queue-processor.yaml + asserts: + - hasDocuments: + count: 1 + - isKind: + of: CronJob + - equal: + path: metadata.name + value: RELEASE-NAME-pgstac-queue-processor + - equal: + path: spec.schedule + value: "0 * * * *" + + - it: should NOT create queue processor when use_queue is false + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + use_queue: "false" + template: templates/pgstacbootstrap/queue-processor.yaml + asserts: + - hasDocuments: + count: 0 + + - it: should use custom queue processor schedule + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + use_queue: "true" + queueProcessor: + schedule: "*/15 * * * *" + command: "SELECT custom_queue_function();" + template: templates/pgstacbootstrap/queue-processor.yaml + asserts: + - equal: + path: spec.schedule + value: "*/15 * * * *" + - matchRegex: + path: spec.jobTemplate.spec.template.spec.containers[0].command[2] + pattern: "SELECT custom_queue_function\\(\\);" + + # Extent Updater Tests + - it: should create extent updater when update_collection_extent is false + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + update_collection_extent: "false" + template: templates/pgstacbootstrap/extent-updater.yaml + asserts: + - hasDocuments: + count: 1 + - isKind: + of: CronJob + - equal: + path: metadata.name + value: RELEASE-NAME-pgstac-extent-updater + - equal: + path: spec.schedule + value: "0 2 * * *" + + - it: should NOT create extent updater when update_collection_extent is true + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + update_collection_extent: "true" + template: templates/pgstacbootstrap/extent-updater.yaml + asserts: + - hasDocuments: + count: 0 + + - it: should use custom extent updater schedule + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + update_collection_extent: "false" + extentUpdater: + schedule: "0 */6 * * *" + command: "SELECT custom_extent_function();" + template: templates/pgstacbootstrap/extent-updater.yaml + asserts: + - equal: + path: spec.schedule + value: "0 */6 * * *" + - matchRegex: + path: spec.jobTemplate.spec.template.spec.containers[0].command[2] + pattern: "SELECT custom_extent_function\\(\\);" + + # Combined scenario tests + - it: should create both cronjobs with proper settings + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + use_queue: "true" + update_collection_extent: "false" + templates: + - templates/pgstacbootstrap/queue-processor.yaml + - templates/pgstacbootstrap/extent-updater.yaml + asserts: + - hasDocuments: + count: 1 + - isKind: + of: CronJob + + - it: should not create any cronjobs when disabled + set: + pgstacBootstrap: + enabled: false + templates: + - templates/pgstacbootstrap/queue-processor.yaml + - templates/pgstacbootstrap/extent-updater.yaml + asserts: + - hasDocuments: + count: 0 + + # Resource limits tests + - it: should set correct resource limits for queue processor + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + use_queue: "true" + template: templates/pgstacbootstrap/queue-processor.yaml + asserts: + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources.limits.cpu + value: "256m" + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources.limits.memory + value: "512Mi" + + - it: should set correct resource limits for extent updater + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + update_collection_extent: "false" + template: templates/pgstacbootstrap/extent-updater.yaml + asserts: + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources.limits.cpu + value: "512m" + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources.limits.memory + value: "1024Mi" diff --git a/charts/eoapi/values.yaml b/charts/eoapi/values.yaml index 36d2f5a3..8c71034a 100644 --- a/charts/eoapi/values.yaml +++ b/charts/eoapi/values.yaml @@ -182,10 +182,10 @@ pgstacBootstrap: pgstacSettings: # Queue configuration for handling complex queries queue_timeout: "10 minutes" # Timeout for queued queries (PostgreSQL interval format) - use_queue: "true" # Enable/disable query queue mechanism + use_queue: "false" # Enable/disable query queue mechanism; if enabled a 'queueProcessor' cronjob is being set # Collection extent management - update_collection_extent: "false" # Auto-update collection extents when items are added/modified + update_collection_extent: "true" # Auto-update collection extents when items are added/modified; if disabled a 'extentUpdater' cronjob is being set # Context configuration for search results context: "auto" # Context mode: "on", "off", or "auto" @@ -195,12 +195,12 @@ pgstacBootstrap: # Queue processing configuration (only used when use_queue is "true") queueProcessor: - schedule: "0 * * * *" # Run every hour (cron format) + schedule: "0 * * * *" # Run every hour command: "SELECT run_queued_queries();" # Extent updater configuration (only used when update_collection_extent is "false") extentUpdater: - schedule: "0 2 * * *" # Run daily at 2 AM (cron format) + schedule: "0 2 * * *" # Run daily at 2 AM command: "SELECT update_collection_extents();" # Wait configuration for init containers waiting for pgstac jobs From 45a4b9bba089228014718160ab942bae2c2577d8 Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Tue, 11 Nov 2025 16:55:27 +0100 Subject: [PATCH 5/6] Updated values.schema.json --- charts/eoapi/values.schema.json | 76 +++++++++++++++++++++++++++++++++ charts/eoapi/values.yaml | 2 +- docs/configuration.md | 2 +- 3 files changed, 78 insertions(+), 2 deletions(-) diff --git a/charts/eoapi/values.schema.json b/charts/eoapi/values.schema.json index 8ee7014b..34ffdd91 100644 --- a/charts/eoapi/values.schema.json +++ b/charts/eoapi/values.schema.json @@ -272,6 +272,82 @@ "resources": { "type": "object", "description": "Resource requirements" + }, + "pgstacSettings": { + "type": "object", + "description": "PgSTAC database configuration settings", + "properties": { + "queue_timeout": { + "type": "string", + "default": "10 minutes", + "description": "Timeout for queued queries (PostgreSQL interval format)" + }, + "use_queue": { + "type": "string", + "enum": ["true", "false"], + "default": "true", + "description": "Enable/disable query queue mechanism" + }, + "update_collection_extent": { + "type": "string", + "enum": ["true", "false"], + "default": "false", + "description": "Auto-update collection extents when items are added/modified" + }, + "context": { + "type": "string", + "enum": ["on", "off", "auto"], + "default": "auto", + "description": "Context mode for search results" + }, + "context_estimated_count": { + "type": "string", + "default": "100000", + "description": "Row threshold for using estimates instead of exact counts" + }, + "context_estimated_cost": { + "type": "string", + "default": "100000", + "description": "Query cost threshold for using estimates" + }, + "context_stats_ttl": { + "type": "string", + "default": "1 day", + "description": "Cache duration for context statistics (PostgreSQL interval)" + } + } + }, + "queueProcessor": { + "type": "object", + "description": "Queue processor CronJob configuration (active when use_queue is true)", + "properties": { + "schedule": { + "type": "string", + "default": "0 * * * *", + "description": "Cron schedule for processing queued queries" + }, + "command": { + "type": "string", + "default": "RUN run_queued_queries();", + "description": "SQL command to process queue" + } + } + }, + "extentUpdater": { + "type": "object", + "description": "Extent updater CronJob configuration (active when update_collection_extent is false)", + "properties": { + "schedule": { + "type": "string", + "default": "0 2 * * *", + "description": "Cron schedule for updating collection extents" + }, + "command": { + "type": "string", + "default": "SELECT update_collection_extents();", + "description": "SQL command to update extents" + } + } } } } diff --git a/charts/eoapi/values.yaml b/charts/eoapi/values.yaml index 8c71034a..41a0bcb5 100644 --- a/charts/eoapi/values.yaml +++ b/charts/eoapi/values.yaml @@ -196,7 +196,7 @@ pgstacBootstrap: # Queue processing configuration (only used when use_queue is "true") queueProcessor: schedule: "0 * * * *" # Run every hour - command: "SELECT run_queued_queries();" + command: "RUN run_queued_queries();" # Extent updater configuration (only used when update_collection_extent is "false") extentUpdater: diff --git a/docs/configuration.md b/docs/configuration.md index d9c84dc0..f4785849 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -108,7 +108,7 @@ Two CronJobs are conditionally created based on PgSTAC settings: **Queue Processor** (when `use_queue: "true"`): - `queueProcessor.schedule`: "0 * * * *" (hourly) -- `queueProcessor.command`: "SELECT run_queued_queries();" +- `queueProcessor.command`: "RUN run_queued_queries();" **Extent Updater** (when `update_collection_extent: "false"`): - `extentUpdater.schedule`: "0 2 * * *" (daily at 2 AM) From ae9dcd1387361c438b727035548bd7eb3437f595 Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Tue, 11 Nov 2025 17:57:32 +0100 Subject: [PATCH 6/6] Moved as much as possible into scripts. --- .../pgstacbootstrap/extent-updater.yaml | 2 +- .../pgstacbootstrap/queue-processor.yaml | 2 +- charts/eoapi/tests/pgstac_config_tests.yaml | 10 ++++------ charts/eoapi/values.schema.json | 14 ++------------ charts/eoapi/values.yaml | 6 ++---- docs/configuration.md | 16 +++++++++------- 6 files changed, 19 insertions(+), 31 deletions(-) diff --git a/charts/eoapi/templates/pgstacbootstrap/extent-updater.yaml b/charts/eoapi/templates/pgstacbootstrap/extent-updater.yaml index 7ef410b6..6dd3ad23 100644 --- a/charts/eoapi/templates/pgstacbootstrap/extent-updater.yaml +++ b/charts/eoapi/templates/pgstacbootstrap/extent-updater.yaml @@ -31,7 +31,7 @@ spec: - "/bin/sh" - "-c" - | - psql -c "{{ .Values.pgstacBootstrap.settings.extentUpdater.command }}" + psql -c "SELECT update_collection_extents();" env: {{- include "eoapi.postgresqlEnv" . | nindent 14 }} resources: diff --git a/charts/eoapi/templates/pgstacbootstrap/queue-processor.yaml b/charts/eoapi/templates/pgstacbootstrap/queue-processor.yaml index 4de8a8d0..5b771dd1 100644 --- a/charts/eoapi/templates/pgstacbootstrap/queue-processor.yaml +++ b/charts/eoapi/templates/pgstacbootstrap/queue-processor.yaml @@ -31,7 +31,7 @@ spec: - "/bin/sh" - "-c" - | - psql -c "{{ .Values.pgstacBootstrap.settings.queueProcessor.command }}" + psql -c "RUN run_queued_queries();" env: {{- include "eoapi.postgresqlEnv" . | nindent 14 }} resources: diff --git a/charts/eoapi/tests/pgstac_config_tests.yaml b/charts/eoapi/tests/pgstac_config_tests.yaml index 9f653ded..ae10884d 100644 --- a/charts/eoapi/tests/pgstac_config_tests.yaml +++ b/charts/eoapi/tests/pgstac_config_tests.yaml @@ -44,10 +44,10 @@ tests: pattern: "VALUES \\('queue_timeout', '10 minutes'\\)" - matchRegex: path: data["pgstac-settings.sql"] - pattern: "VALUES \\('use_queue', 'true'\\)" + pattern: "VALUES \\('use_queue', 'false'\\)" - matchRegex: path: data["pgstac-settings.sql"] - pattern: "VALUES \\('update_collection_extent', 'false'\\)" + pattern: "VALUES \\('update_collection_extent', 'true'\\)" # Queue Processor Tests - it: should create queue processor when use_queue is true @@ -91,7 +91,6 @@ tests: use_queue: "true" queueProcessor: schedule: "*/15 * * * *" - command: "SELECT custom_queue_function();" template: templates/pgstacbootstrap/queue-processor.yaml asserts: - equal: @@ -99,7 +98,7 @@ tests: value: "*/15 * * * *" - matchRegex: path: spec.jobTemplate.spec.template.spec.containers[0].command[2] - pattern: "SELECT custom_queue_function\\(\\);" + pattern: "RUN run_queued_queries\\(\\);" # Extent Updater Tests - it: should create extent updater when update_collection_extent is false @@ -143,7 +142,6 @@ tests: update_collection_extent: "false" extentUpdater: schedule: "0 */6 * * *" - command: "SELECT custom_extent_function();" template: templates/pgstacbootstrap/extent-updater.yaml asserts: - equal: @@ -151,7 +149,7 @@ tests: value: "0 */6 * * *" - matchRegex: path: spec.jobTemplate.spec.template.spec.containers[0].command[2] - pattern: "SELECT custom_extent_function\\(\\);" + pattern: "SELECT update_collection_extents\\(\\);" # Combined scenario tests - it: should create both cronjobs with proper settings diff --git a/charts/eoapi/values.schema.json b/charts/eoapi/values.schema.json index 34ffdd91..ce836105 100644 --- a/charts/eoapi/values.schema.json +++ b/charts/eoapi/values.schema.json @@ -285,13 +285,13 @@ "use_queue": { "type": "string", "enum": ["true", "false"], - "default": "true", + "default": "false", "description": "Enable/disable query queue mechanism" }, "update_collection_extent": { "type": "string", "enum": ["true", "false"], - "default": "false", + "default": "true", "description": "Auto-update collection extents when items are added/modified" }, "context": { @@ -325,11 +325,6 @@ "type": "string", "default": "0 * * * *", "description": "Cron schedule for processing queued queries" - }, - "command": { - "type": "string", - "default": "RUN run_queued_queries();", - "description": "SQL command to process queue" } } }, @@ -341,11 +336,6 @@ "type": "string", "default": "0 2 * * *", "description": "Cron schedule for updating collection extents" - }, - "command": { - "type": "string", - "default": "SELECT update_collection_extents();", - "description": "SQL command to update extents" } } } diff --git a/charts/eoapi/values.yaml b/charts/eoapi/values.yaml index 41a0bcb5..1806cd9e 100644 --- a/charts/eoapi/values.yaml +++ b/charts/eoapi/values.yaml @@ -182,10 +182,10 @@ pgstacBootstrap: pgstacSettings: # Queue configuration for handling complex queries queue_timeout: "10 minutes" # Timeout for queued queries (PostgreSQL interval format) - use_queue: "false" # Enable/disable query queue mechanism; if enabled a 'queueProcessor' cronjob is being set + use_queue: "false" # Enable/disable query queue mechanism # Collection extent management - update_collection_extent: "true" # Auto-update collection extents when items are added/modified; if disabled a 'extentUpdater' cronjob is being set + update_collection_extent: "true" # Auto-update collection extents when items are added/modified # Context configuration for search results context: "auto" # Context mode: "on", "off", or "auto" @@ -196,12 +196,10 @@ pgstacBootstrap: # Queue processing configuration (only used when use_queue is "true") queueProcessor: schedule: "0 * * * *" # Run every hour - command: "RUN run_queued_queries();" # Extent updater configuration (only used when update_collection_extent is "false") extentUpdater: schedule: "0 2 * * *" # Run daily at 2 AM - command: "SELECT update_collection_extents();" # Wait configuration for init containers waiting for pgstac jobs # These parameters control how long services wait for pgstac migration jobs to complete diff --git a/docs/configuration.md b/docs/configuration.md index f4785849..8bc38c63 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -88,8 +88,8 @@ Configure via `pgstacBootstrap.settings.pgstacSettings`: | **Values Key** | **Description** | **Default** | **Format** | |:--------------|:----------------|:------------|:-----------| | `queue_timeout` | Timeout for queued queries | "10 minutes" | PostgreSQL interval | -| `use_queue` | Enable query queue mechanism | "true" | boolean string | -| `update_collection_extent` | Auto-update collection extents | "false" | boolean string | +| `use_queue` | Enable query queue mechanism | "false" | boolean string | +| `update_collection_extent` | Auto-update collection extents | "true" | boolean string | ### Context Settings @@ -104,15 +104,17 @@ Control search result count calculations: ### Automatic Maintenance Jobs -Two CronJobs are conditionally created based on PgSTAC settings: +CronJobs are conditionally created based on PgSTAC settings: -**Queue Processor** (when `use_queue: "true"`): +**Queue Processor** (created when `use_queue: "true"`): - `queueProcessor.schedule`: "0 * * * *" (hourly) -- `queueProcessor.command`: "RUN run_queued_queries();" +- Processes queries that exceeded timeout -**Extent Updater** (when `update_collection_extent: "false"`): +**Extent Updater** (created when `update_collection_extent: "false"`): - `extentUpdater.schedule`: "0 2 * * *" (daily at 2 AM) -- `extentUpdater.command`: "SELECT update_collection_extents();" +- Updates collection spatial/temporal boundaries + +By default, no CronJobs are created (use_queue=false, update_collection_extent=true). Both schedules are customizable using standard cron format.