diff --git a/CHANGELOG.md b/CHANGELOG.md index f37b059d..68131123 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Expose PgSTAC configuration options in Helm chart values (`pgstacBootstrap.settings.pgstacSettings`). These are being dynamically applied via templated SQL during bootstrap. + - Added `queue_timeout`, `use_queue`, and `update_collection_extent` settings for database performance tuning + - Made existing context settings configurable (`context`, `context_estimated_count`, `context_estimated_cost`, `context_stats_ttl`) + - Automatic queue processor CronJob created when `use_queue` is "true" (configurable schedule via `queueProcessor.schedule`) + - Automatic extent updater CronJob created when `update_collection_extent` is "false" (configurable schedule via `extentUpdater.schedule`) + ## [0.7.13] - 2025-11-04 ### Added diff --git a/charts/eoapi/initdb-data/settings/pgstac-settings.sql b/charts/eoapi/initdb-data/settings/pgstac-settings.sql deleted file mode 100644 index 2827508a..00000000 --- a/charts/eoapi/initdb-data/settings/pgstac-settings.sql +++ /dev/null @@ -1,12 +0,0 @@ --- Apply pgstac settings -DELETE FROM pgstac.pgstac_settings WHERE name = 'context'; -INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context', 'auto'); - -DELETE FROM pgstac.pgstac_settings WHERE name = 'context_estimated_count'; -INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_estimated_count', '100000'); - -DELETE FROM pgstac.pgstac_settings WHERE name = 'context_estimated_cost'; -INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_estimated_cost', '100000'); - -DELETE FROM pgstac.pgstac_settings WHERE name = 'context_stats_ttl'; -INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_stats_ttl', '1 day'); diff --git a/charts/eoapi/initdb-data/settings/pgstac-settings.sql.tpl b/charts/eoapi/initdb-data/settings/pgstac-settings.sql.tpl new file mode 100644 index 00000000..52eb1d9e --- /dev/null +++ b/charts/eoapi/initdb-data/settings/pgstac-settings.sql.tpl @@ -0,0 +1,26 @@ +-- Apply pgstac settings +-- These settings are configured via Helm values at pgstacBootstrap.settings.pgstacSettings + +-- Queue settings +DELETE FROM pgstac.pgstac_settings WHERE name = 'queue_timeout'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('queue_timeout', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.queue_timeout }}'); + +DELETE FROM pgstac.pgstac_settings WHERE name = 'use_queue'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('use_queue', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.use_queue }}'); + +-- Collection extent management +DELETE FROM pgstac.pgstac_settings WHERE name = 'update_collection_extent'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('update_collection_extent', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.update_collection_extent }}'); + +-- Context settings +DELETE FROM pgstac.pgstac_settings WHERE name = 'context'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.context }}'); + +DELETE FROM pgstac.pgstac_settings WHERE name = 'context_estimated_count'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_estimated_count', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.context_estimated_count }}'); + +DELETE FROM pgstac.pgstac_settings WHERE name = 'context_estimated_cost'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_estimated_cost', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.context_estimated_cost }}'); + +DELETE FROM pgstac.pgstac_settings WHERE name = 'context_stats_ttl'; +INSERT INTO pgstac.pgstac_settings (name, value) VALUES ('context_stats_ttl', '{{ .Values.pgstacBootstrap.settings.pgstacSettings.context_stats_ttl }}'); diff --git a/charts/eoapi/templates/pgstacbootstrap/configmap.yaml b/charts/eoapi/templates/pgstacbootstrap/configmap.yaml index b00c0d47..ce5b77d0 100644 --- a/charts/eoapi/templates/pgstacbootstrap/configmap.yaml +++ b/charts/eoapi/templates/pgstacbootstrap/configmap.yaml @@ -13,7 +13,7 @@ metadata: helm.sh/hook-delete-policy: "before-hook-creation,hook-succeeded" data: pgstac-settings.sql: | - {{ $.Files.Get "initdb-data/settings/pgstac-settings.sql" | nindent 4 }} + {{- tpl ($.Files.Get "initdb-data/settings/pgstac-settings.sql.tpl") $ | nindent 4 }} {{- if (index .Values "eoapi-notifier").enabled }} {{ $.Files.Get "initdb-data/settings/pgstac-notification-triggers.sql" | nindent 4 }} {{- end }} diff --git a/charts/eoapi/templates/pgstacbootstrap/extent-updater.yaml b/charts/eoapi/templates/pgstacbootstrap/extent-updater.yaml new file mode 100644 index 00000000..6dd3ad23 --- /dev/null +++ b/charts/eoapi/templates/pgstacbootstrap/extent-updater.yaml @@ -0,0 +1,46 @@ +{{- if .Values.pgstacBootstrap.enabled }} +{{- if .Values.pgstacBootstrap.settings.pgstacSettings }} +{{- if eq (default "false" .Values.pgstacBootstrap.settings.pgstacSettings.update_collection_extent) "false" }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-pgstac-extent-updater + labels: + {{- include "eoapi.labels" . | nindent 4 }} + app.kubernetes.io/component: pgstac-extents +spec: + schedule: {{ .Values.pgstacBootstrap.settings.extentUpdater.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + metadata: + labels: + {{- include "eoapi.labels" . | nindent 12 }} + app.kubernetes.io/component: pgstac-extents + spec: + restartPolicy: OnFailure + containers: + - name: extent-updater + image: {{ .Values.pgstacBootstrap.image.name }}:{{ .Values.pgstacBootstrap.image.tag }} + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-c" + - | + psql -c "SELECT update_collection_extents();" + env: + {{- include "eoapi.postgresqlEnv" . | nindent 14 }} + resources: + limits: + cpu: "512m" + memory: "1024Mi" + requests: + cpu: "256m" + memory: "512Mi" +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/eoapi/templates/pgstacbootstrap/queue-processor.yaml b/charts/eoapi/templates/pgstacbootstrap/queue-processor.yaml new file mode 100644 index 00000000..5b771dd1 --- /dev/null +++ b/charts/eoapi/templates/pgstacbootstrap/queue-processor.yaml @@ -0,0 +1,46 @@ +{{- if .Values.pgstacBootstrap.enabled }} +{{- if .Values.pgstacBootstrap.settings.pgstacSettings }} +{{- if eq (.Values.pgstacBootstrap.settings.pgstacSettings.use_queue | default "true") "true" }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-pgstac-queue-processor + labels: + {{- include "eoapi.labels" . | nindent 4 }} + app.kubernetes.io/component: pgstac-queue +spec: + schedule: {{ .Values.pgstacBootstrap.settings.queueProcessor.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + metadata: + labels: + {{- include "eoapi.labels" . | nindent 12 }} + app.kubernetes.io/component: pgstac-queue + spec: + restartPolicy: OnFailure + containers: + - name: queue-processor + image: {{ .Values.pgstacBootstrap.image.name }}:{{ .Values.pgstacBootstrap.image.tag }} + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-c" + - | + psql -c "RUN run_queued_queries();" + env: + {{- include "eoapi.postgresqlEnv" . | nindent 14 }} + resources: + limits: + cpu: "256m" + memory: "512Mi" + requests: + cpu: "128m" + memory: "256Mi" +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/eoapi/tests/pgstac_config_tests.yaml b/charts/eoapi/tests/pgstac_config_tests.yaml new file mode 100644 index 00000000..ae10884d --- /dev/null +++ b/charts/eoapi/tests/pgstac_config_tests.yaml @@ -0,0 +1,214 @@ +suite: pgstac configuration tests +templates: + - templates/pgstacbootstrap/configmap.yaml + - templates/pgstacbootstrap/queue-processor.yaml + - templates/pgstacbootstrap/extent-updater.yaml +tests: + # PgSTAC Settings Tests + - it: should apply custom pgstac settings + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + queue_timeout: "30 minutes" + use_queue: "false" + update_collection_extent: "true" + context: "on" + context_estimated_count: "50000" + template: templates/pgstacbootstrap/configmap.yaml + documentIndex: 0 + asserts: + - matchRegex: + path: data["pgstac-settings.sql"] + pattern: "VALUES \\('queue_timeout', '30 minutes'\\)" + - matchRegex: + path: data["pgstac-settings.sql"] + pattern: "VALUES \\('use_queue', 'false'\\)" + - matchRegex: + path: data["pgstac-settings.sql"] + pattern: "VALUES \\('update_collection_extent', 'true'\\)" + - matchRegex: + path: data["pgstac-settings.sql"] + pattern: "VALUES \\('context_estimated_count', '50000'\\)" + + - it: should use default pgstac settings + set: + pgstacBootstrap: + enabled: true + template: templates/pgstacbootstrap/configmap.yaml + documentIndex: 0 + asserts: + - matchRegex: + path: data["pgstac-settings.sql"] + pattern: "VALUES \\('queue_timeout', '10 minutes'\\)" + - matchRegex: + path: data["pgstac-settings.sql"] + pattern: "VALUES \\('use_queue', 'false'\\)" + - matchRegex: + path: data["pgstac-settings.sql"] + pattern: "VALUES \\('update_collection_extent', 'true'\\)" + + # Queue Processor Tests + - it: should create queue processor when use_queue is true + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + use_queue: "true" + template: templates/pgstacbootstrap/queue-processor.yaml + asserts: + - hasDocuments: + count: 1 + - isKind: + of: CronJob + - equal: + path: metadata.name + value: RELEASE-NAME-pgstac-queue-processor + - equal: + path: spec.schedule + value: "0 * * * *" + + - it: should NOT create queue processor when use_queue is false + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + use_queue: "false" + template: templates/pgstacbootstrap/queue-processor.yaml + asserts: + - hasDocuments: + count: 0 + + - it: should use custom queue processor schedule + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + use_queue: "true" + queueProcessor: + schedule: "*/15 * * * *" + template: templates/pgstacbootstrap/queue-processor.yaml + asserts: + - equal: + path: spec.schedule + value: "*/15 * * * *" + - matchRegex: + path: spec.jobTemplate.spec.template.spec.containers[0].command[2] + pattern: "RUN run_queued_queries\\(\\);" + + # Extent Updater Tests + - it: should create extent updater when update_collection_extent is false + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + update_collection_extent: "false" + template: templates/pgstacbootstrap/extent-updater.yaml + asserts: + - hasDocuments: + count: 1 + - isKind: + of: CronJob + - equal: + path: metadata.name + value: RELEASE-NAME-pgstac-extent-updater + - equal: + path: spec.schedule + value: "0 2 * * *" + + - it: should NOT create extent updater when update_collection_extent is true + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + update_collection_extent: "true" + template: templates/pgstacbootstrap/extent-updater.yaml + asserts: + - hasDocuments: + count: 0 + + - it: should use custom extent updater schedule + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + update_collection_extent: "false" + extentUpdater: + schedule: "0 */6 * * *" + template: templates/pgstacbootstrap/extent-updater.yaml + asserts: + - equal: + path: spec.schedule + value: "0 */6 * * *" + - matchRegex: + path: spec.jobTemplate.spec.template.spec.containers[0].command[2] + pattern: "SELECT update_collection_extents\\(\\);" + + # Combined scenario tests + - it: should create both cronjobs with proper settings + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + use_queue: "true" + update_collection_extent: "false" + templates: + - templates/pgstacbootstrap/queue-processor.yaml + - templates/pgstacbootstrap/extent-updater.yaml + asserts: + - hasDocuments: + count: 1 + - isKind: + of: CronJob + + - it: should not create any cronjobs when disabled + set: + pgstacBootstrap: + enabled: false + templates: + - templates/pgstacbootstrap/queue-processor.yaml + - templates/pgstacbootstrap/extent-updater.yaml + asserts: + - hasDocuments: + count: 0 + + # Resource limits tests + - it: should set correct resource limits for queue processor + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + use_queue: "true" + template: templates/pgstacbootstrap/queue-processor.yaml + asserts: + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources.limits.cpu + value: "256m" + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources.limits.memory + value: "512Mi" + + - it: should set correct resource limits for extent updater + set: + pgstacBootstrap: + enabled: true + settings: + pgstacSettings: + update_collection_extent: "false" + template: templates/pgstacbootstrap/extent-updater.yaml + asserts: + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources.limits.cpu + value: "512m" + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources.limits.memory + value: "1024Mi" diff --git a/charts/eoapi/values.schema.json b/charts/eoapi/values.schema.json index 8ee7014b..ce836105 100644 --- a/charts/eoapi/values.schema.json +++ b/charts/eoapi/values.schema.json @@ -272,6 +272,72 @@ "resources": { "type": "object", "description": "Resource requirements" + }, + "pgstacSettings": { + "type": "object", + "description": "PgSTAC database configuration settings", + "properties": { + "queue_timeout": { + "type": "string", + "default": "10 minutes", + "description": "Timeout for queued queries (PostgreSQL interval format)" + }, + "use_queue": { + "type": "string", + "enum": ["true", "false"], + "default": "false", + "description": "Enable/disable query queue mechanism" + }, + "update_collection_extent": { + "type": "string", + "enum": ["true", "false"], + "default": "true", + "description": "Auto-update collection extents when items are added/modified" + }, + "context": { + "type": "string", + "enum": ["on", "off", "auto"], + "default": "auto", + "description": "Context mode for search results" + }, + "context_estimated_count": { + "type": "string", + "default": "100000", + "description": "Row threshold for using estimates instead of exact counts" + }, + "context_estimated_cost": { + "type": "string", + "default": "100000", + "description": "Query cost threshold for using estimates" + }, + "context_stats_ttl": { + "type": "string", + "default": "1 day", + "description": "Cache duration for context statistics (PostgreSQL interval)" + } + } + }, + "queueProcessor": { + "type": "object", + "description": "Queue processor CronJob configuration (active when use_queue is true)", + "properties": { + "schedule": { + "type": "string", + "default": "0 * * * *", + "description": "Cron schedule for processing queued queries" + } + } + }, + "extentUpdater": { + "type": "object", + "description": "Extent updater CronJob configuration (active when update_collection_extent is false)", + "properties": { + "schedule": { + "type": "string", + "default": "0 2 * * *", + "description": "Cron schedule for updating collection extents" + } + } } } } diff --git a/charts/eoapi/values.yaml b/charts/eoapi/values.yaml index 2fc60ae0..1806cd9e 100644 --- a/charts/eoapi/values.yaml +++ b/charts/eoapi/values.yaml @@ -177,6 +177,30 @@ pgstacBootstrap: # deleteMissing: true queryables: [] + # PgSTAC settings configuration + # These settings control key PgSTAC behaviors and performance characteristics + pgstacSettings: + # Queue configuration for handling complex queries + queue_timeout: "10 minutes" # Timeout for queued queries (PostgreSQL interval format) + use_queue: "false" # Enable/disable query queue mechanism + + # Collection extent management + update_collection_extent: "true" # Auto-update collection extents when items are added/modified + + # Context configuration for search results + context: "auto" # Context mode: "on", "off", or "auto" + context_estimated_count: "100000" # Row threshold for using estimates instead of exact counts + context_estimated_cost: "100000" # Query cost threshold for using estimates + context_stats_ttl: "1 day" # Cache duration for context statistics + + # Queue processing configuration (only used when use_queue is "true") + queueProcessor: + schedule: "0 * * * *" # Run every hour + + # Extent updater configuration (only used when update_collection_extent is "false") + extentUpdater: + schedule: "0 2 * * *" # Run daily at 2 AM + # Wait configuration for init containers waiting for pgstac jobs # These parameters control how long services wait for pgstac migration jobs to complete waitConfig: diff --git a/docs/configuration.md b/docs/configuration.md index 9a036433..8bc38c63 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -48,6 +48,7 @@ Using Crunchydata's PostgreSQL Operator (`postgresql.type: "postgrescluster"`): For external databases, set `postgresql.type` to either: + 1. Using plaintext credentials (`external-plaintext`): ```yaml postgresql: @@ -76,6 +77,65 @@ postgresql: database: "eoapi" # can also be in secret ``` +## PgSTAC Configuration + +Control PgSTAC database behavior and performance tuning: + +### Core Settings + +Configure via `pgstacBootstrap.settings.pgstacSettings`: + +| **Values Key** | **Description** | **Default** | **Format** | +|:--------------|:----------------|:------------|:-----------| +| `queue_timeout` | Timeout for queued queries | "10 minutes" | PostgreSQL interval | +| `use_queue` | Enable query queue mechanism | "false" | boolean string | +| `update_collection_extent` | Auto-update collection extents | "true" | boolean string | + +### Context Settings + +Control search result count calculations: + +| **Values Key** | **Description** | **Default** | **Format** | +|:--------------|:----------------|:------------|:-----------| +| `context` | Context mode | "auto" | "on", "off", "auto" | +| `context_estimated_count` | Row threshold for estimates | "100000" | integer string | +| `context_estimated_cost` | Cost threshold for estimates | "100000" | integer string | +| `context_stats_ttl` | Stats cache duration | "1 day" | PostgreSQL interval | + +### Automatic Maintenance Jobs + +CronJobs are conditionally created based on PgSTAC settings: + +**Queue Processor** (created when `use_queue: "true"`): +- `queueProcessor.schedule`: "0 * * * *" (hourly) +- Processes queries that exceeded timeout + +**Extent Updater** (created when `update_collection_extent: "false"`): +- `extentUpdater.schedule`: "0 2 * * *" (daily at 2 AM) +- Updates collection spatial/temporal boundaries + +By default, no CronJobs are created (use_queue=false, update_collection_extent=true). + +Both schedules are customizable using standard cron format. + +Example configuration: + +```yaml +pgstacBootstrap: + settings: + pgstacSettings: + # Performance tuning for large datasets + queue_timeout: "20 minutes" + use_queue: "true" + update_collection_extent: "false" + + # Optimize context for performance + context: "auto" + context_estimated_count: "50000" + context_estimated_cost: "75000" + context_stats_ttl: "12 hours" +``` + ## Ingress Configuration Unified ingress configuration supporting both NGINX and Traefik: