From 82dce80e5cb147bc1bc62cb88e462bef1e2aa9ba Mon Sep 17 00:00:00 2001 From: Emmanuel Mathot Date: Wed, 19 Nov 2025 16:21:24 +0100 Subject: [PATCH 01/10] feat: Add support for file-based and ConfigMapRef queryables in pgstacBootstrap --- .../database/pgstacbootstrap/configmap.yaml | 13 +- .../database/pgstacbootstrap/job.yaml | 38 +- charts/eoapi/tests/queryables_tests.yaml | 402 ++++++++++++++++++ charts/eoapi/values.yaml | 19 +- 4 files changed, 460 insertions(+), 12 deletions(-) create mode 100644 charts/eoapi/tests/queryables_tests.yaml diff --git a/charts/eoapi/templates/database/pgstacbootstrap/configmap.yaml b/charts/eoapi/templates/database/pgstacbootstrap/configmap.yaml index ce5b77d0..564c853d 100644 --- a/charts/eoapi/templates/database/pgstacbootstrap/configmap.yaml +++ b/charts/eoapi/templates/database/pgstacbootstrap/configmap.yaml @@ -48,6 +48,13 @@ data: {{- end }} --- {{- if .Values.pgstacBootstrap.settings.queryables }} +{{- $hasFileBasedQueryables := false }} +{{- range $config := .Values.pgstacBootstrap.settings.queryables }} + {{- if $config.file }} + {{- $hasFileBasedQueryables = true }} + {{- end }} +{{- end }} +{{- if $hasFileBasedQueryables }} apiVersion: v1 kind: ConfigMap metadata: @@ -58,10 +65,12 @@ metadata: helm.sh/hook-delete-policy: "before-hook-creation,hook-succeeded" data: {{- range $config := .Values.pgstacBootstrap.settings.queryables }} - {{- $filename := splitList "/" $config.file | last }} - {{ $filename }}: | + {{- if $config.file }} + {{ $config.name }}: | {{- $.Files.Get $config.file | nindent 4 }} {{- end }} + {{- end }} +{{- end }} {{- end }} {{- end }} --- diff --git a/charts/eoapi/templates/database/pgstacbootstrap/job.yaml b/charts/eoapi/templates/database/pgstacbootstrap/job.yaml index aa70cab8..de4ad8b7 100644 --- a/charts/eoapi/templates/database/pgstacbootstrap/job.yaml +++ b/charts/eoapi/templates/database/pgstacbootstrap/job.yaml @@ -197,15 +197,14 @@ spec: # Load queryables configurations echo "Loading queryables configurations..." {{- range $idx, $config := .Values.pgstacBootstrap.settings.queryables }} - {{- $filename := splitList "/" $config.file | last }} - echo "Processing queryables file: {{ $filename }}" + echo "Processing queryables: {{ $config.name }}" pypgstac load-queryables \ {{- if $config.deleteMissing }} --delete-missing \ {{- end }} {{- if $config.collections }} # Complex escaping needed due to multiple interpretation layers: - # 1. Helm processes the template: \\\" becomes \" in the rendered YAML + # 1. Helm processes the template: \\\\\" becomes \" in the rendered YAML # 2. YAML processes the string: \" becomes " in the shell script # 3. Shell receives: --collection-ids '["collection1","collection2"]' # 4. pypgstac gets a proper JSON array string as expected @@ -214,21 +213,48 @@ spec: {{- if $config.indexFields }} --index-fields {{ join "," $config.indexFields }} \ {{- end }} - "/opt/queryables/{{ $filename }}" + "/opt/queryables/{{ $config.name }}" {{- end }} echo "Queryables loading complete" resources: {{- toYaml .Values.pgstacBootstrap.settings.resources | nindent 12 }} volumeMounts: - - mountPath: /opt/queryables - name: {{ .Release.Name }}-queryables-volume + {{- range $idx, $config := .Values.pgstacBootstrap.settings.queryables }} + {{- if $config.file }} + - mountPath: /opt/queryables/{{ $config.name }} + name: {{ $.Release.Name }}-queryables-volume + subPath: {{ $config.name }} + {{- else if $config.configMapRef }} + - mountPath: /opt/queryables/{{ $config.name }} + name: {{ $.Release.Name }}-queryables-configmapref-{{ $idx }} + subPath: {{ $config.configMapRef.key }} + {{- end }} + {{- end }} env: {{- include "eoapi.postgresqlEnv" . | nindent 12 }} volumes: + {{- $hasFileBasedQueryables := false }} + {{- range $config := .Values.pgstacBootstrap.settings.queryables }} + {{- if $config.file }} + {{- $hasFileBasedQueryables = true }} + {{- end }} + {{- end }} + {{- if $hasFileBasedQueryables }} - name: {{ .Release.Name }}-queryables-volume configMap: name: {{ .Release.Name }}-pgstac-queryables-config + {{- end }} + {{- range $idx, $config := .Values.pgstacBootstrap.settings.queryables }} + {{- if $config.configMapRef }} + - name: {{ $.Release.Name }}-queryables-configmapref-{{ $idx }} + configMap: + name: {{ $config.configMapRef.name }} + items: + - key: {{ $config.configMapRef.key }} + path: {{ $config.configMapRef.key }} + {{- end }} + {{- end }} {{- with .Values.pgstacBootstrap.settings.affinity }} affinity: {{- toYaml . | nindent 8 }} diff --git a/charts/eoapi/tests/queryables_tests.yaml b/charts/eoapi/tests/queryables_tests.yaml new file mode 100644 index 00000000..f9b6cafc --- /dev/null +++ b/charts/eoapi/tests/queryables_tests.yaml @@ -0,0 +1,402 @@ +suite: queryables configuration tests +templates: + - templates/_helpers/core.tpl + - templates/_helpers/database.tpl + - templates/database/pgstacbootstrap/configmap.yaml + - templates/database/pgstacbootstrap/job.yaml +tests: + # ConfigMap Tests - File-based queryables + - it: should create queryables configmap with file-based queryables + set: + pgstacBootstrap: + enabled: true + settings: + loadSamples: false + queryables: + - name: "common-queryables.json" + file: "initdb-data/queryables/test-queryables.json" + indexFields: ["platform", "instruments"] + deleteMissing: true + template: templates/database/pgstacbootstrap/configmap.yaml + documentIndex: 1 + asserts: + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: RELEASE-NAME-pgstac-queryables-config + - isNotEmpty: + path: data["common-queryables.json"] + + - it: should NOT create queryables configmap when only configMapRef is used + set: + pgstacBootstrap: + enabled: true + settings: + loadSamples: false + queryables: + - name: "custom-queryables.json" + configMapRef: + name: my-custom-configmap + key: queryables.json + template: templates/database/pgstacbootstrap/configmap.yaml + documentIndex: 0 + asserts: + - equal: + path: metadata.name + value: RELEASE-NAME-pgstac-settings-config + + - it: should create queryables configmap for mixed file and configMapRef + set: + pgstacBootstrap: + enabled: true + settings: + loadSamples: false + queryables: + - name: "common-queryables.json" + file: "initdb-data/queryables/test-queryables.json" + - name: "custom-queryables.json" + configMapRef: + name: my-custom-configmap + key: queryables.json + template: templates/database/pgstacbootstrap/configmap.yaml + documentIndex: 1 + asserts: + - isKind: + of: ConfigMap + - isNotEmpty: + path: data["common-queryables.json"] + + - it: should NOT create queryables configmap when queryables is empty + set: + pgstacBootstrap: + enabled: true + settings: + loadSamples: false + queryables: [] + template: templates/database/pgstacbootstrap/configmap.yaml + documentIndex: 0 + asserts: + - equal: + path: metadata.name + value: RELEASE-NAME-pgstac-settings-config + + # Job Tests - File-based queryables + - it: should create queryables job with file-based queryables + set: + pgstacBootstrap: + enabled: true + settings: + queryables: + - name: "common-queryables.json" + file: "initdb-data/queryables/test-queryables.json" + indexFields: ["platform", "instruments"] + deleteMissing: true + template: templates/database/pgstacbootstrap/job.yaml + documentIndex: 2 + asserts: + - isKind: + of: Job + - equal: + path: metadata.name + value: RELEASE-NAME-pgstac-load-queryables + + - it: should mount file-based queryables with correct path + set: + pgstacBootstrap: + enabled: true + settings: + queryables: + - name: "common-queryables.json" + file: "initdb-data/queryables/test-queryables.json" + template: templates/database/pgstacbootstrap/job.yaml + documentIndex: 2 + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + mountPath: /opt/queryables/common-queryables.json + name: RELEASE-NAME-queryables-volume + subPath: common-queryables.json + + - it: should create volume for file-based queryables + set: + pgstacBootstrap: + enabled: true + settings: + queryables: + - name: "common-queryables.json" + file: "initdb-data/queryables/test-queryables.json" + template: templates/database/pgstacbootstrap/job.yaml + documentIndex: 2 + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: RELEASE-NAME-queryables-volume + configMap: + name: RELEASE-NAME-pgstac-queryables-config + + - it: should include correct pypgstac command for file-based queryables + set: + pgstacBootstrap: + enabled: true + settings: + queryables: + - name: "common-queryables.json" + file: "initdb-data/queryables/test-queryables.json" + indexFields: ["platform", "instruments"] + deleteMissing: true + template: templates/database/pgstacbootstrap/job.yaml + documentIndex: 2 + asserts: + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: '--delete-missing' + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: '--index-fields platform,instruments' + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: '"/opt/queryables/common-queryables.json"' + + # Job Tests - ConfigMapRef queryables + - it: should mount configMapRef queryables with correct path + set: + pgstacBootstrap: + enabled: true + settings: + queryables: + - name: "custom-queryables.json" + configMapRef: + name: my-custom-configmap + key: queryables.json + template: templates/database/pgstacbootstrap/job.yaml + documentIndex: 2 + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + mountPath: /opt/queryables/custom-queryables.json + name: RELEASE-NAME-queryables-configmapref-0 + subPath: queryables.json + + - it: should create volume for configMapRef queryables + set: + pgstacBootstrap: + enabled: true + settings: + queryables: + - name: "custom-queryables.json" + configMapRef: + name: my-custom-configmap + key: queryables.json + template: templates/database/pgstacbootstrap/job.yaml + documentIndex: 2 + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: RELEASE-NAME-queryables-configmapref-0 + configMap: + name: my-custom-configmap + items: + - key: queryables.json + path: queryables.json + + - it: should include correct pypgstac command for configMapRef queryables + set: + pgstacBootstrap: + enabled: true + settings: + queryables: + - name: "custom-queryables.json" + configMapRef: + name: my-custom-configmap + key: queryables.json + indexFields: ["custom:field1"] + template: templates/database/pgstacbootstrap/job.yaml + documentIndex: 2 + asserts: + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: '--index-fields custom:field1' + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: '"/opt/queryables/custom-queryables.json"' + + # Job Tests - Mixed queryables + - it: should handle mixed file and configMapRef queryables + set: + pgstacBootstrap: + enabled: true + settings: + queryables: + - name: "common-queryables.json" + file: "initdb-data/queryables/test-queryables.json" + indexFields: ["platform"] + - name: "custom-queryables.json" + configMapRef: + name: my-custom-configmap + key: queryables.json + indexFields: ["custom:field1"] + template: templates/database/pgstacbootstrap/job.yaml + documentIndex: 2 + asserts: + - lengthEqual: + path: spec.template.spec.containers[0].volumeMounts + count: 2 + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + mountPath: /opt/queryables/common-queryables.json + name: RELEASE-NAME-queryables-volume + subPath: common-queryables.json + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + mountPath: /opt/queryables/custom-queryables.json + name: RELEASE-NAME-queryables-configmapref-1 + subPath: queryables.json + + - it: should create correct volumes for mixed queryables + set: + pgstacBootstrap: + enabled: true + settings: + queryables: + - name: "common-queryables.json" + file: "initdb-data/queryables/test-queryables.json" + - name: "custom-queryables.json" + configMapRef: + name: my-custom-configmap + key: queryables.json + template: templates/database/pgstacbootstrap/job.yaml + documentIndex: 2 + asserts: + - lengthEqual: + path: spec.template.spec.volumes + count: 2 + - contains: + path: spec.template.spec.volumes + content: + name: RELEASE-NAME-queryables-volume + configMap: + name: RELEASE-NAME-pgstac-queryables-config + - contains: + path: spec.template.spec.volumes + content: + name: RELEASE-NAME-queryables-configmapref-1 + configMap: + name: my-custom-configmap + items: + - key: queryables.json + path: queryables.json + + # Job Tests - Multiple configMapRef queryables + - it: should handle multiple configMapRef queryables + set: + pgstacBootstrap: + enabled: true + settings: + queryables: + - name: "custom1.json" + configMapRef: + name: configmap-1 + key: data.json + - name: "custom2.json" + configMapRef: + name: configmap-2 + key: queryables.json + template: templates/database/pgstacbootstrap/job.yaml + documentIndex: 2 + asserts: + - lengthEqual: + path: spec.template.spec.containers[0].volumeMounts + count: 2 + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + mountPath: /opt/queryables/custom1.json + name: RELEASE-NAME-queryables-configmapref-0 + subPath: data.json + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + mountPath: /opt/queryables/custom2.json + name: RELEASE-NAME-queryables-configmapref-1 + subPath: queryables.json + + # Job Tests - Optional parameters + - it: should include collections parameter when specified + set: + pgstacBootstrap: + enabled: true + settings: + queryables: + - name: "collection-queryables.json" + file: "initdb-data/queryables/test-queryables.json" + collections: ["collection-1", "collection-2"] + template: templates/database/pgstacbootstrap/job.yaml + documentIndex: 2 + asserts: + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: '--collection-ids' + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: 'collection-1' + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: 'collection-2' + + - it: should work without optional parameters + set: + pgstacBootstrap: + enabled: true + settings: + queryables: + - name: "simple-queryables.json" + file: "initdb-data/queryables/test-queryables.json" + template: templates/database/pgstacbootstrap/job.yaml + documentIndex: 2 + asserts: + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: '\"/opt/queryables/simple-queryables.json\"' + - notMatchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: '--delete-missing' + - notMatchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: '--collection-ids' + - notMatchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: '--index-fields' + + # Job Tests - NOT created when queryables is empty + - it: should NOT create queryables job when queryables is empty + set: + pgstacBootstrap: + enabled: true + settings: + queryables: [] + template: templates/database/pgstacbootstrap/job.yaml + documentIndex: 1 + asserts: + - equal: + path: metadata.name + value: RELEASE-NAME-pgstac-load-samples + + - it: should NOT create queryables job when pgstacBootstrap is disabled + set: + pgstacBootstrap: + enabled: false + settings: + queryables: + - name: "test.json" + file: "initdb-data/queryables/test-queryables.json" + template: templates/database/pgstacbootstrap/job.yaml + asserts: + - hasDocuments: + count: 0 diff --git a/charts/eoapi/values.yaml b/charts/eoapi/values.yaml index b48380d0..b092a7c9 100644 --- a/charts/eoapi/values.yaml +++ b/charts/eoapi/values.yaml @@ -165,16 +165,27 @@ pgstacBootstrap: # Queryables configuration # List of queryables configurations to load using pypgstac load-queryables - # Each item specifies a file path and optional parameters - # Example: + # Each item requires a mandatory 'name' field and either 'file' or 'configMapRef' + # + # Example using files from the chart: # queryables: - # - file: "initdb-data/queryables/common-queryables.json" + # - name: "common-queryables.json" + # file: "initdb-data/queryables/common-queryables.json" # indexFields: ["platform", "instruments"] # deleteMissing: true - # - file: "initdb-data/queryables/collection-specific.json" + # - name: "collection-specific.json" + # file: "initdb-data/queryables/collection-specific.json" # collections: ["my-collection-1", "my-collection-2"] # indexFields: ["custom:field1", "custom:field2"] # deleteMissing: true + # + # Example using external ConfigMap references: + # - name: "custom-queryables.json" + # configMapRef: + # name: my-custom-queryables-configmap + # key: queryables.json + # indexFields: ["custom:field1"] + # deleteMissing: true queryables: [] # PgSTAC settings configuration From dc84883faa20bfa16aff18f4fd7b65bffb844426 Mon Sep 17 00:00:00 2001 From: Emmanuel Mathot Date: Wed, 19 Nov 2025 16:24:18 +0100 Subject: [PATCH 02/10] feat: Add STAC queryables test to validate API response schema --- scripts/test.sh | 5 +++-- tests/integration/test_stac.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index 9270f7cd..f27597c8 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -150,9 +150,10 @@ test_all() { if validate_cluster 2>/dev/null; then test_integration || ((failed++)) - test_autoscaling || ((failed++)) + test_autoscaling || ((failed++)) test_autoscaling || ((failed++)) + test_notification || ((failed++)) else - log_warn "Skipping integration and autoscaling tests - no cluster connection" + log_warn "Skipping integration tests - no cluster connection" fi if [[ $failed -eq 0 ]]; then diff --git a/tests/integration/test_stac.py b/tests/integration/test_stac.py index ea67c9e2..40bd74a8 100644 --- a/tests/integration/test_stac.py +++ b/tests/integration/test_stac.py @@ -1,6 +1,8 @@ """test EOapi.""" +import json import os +from pathlib import Path import httpx @@ -126,3 +128,35 @@ def test_stac_custom_path(stac_endpoint: str) -> None: ) # assert resp.status_code == 307 assert resp.status_code == 404 + + +def test_stac_queryables(stac_endpoint: str) -> None: + """test queryables endpoint returns expected schema.""" + # Load expected queryables from test file + queryables_file = Path(__file__).parent.parent.parent / "charts" / "eoapi" / "initdb-data" / "queryables" / "test-queryables.json" + with open(queryables_file) as f: + expected_queryables = json.load(f) + + # Get queryables from API + resp = client.get(f"{stac_endpoint}/queryables") + assert resp.status_code == 200 + actual_queryables = resp.json() + + # Verify the queryables match the expected schema + assert actual_queryables["$schema"] == expected_queryables["$schema"] + assert actual_queryables["$id"] == expected_queryables["$id"] + assert actual_queryables["title"] == expected_queryables["title"] + assert actual_queryables["description"] == expected_queryables["description"] + assert actual_queryables["type"] == expected_queryables["type"] + + # Verify all expected properties are present + for prop_name, prop_schema in expected_queryables["properties"].items(): + assert prop_name in actual_queryables["properties"], ( + f"Expected property '{prop_name}' not found in queryables" + ) + assert actual_queryables["properties"][prop_name] == prop_schema, ( + f"Property '{prop_name}' schema doesn't match expected schema" + ) + + # Verify additionalProperties setting + assert actual_queryables.get("additionalProperties") == expected_queryables.get("additionalProperties") From b31e35da5e24df02142cbc308535f140c09ee260 Mon Sep 17 00:00:00 2001 From: Emmanuel Mathot Date: Wed, 19 Nov 2025 16:29:55 +0100 Subject: [PATCH 03/10] feat: Add support for ConfigMap reference-based queryables configuration --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afb52a87..c0ffbff6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added support for ConfigMap reference-based queryables configuration in addition to file-based queryables. Queryables can now be sourced from external ConfigMaps using `configMapRef`, from chart files using `file`, or a combination of both [#360](https://github.com/developmentseed/eoapi-k8s/pull/360) + ## Changed - Unified scripts and removed Makefile, combined all into one CLI command `eoapi-cli` [#359](https://github.com/developmentseed/eoapi-k8s/pull/359) @@ -40,7 +42,7 @@ stac: ### Added - Exposed PgSTAC configuration options in Helm chart values (`pgstacBootstrap.settings.pgstacSettings`). These are dynamically applied via templated SQL during bootstrap. [#340](https://github.com/developmentseed/eoapi-k8s/pull/340) - - Added `queue_timeout`, `use_queue`, and `update_collection_extent` settings for database performance tuning +- Added `queue_timeout`, `use_queue`, and `update_collection_extent` settings for database performance tuning - Made existing context settings configurable (`context`, `context_estimated_count`, `context_estimated_cost`, `context_stats_ttl`) - Automatic queue processor CronJob created when `use_queue` is "true" (configurable schedule via `queueProcessor.schedule`) - Automatic extent updater CronJob created when `update_collection_extent` is "false" (configurable schedule via `extentUpdater.schedule`) From 1f633b4b8b7bb8801c057aa4ce42b3d4fffef9c5 Mon Sep 17 00:00:00 2001 From: Emmanuel Mathot Date: Wed, 19 Nov 2025 16:32:13 +0100 Subject: [PATCH 04/10] test: Update STAC queryables test for improved schema validation --- charts/eoapi/values.yaml | 2 +- scripts/test.sh | 2 +- tests/integration/test_stac.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/eoapi/values.yaml b/charts/eoapi/values.yaml index b092a7c9..9678ef18 100644 --- a/charts/eoapi/values.yaml +++ b/charts/eoapi/values.yaml @@ -166,7 +166,7 @@ pgstacBootstrap: # Queryables configuration # List of queryables configurations to load using pypgstac load-queryables # Each item requires a mandatory 'name' field and either 'file' or 'configMapRef' - # + # # Example using files from the chart: # queryables: # - name: "common-queryables.json" diff --git a/scripts/test.sh b/scripts/test.sh index f27597c8..8fb3e10e 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -150,7 +150,7 @@ test_all() { if validate_cluster 2>/dev/null; then test_integration || ((failed++)) - test_autoscaling || ((failed++)) test_autoscaling || ((failed++)) + test_autoscaling || ((failed++)) test_notification || ((failed++)) else log_warn "Skipping integration tests - no cluster connection" diff --git a/tests/integration/test_stac.py b/tests/integration/test_stac.py index 40bd74a8..236ec16a 100644 --- a/tests/integration/test_stac.py +++ b/tests/integration/test_stac.py @@ -136,19 +136,19 @@ def test_stac_queryables(stac_endpoint: str) -> None: queryables_file = Path(__file__).parent.parent.parent / "charts" / "eoapi" / "initdb-data" / "queryables" / "test-queryables.json" with open(queryables_file) as f: expected_queryables = json.load(f) - + # Get queryables from API resp = client.get(f"{stac_endpoint}/queryables") assert resp.status_code == 200 actual_queryables = resp.json() - + # Verify the queryables match the expected schema assert actual_queryables["$schema"] == expected_queryables["$schema"] assert actual_queryables["$id"] == expected_queryables["$id"] assert actual_queryables["title"] == expected_queryables["title"] assert actual_queryables["description"] == expected_queryables["description"] assert actual_queryables["type"] == expected_queryables["type"] - + # Verify all expected properties are present for prop_name, prop_schema in expected_queryables["properties"].items(): assert prop_name in actual_queryables["properties"], ( @@ -157,6 +157,6 @@ def test_stac_queryables(stac_endpoint: str) -> None: assert actual_queryables["properties"][prop_name] == prop_schema, ( f"Property '{prop_name}' schema doesn't match expected schema" ) - + # Verify additionalProperties setting assert actual_queryables.get("additionalProperties") == expected_queryables.get("additionalProperties") From 072f9c6e105c99e45630d1b04d3b3c9e2107bade Mon Sep 17 00:00:00 2001 From: Emmanuel Mathot Date: Wed, 19 Nov 2025 16:34:59 +0100 Subject: [PATCH 05/10] feat: Add queryables configuration section for STAC API search --- docs/configuration.md | 85 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 32de9912..e6ed87b3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -137,6 +137,91 @@ pgstacBootstrap: context_stats_ttl: "12 hours" ``` +### Queryables Configuration + +Configure custom queryables for STAC API search using `pypgstac load-queryables`. Queryables can be loaded from files in the chart or from external ConfigMaps. + +#### Basic Configuration + +Each queryable requires a `name` field and either a `file` (from chart) or `configMapRef` (external ConfigMap): + +```yaml +pgstacBootstrap: + settings: + queryables: + # File-based queryable from chart + - name: "common-queryables.json" + file: "initdb-data/queryables/test-queryables.json" + + # External ConfigMap reference + - name: "custom-queryables.json" + configMapRef: + name: my-custom-queryables-configmap + key: queryables.json +``` + +#### Configuration Parameters + +| **Parameter** | **Description** | **Required** | **Example** | +|:--------------|:----------------|:-------------|:------------| +| `name` | Name for the queryables file | Yes | "common-queryables.json" | +| `file` | Path to queryables file in chart | No* | "initdb-data/queryables/test-queryables.json" | +| `configMapRef.name` | Name of external ConfigMap | No* | "my-queryables-cm" | +| `configMapRef.key` | Key in the ConfigMap | No* | "queryables.json" | +| `indexFields` | Fields to create indexes for | No | ["platform", "instruments"] | +| `deleteMissing` | Delete queryables not in this file | No | true | +| `collections` | Apply to specific collections | No | ["collection-1", "collection-2"] | + +\* Either `file` or `configMapRef` must be provided + +#### Advanced Example + +Mix file-based and ConfigMap-based queryables with optional parameters: + +```yaml +pgstacBootstrap: + settings: + queryables: + # Standard queryables from chart with indexes + - name: "common-queryables.json" + file: "initdb-data/queryables/common-queryables.json" + indexFields: ["platform", "instruments"] + deleteMissing: true + + # Custom queryables from external ConfigMap + - name: "sentinel-queryables.json" + configMapRef: + name: sentinel-queryables-cm + key: queryables.json + indexFields: ["sat:orbit_state", "sar:instrument_mode"] + collections: ["sentinel-1-grd"] + + # Collection-specific queryables + - name: "landsat-queryables.json" + configMapRef: + name: landsat-queryables-cm + key: data.json + collections: ["landsat-c2-l2"] +``` + +#### External ConfigMap Setup + +When using `configMapRef`, create the ConfigMap separately: + +```bash +# From a file +kubectl create configmap my-queryables-cm \ + --from-file=queryables.json=./my-queryables.json \ + -n eoapi + +# From literal JSON +kubectl create configmap my-queryables-cm \ + --from-literal=queryables.json='{"$schema": "...", "properties": {...}}' \ + -n eoapi +``` + +The queryables will be automatically loaded during the PgSTAC bootstrap process. + ## Ingress Configuration Unified ingress configuration supporting both NGINX and Traefik: From ce31a2d434b7aae60582ea6777bf54dc8d022ed8 Mon Sep 17 00:00:00 2001 From: Emmanuel Mathot Date: Wed, 19 Nov 2025 16:55:05 +0100 Subject: [PATCH 06/10] feat: Update queryables configuration to include name field for testing --- charts/eoapi/profiles/experimental.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/eoapi/profiles/experimental.yaml b/charts/eoapi/profiles/experimental.yaml index 245b6809..47212f24 100644 --- a/charts/eoapi/profiles/experimental.yaml +++ b/charts/eoapi/profiles/experimental.yaml @@ -68,7 +68,8 @@ pgstacBootstrap: # Queryables configuration for testing queryables: - - file: "initdb-data/queryables/test-queryables.json" + - name: test-queryables.json + file: "initdb-data/queryables/test-queryables.json" indexFields: ["platform", "instruments"] deleteMissing: true From 315145457b099c3516f561fe5d1f7ec339b88b06 Mon Sep 17 00:00:00 2001 From: Emmanuel Mathot Date: Wed, 19 Nov 2025 16:57:06 +0100 Subject: [PATCH 07/10] fix: Remove trailing whitespace in queryables configuration --- docs/configuration.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index e6ed87b3..2c587222 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -152,7 +152,7 @@ pgstacBootstrap: # File-based queryable from chart - name: "common-queryables.json" file: "initdb-data/queryables/test-queryables.json" - + # External ConfigMap reference - name: "custom-queryables.json" configMapRef: @@ -187,7 +187,7 @@ pgstacBootstrap: file: "initdb-data/queryables/common-queryables.json" indexFields: ["platform", "instruments"] deleteMissing: true - + # Custom queryables from external ConfigMap - name: "sentinel-queryables.json" configMapRef: @@ -195,7 +195,7 @@ pgstacBootstrap: key: queryables.json indexFields: ["sat:orbit_state", "sar:instrument_mode"] collections: ["sentinel-1-grd"] - + # Collection-specific queryables - name: "landsat-queryables.json" configMapRef: From 0341ddeaf7f5b6a7c07b8e6530e9a86e44aa9800 Mon Sep 17 00:00:00 2001 From: Emmanuel Mathot Date: Wed, 19 Nov 2025 17:08:54 +0100 Subject: [PATCH 08/10] fix: Correct path to queryables test file in test_stac_queryables function --- tests/integration/test_stac.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_stac.py b/tests/integration/test_stac.py index 236ec16a..832dd720 100644 --- a/tests/integration/test_stac.py +++ b/tests/integration/test_stac.py @@ -133,7 +133,7 @@ def test_stac_custom_path(stac_endpoint: str) -> None: def test_stac_queryables(stac_endpoint: str) -> None: """test queryables endpoint returns expected schema.""" # Load expected queryables from test file - queryables_file = Path(__file__).parent.parent.parent / "charts" / "eoapi" / "initdb-data" / "queryables" / "test-queryables.json" + queryables_file = Path(__file__).parent.parent.parent.parent / "charts" / "eoapi" / "initdb-data" / "queryables" / "test-queryables.json" with open(queryables_file) as f: expected_queryables = json.load(f) From cdbb8e02270691553c9b07d7c41b6b46f0884428 Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Fri, 21 Nov 2025 20:46:33 +0100 Subject: [PATCH 09/10] Fixed tests. --- .../queryables/test-queryables.json | 50 ++++++++++++------- tests/integration/test_stac.py | 14 ++++-- 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/charts/eoapi/initdb-data/queryables/test-queryables.json b/charts/eoapi/initdb-data/queryables/test-queryables.json index 2a6d10e2..e4332696 100644 --- a/charts/eoapi/initdb-data/queryables/test-queryables.json +++ b/charts/eoapi/initdb-data/queryables/test-queryables.json @@ -1,10 +1,39 @@ { - "$schema": "https://json-schema.org/draft/2019-09/schema", - "$id": "https://stac-extensions.github.io/item-search/v1.0.0/schema.json", - "title": "Test Queryables", - "description": "Test queryables for eoapi", + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://localhost/stac/queryables", + "title": "STAC Queryables.", "type": "object", "properties": { + "id": { + "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id", + "title": "Item ID", + "description": "Item identifier" + }, + "datetime": { + "type": "string", + "title": "Acquired", + "format": "date-time", + "pattern": "(\\+00:00|Z)$", + "description": "Datetime" + }, + "geometry": { + "$ref": "https://geojson.org/schema/Feature.json", + "title": "Item Geometry", + "description": "Item Geometry" + }, + "platform": { + "description": "Platform or satellite name", + "type": "string", + "title": "Platform" + }, + "instruments": { + "description": "Instrument(s) used", + "type": "array", + "title": "Instruments", + "items": { + "type": "string" + } + }, "eo:cloud_cover": { "description": "Estimate of cloud cover as a percentage (0-100) of the entire scene", "type": "number", @@ -25,19 +54,6 @@ "title": "Sun Elevation", "minimum": -90, "maximum": 90 - }, - "platform": { - "description": "Platform or satellite name", - "type": "string", - "title": "Platform" - }, - "instruments": { - "description": "Instrument(s) used", - "type": "array", - "title": "Instruments", - "items": { - "type": "string" - } } }, "additionalProperties": true diff --git a/tests/integration/test_stac.py b/tests/integration/test_stac.py index 832dd720..a0c695ad 100644 --- a/tests/integration/test_stac.py +++ b/tests/integration/test_stac.py @@ -133,7 +133,14 @@ def test_stac_custom_path(stac_endpoint: str) -> None: def test_stac_queryables(stac_endpoint: str) -> None: """test queryables endpoint returns expected schema.""" # Load expected queryables from test file - queryables_file = Path(__file__).parent.parent.parent.parent / "charts" / "eoapi" / "initdb-data" / "queryables" / "test-queryables.json" + queryables_file = ( + Path(__file__).parent.parent.parent + / "charts" + / "eoapi" + / "initdb-data" + / "queryables" + / "test-queryables.json" + ) with open(queryables_file) as f: expected_queryables = json.load(f) @@ -146,7 +153,6 @@ def test_stac_queryables(stac_endpoint: str) -> None: assert actual_queryables["$schema"] == expected_queryables["$schema"] assert actual_queryables["$id"] == expected_queryables["$id"] assert actual_queryables["title"] == expected_queryables["title"] - assert actual_queryables["description"] == expected_queryables["description"] assert actual_queryables["type"] == expected_queryables["type"] # Verify all expected properties are present @@ -159,4 +165,6 @@ def test_stac_queryables(stac_endpoint: str) -> None: ) # Verify additionalProperties setting - assert actual_queryables.get("additionalProperties") == expected_queryables.get("additionalProperties") + assert actual_queryables.get( + "additionalProperties" + ) == expected_queryables.get("additionalProperties") From d55db4e231ecb7a31fe4c105bf0633d485385d89 Mon Sep 17 00:00:00 2001 From: Emmanuel Mathot Date: Sun, 23 Nov 2025 22:48:20 +0100 Subject: [PATCH 10/10] fix: Remove redundant assertions in test_stac_queryables function --- tests/integration/test_stac.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/test_stac.py b/tests/integration/test_stac.py index a0c695ad..5d17ca03 100644 --- a/tests/integration/test_stac.py +++ b/tests/integration/test_stac.py @@ -151,8 +151,6 @@ def test_stac_queryables(stac_endpoint: str) -> None: # Verify the queryables match the expected schema assert actual_queryables["$schema"] == expected_queryables["$schema"] - assert actual_queryables["$id"] == expected_queryables["$id"] - assert actual_queryables["title"] == expected_queryables["title"] assert actual_queryables["type"] == expected_queryables["type"] # Verify all expected properties are present