add back temp db option

.github/workflows/tests/test_vector.py

@@ -1,5 +1,6 @@
 import httpx
 import os
+import time
 
 timeout = httpx.Timeout(15.0, connect=60.0)
 if bool(os.getenv("IGNORE_SSL_VERIFICATION", False)):
@@ -37,6 +38,20 @@ def test_vector_api(vector_endpoint):
         "numberReturned",
         "collections",
     ]
+
+    total_timeout = 60 * 5
+    start_time = time.time()
+    while True:
+        if resp.json()["numberMatched"] == 7:
+            break
+
+        if time.time() - start_time > total_timeout:
+            print("Timeout exceeded")
+            assert False
+
+        time.sleep(20)
+        resp = client.get(f"{vector_endpoint}/collections")
+
     assert resp.json()["numberMatched"] == 7  # one public table + 5 functions
     assert resp.json()["numberReturned"] == 7
 

helm-chart/eoapi/CHANGELOG.md

@@ -1,5 +1,9 @@
 version numbers below correspond to helm chart `appVersion`: see `./helm-chart/eoapi/Chart.yaml`
 ---
+### 0.3.4 (2024-07-05)
+
+* add back in a hidden non-pgo option for EOEPCA+ 
+
 ### 0.3.3 (2024-06-26)
 
 * add k3 integration tests

helm-chart/eoapi/Chart.yaml

@@ -17,15 +17,16 @@ kubeVersion: ">=1.23.0-0"
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: "0.3.1"
+version: "0.3.2"
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.3.3"
+appVersion: "0.3.4"
 
 dependencies:
   - name: postgrescluster
     version: 5.5.2
     respository: "file://charts/postgrescluster"
+    condition: postgrescluster.enabled

helm-chart/eoapi/templates/_helpers.tpl

@@ -61,6 +61,18 @@ Create the name of the service account to use
 {{- end }}
 {{- end }}
 
+{{/*
+Create pgstac host string depending if .Values.testing
+*/}}
+{{- define "eoapi.pgstacTempDbHostName" -}}
+{{- if .Values.testing }}
+{{- printf "%s-%s" "pgstac" .Release.Name }}
+{{- else }}
+{{/* need to match what is default in values.yamls */}}
+{{- printf "%s" "pgstac" }}
+{{- end }}
+{{- end }}
+
 {{/*
 Create pgstac host string depending if .Values.testing
 */}}
@@ -191,4 +203,18 @@ that you can only use traefik as ingress when `testing=true`
   {{- fail "you cannot use traefik yet outside of testing" -}}
 {{- end -}}
 
+{{- end -}}
+
+{{/*
+validate:
+that you cannot have db.enabled and (postgrescluster.enabled or pgstacBootstrap.enabled)
+*/}}
+{{- define "eoapi.validateTempDB" -}}
+{{- if and (.Values.db.enabled) (.Values.postgrescluster.enabled) -}}
+  {{- fail "you cannot use have both db.enabled and postgresclsuter.enabled" -}}
+{{- end -}}
+{{- if and (.Values.db.enabled) (.Values.pgstacBootstrap.enabled) -}}
+  {{- fail "you cannot use have both db.enabled and pgstacBootstrap.enabled" -}}
+{{- end -}}
+
 {{- end -}}

helm-chart/eoapi/templates/db/configmap.yaml

@@ -0,0 +1,39 @@
+{{- if .Values.db.enabled }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: initdb-sql-config-{{ $.Release.Name }}
+data:
+  initdb.sql: |
+    {{- range $path, $bytes := $.Files.Glob "initdb-data/*.sql"  -}}
+    {{ $.Files.Get $path | nindent 4 }}
+    {{- end }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: initdb-json-config-{{ $.Release.Name }}
+data:
+  {{- range $path, $bytes := $.Files.Glob "initdb-data/*.json"  -}}
+  {{- base $path | nindent 2 -}}: | {{- $.Files.Get $path | nindent 4 -}}
+  {{- end }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: initdb-sh-config-{{ $.Release.Name }}
+data:
+  load.sh: |
+    #!/bin/bash
+    apt update -y && apt install python3 python3-pip -y
+    pip install pypgstac[psycopg]
+    DSN="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB"
+    pypgstac pgready --dsn $DSN
+    pypgstac load collections /opt/initdb/json-data/noaa-emergency-response.json --dsn $DSN --method insert_ignore
+    pypgstac load items /opt/initdb/json-data/noaa-eri-nashville2020.json --dsn $DSN --method insert_ignore
+    psql $DSN -f /opt/initdb/sql-data/initdb.sql
+    echo "DONE LOADING!!!!!!"
+    # run it forever like a docker process should
+    tail -f /dev/null
+{{- end }}

helm-chart/eoapi/templates/db/deployment.yaml

@@ -0,0 +1,76 @@
+{{- if .Values.db.enabled }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "eoapi.pgstacTempDbHostName" . | nindent 8 }}
+  labels:
+    app: {{ include "eoapi.pgstacTempDbHostName" . | nindent 10 }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ include "eoapi.pgstacTempDbHostName" . | nindent 12 }}
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: {{ include "eoapi.pgstacTempDbHostName" . | nindent 14 }}
+    spec:
+      restartPolicy: Always
+      containers:
+        - name: pgstac
+          image: {{ .Values.db.image.name }}:{{ .Values.db.image.tag }}
+          args:
+            {{- toYaml .Values.db.command | nindent 12 }}
+          envFrom:
+          - secretRef:
+              name: pgstac-secrets-{{ $.Release.Name }}
+          ports:
+            - containerPort: 5432
+          resources:
+            limits:
+              cpu: {{ .Values.db.settings.resources.limits.cpu }}
+              memory: {{ .Values.db.settings.resources.limits.memory }}
+            requests:
+              cpu: {{ .Values.db.settings.resources.requests.cpu }}
+              memory: {{ .Values.db.settings.resources.requests.memory }}
+          volumeMounts:
+            - mountPath: /data
+              name: pgstac-claim-{{ $.Release.Name }}
+        {{- if .Values.db.enable_data_fixtures }}
+        - name: loader
+          image: {{ .Values.db.image.name }}:{{ .Values.db.image.tag }}
+          command:
+            - "sh"
+          args:
+            - "/opt/initdb/load.sh"
+          envFrom:
+            - secretRef:
+                name: pgstac-secrets-{{ $.Release.Name }}
+          ports:
+            - containerPort: 6543
+          volumeMounts:
+            - mountPath: /data
+              name: pgstac-claim-{{ $.Release.Name }}
+            - mountPath: /opt/initdb/sql-data
+              name: initdb-sql-volume-{{ $.Release.Name }}
+            - mountPath: /opt/initdb/json-data
+              name: initdb-json-volume-{{ $.Release.Name }}
+            - mountPath: /opt/initdb/
+              name: initdb-sh-volume-{{ $.Release.Name }}
+        {{- end }}
+      volumes:
+        - name: pgstac-claim-{{ $.Release.Name }}
+          persistentVolumeClaim:
+            claimName: pgstac-claim-{{ $.Release.Name }}
+        - name: initdb-sql-volume-{{ $.Release.Name }}
+          configMap:
+            name: initdb-sql-config-{{ $.Release.Name }}
+        - name: initdb-json-volume-{{ $.Release.Name }}
+          configMap:
+            name: initdb-json-config-{{ $.Release.Name }}
+        - name: initdb-sh-volume-{{ $.Release.Name }}
+          configMap:
+            name: initdb-sh-config-{{ $.Release.Name }}
+{{- end }}

helm-chart/eoapi/templates/db/pvc.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pgstac-claim-{{ $.Release.Name }}
+spec:
+  storageClassName: {{ .Values.db.storageClassName }}
+{{- if .Values.db.accessModes }}
+  accessModes:
+{{ toYaml .Values.db.accessModes | indent 4 }}
+{{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.db.settings.resources.requests.storage }}

helm-chart/eoapi/templates/db/secrets.yaml

@@ -0,0 +1,20 @@
+{{- if .Values.db.enabled }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: pgstac-secrets-{{ $.Release.Name }}
+type: "Opaque"
+stringData:
+  {{- range $envKey, $envValue := .Values.db.settings.secrets }}
+  {{ upper $envKey }}: "{{ $envValue }}"
+  {{- /* stac-utils seems to require different environment variable for postgres so handle here via if/else to
+      avoid having to pass more arg secrets */ -}}
+  {{- if eq $envKey "PGPASSWORD" }}
+  POSTGRES_PASS: "{{ $envValue }}"
+  {{- end }}
+  {{- if eq $envKey "PGDATABASE" }}
+  POSTGRES_DBNAME: "{{ $envValue }}"
+  {{- end }}
+  {{- end }}
+{{- end }}

helm-chart/eoapi/templates/db/service.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.db.enabled }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: {{ include "eoapi.pgstacTempDbHostName" . | nindent 10 }}
+  name: {{ include "eoapi.pgstacTempDbHostName" . | nindent 8 }}
+spec:
+  ports:
+  - name: "5432"
+    port: 5432
+    targetPort: 5432
+  selector:
+    app: {{ include "eoapi.pgstacTempDbHostName" . | nindent 10 }}
+{{- end }}

helm-chart/eoapi/templates/pgstacboostrap/configmap.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.pgstacBootstrap.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -62,3 +63,4 @@ data:
 
     # let the k8's pod know we've completed successfully
     exit 0
+{{- end }}

helm-chart/eoapi/templates/services/deployment.yaml

@@ -1,3 +1,4 @@
+{{ include "eoapi.validateTempDB" . }}
 {{- range $serviceName, $v := .Values -}}
 {{- if (or (eq $serviceName "raster") (eq $serviceName "stac") (eq $serviceName "vector")) }}
 {{- if index $v "enabled" }}
@@ -50,14 +51,20 @@ spec:
           - containerPort: {{ $.Values.service.port }}
         resources:
           {{- toYaml (index $v "settings" "resources") | nindent 10 }}
+        {{- if $.Values.postgrescluster.enabled }}
         env:
           {{- include "eoapi.pgstacSecrets" $ | nindent 12 }}
+        {{- end }}
         envFrom:
           # NOTE: there's no reason we need to use a `ConfigMap` or `Secret` here to get os env vars into the pod.
           # we could just template them out here immediately with `value: $_` but this allows us
           # to store them in k8s intermediately and change them and then bounce deploys if needed
         - configMapRef:
             name: {{ $serviceName }}-envvar-configmap-{{ $.Release.Name }}
+        {{- if $.Values.db.enabled }}
+        - secretRef:
+            name: pgstac-secrets-{{ $.Release.Name }}
+        {{- end }}
 ---
 {{/* END: if index $v "enabled" */}}
 {{- end }}

helm-chart/eoapi/values.yaml

@@ -56,8 +56,52 @@ comment_db: >
   The `postgrescluster` specs below are pass-through values to configure those separate
   charts. For more information read https://access.crunchydata.com/documentation/postgres-operator/latest
 
+# temporary solution for EOEPCA until we figure out how to get NFS mounts
+# working with PGO below. Since disabled by default most people SHOULD NOT
+# use this option as it won't be talked about explicitly in the docs
+db:
+  enabled: false
+  image:
+    name: ghcr.io/stac-utils/pgstac
+    tag: v0.8.2
+  command:
+    - "postgres"
+    - "-N"
+    - "500"
+  # toggle to true||false if you want the db test fixtures loaded
+  enable_data_fixtures: true
+  storageClassName: ""
+  accessModes:
+    - ReadWriteOnce
+  settings:
+    resources:
+      requests:
+        storage: "100Mi"
+        cpu: "512m"
+        memory: "1024Mi"
+      limits:
+        cpu: "512m"
+        memory: "1024Mi"
+    secrets:
+      POSTGRES_DB: "postgis"
+      POSTGRES_USER: ""
+      POSTGRES_PASSWORD: ""
+      POSTGRES_PORT: "5432"
+      POSTGRES_HOST: "pgstac"
+      POSTGRES_HOST_READER: "pgstac"
+      POSTGRES_HOST_WRITER: "pgstac"
+      DB_MIN_CONN_SIZE: "1"
+      DB_MAX_CONN_SIZE: "15"
+      # default connect: https://www.postgresql.org/docs/current/libpq-envars.html
+      PGDATA: "/data/pgdata"
+      PGUSER: ""
+      PGPASSWORD: ""
+      PGDATABASE: "postgis"
+
+
 # this is declared as a dependency of eoapi in helm-chart/eoapi/Chart.yaml
 postgrescluster:
+  enabled: true
   postgresVersion: 16
   postGISVersion: 3.4
   pgBouncerReplicas: 1