From 70b7a38ce930f39737974b49c93c43be00e1f692 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Fri, 5 Jul 2024 09:39:59 -0700 Subject: [PATCH 1/2] add back temp db option --- helm-chart/eoapi/CHANGELOG.md | 4 + helm-chart/eoapi/Chart.yaml | 5 +- helm-chart/eoapi/templates/_helpers.tpl | 26 +++++++ helm-chart/eoapi/templates/db/configmap.yaml | 39 ++++++++++ helm-chart/eoapi/templates/db/deployment.yaml | 76 +++++++++++++++++++ helm-chart/eoapi/templates/db/pvc.yaml | 14 ++++ helm-chart/eoapi/templates/db/secrets.yaml | 20 +++++ helm-chart/eoapi/templates/db/service.yaml | 16 ++++ .../templates/pgstacboostrap/configmap.yaml | 2 + .../eoapi/templates/services/deployment.yaml | 7 ++ helm-chart/eoapi/values.yaml | 44 +++++++++++ 11 files changed, 251 insertions(+), 2 deletions(-) create mode 100644 helm-chart/eoapi/templates/db/configmap.yaml create mode 100644 helm-chart/eoapi/templates/db/deployment.yaml create mode 100644 helm-chart/eoapi/templates/db/pvc.yaml create mode 100644 helm-chart/eoapi/templates/db/secrets.yaml create mode 100644 helm-chart/eoapi/templates/db/service.yaml diff --git a/helm-chart/eoapi/CHANGELOG.md b/helm-chart/eoapi/CHANGELOG.md index e0c5bdbc..da3c941a 100644 --- a/helm-chart/eoapi/CHANGELOG.md +++ b/helm-chart/eoapi/CHANGELOG.md @@ -1,5 +1,9 @@ version numbers below correspond to helm chart `appVersion`: see `./helm-chart/eoapi/Chart.yaml` --- +### 0.3.4 (2024-07-05) + +* add back in a hidden non-pgo option for EOEPCA+ + ### 0.3.3 (2024-06-26) * add k3 integration tests diff --git a/helm-chart/eoapi/Chart.yaml b/helm-chart/eoapi/Chart.yaml index 86cead68..89e119af 100644 --- a/helm-chart/eoapi/Chart.yaml +++ b/helm-chart/eoapi/Chart.yaml @@ -17,15 +17,16 @@ kubeVersion: ">=1.23.0-0" # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.3.1" +version: "0.3.2" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.3.3" +appVersion: "0.3.4" dependencies: - name: postgrescluster version: 5.5.2 respository: "file://charts/postgrescluster" + condition: postgrescluster.enabled diff --git a/helm-chart/eoapi/templates/_helpers.tpl b/helm-chart/eoapi/templates/_helpers.tpl index 0ef2452c..ac712858 100644 --- a/helm-chart/eoapi/templates/_helpers.tpl +++ b/helm-chart/eoapi/templates/_helpers.tpl @@ -61,6 +61,18 @@ Create the name of the service account to use {{- end }} {{- end }} +{{/* +Create pgstac host string depending if .Values.testing +*/}} +{{- define "eoapi.pgstacTempDbHostName" -}} +{{- if .Values.testing }} +{{- printf "%s-%s" "pgstac" .Release.Name }} +{{- else }} +{{/* need to match what is default in values.yamls */}} +{{- printf "%s" "pgstac" }} +{{- end }} +{{- end }} + {{/* Create pgstac host string depending if .Values.testing */}} @@ -191,4 +203,18 @@ that you can only use traefik as ingress when `testing=true` {{- fail "you cannot use traefik yet outside of testing" -}} {{- end -}} +{{- end -}} + +{{/* +validate: +that you cannot have db.enabled and (postgrescluster.enabled or pgstacBootstrap.enabled) +*/}} +{{- define "eoapi.validateTempDB" -}} +{{- if and (.Values.db.enabled) (.Values.postgrescluster.enabled) -}} + {{- fail "you cannot use have both db.enabled and postgresclsuter.enabled" -}} +{{- end -}} +{{- if and (.Values.db.enabled) (.Values.pgstacBootstrap.enabled) -}} + {{- fail "you cannot use have both db.enabled and pgstacBootstrap.enabled" -}} +{{- end -}} + {{- end -}} \ No newline at end of file diff --git a/helm-chart/eoapi/templates/db/configmap.yaml b/helm-chart/eoapi/templates/db/configmap.yaml new file mode 100644 index 00000000..568a54ac --- /dev/null +++ b/helm-chart/eoapi/templates/db/configmap.yaml @@ -0,0 +1,39 @@ +{{- if .Values.db.enabled }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: initdb-sql-config-{{ $.Release.Name }} +data: + initdb.sql: | + {{- range $path, $bytes := $.Files.Glob "initdb-data/*.sql" -}} + {{ $.Files.Get $path | nindent 4 }} + {{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: initdb-json-config-{{ $.Release.Name }} +data: + {{- range $path, $bytes := $.Files.Glob "initdb-data/*.json" -}} + {{- base $path | nindent 2 -}}: | {{- $.Files.Get $path | nindent 4 -}} + {{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: initdb-sh-config-{{ $.Release.Name }} +data: + load.sh: | + #!/bin/bash + apt update -y && apt install python3 python3-pip -y + pip install pypgstac[psycopg] + DSN="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" + pypgstac pgready --dsn $DSN + pypgstac load collections /opt/initdb/json-data/noaa-emergency-response.json --dsn $DSN --method insert_ignore + pypgstac load items /opt/initdb/json-data/noaa-eri-nashville2020.json --dsn $DSN --method insert_ignore + psql $DSN -f /opt/initdb/sql-data/initdb.sql + echo "DONE LOADING!!!!!!" + # run it forever like a docker process should + tail -f /dev/null +{{- end }} \ No newline at end of file diff --git a/helm-chart/eoapi/templates/db/deployment.yaml b/helm-chart/eoapi/templates/db/deployment.yaml new file mode 100644 index 00000000..788e0922 --- /dev/null +++ b/helm-chart/eoapi/templates/db/deployment.yaml @@ -0,0 +1,76 @@ +{{- if .Values.db.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "eoapi.pgstacTempDbHostName" . | nindent 8 }} + labels: + app: {{ include "eoapi.pgstacTempDbHostName" . | nindent 10 }} +spec: + selector: + matchLabels: + app: {{ include "eoapi.pgstacTempDbHostName" . | nindent 12 }} + strategy: + type: Recreate + template: + metadata: + labels: + app: {{ include "eoapi.pgstacTempDbHostName" . | nindent 14 }} + spec: + restartPolicy: Always + containers: + - name: pgstac + image: {{ .Values.db.image.name }}:{{ .Values.db.image.tag }} + args: + {{- toYaml .Values.db.command | nindent 12 }} + envFrom: + - secretRef: + name: pgstac-secrets-{{ $.Release.Name }} + ports: + - containerPort: 5432 + resources: + limits: + cpu: {{ .Values.db.settings.resources.limits.cpu }} + memory: {{ .Values.db.settings.resources.limits.memory }} + requests: + cpu: {{ .Values.db.settings.resources.requests.cpu }} + memory: {{ .Values.db.settings.resources.requests.memory }} + volumeMounts: + - mountPath: /data + name: pgstac-claim-{{ $.Release.Name }} + {{- if .Values.db.enable_data_fixtures }} + - name: loader + image: {{ .Values.db.image.name }}:{{ .Values.db.image.tag }} + command: + - "sh" + args: + - "/opt/initdb/load.sh" + envFrom: + - secretRef: + name: pgstac-secrets-{{ $.Release.Name }} + ports: + - containerPort: 6543 + volumeMounts: + - mountPath: /data + name: pgstac-claim-{{ $.Release.Name }} + - mountPath: /opt/initdb/sql-data + name: initdb-sql-volume-{{ $.Release.Name }} + - mountPath: /opt/initdb/json-data + name: initdb-json-volume-{{ $.Release.Name }} + - mountPath: /opt/initdb/ + name: initdb-sh-volume-{{ $.Release.Name }} + {{- end }} + volumes: + - name: pgstac-claim-{{ $.Release.Name }} + persistentVolumeClaim: + claimName: pgstac-claim-{{ $.Release.Name }} + - name: initdb-sql-volume-{{ $.Release.Name }} + configMap: + name: initdb-sql-config-{{ $.Release.Name }} + - name: initdb-json-volume-{{ $.Release.Name }} + configMap: + name: initdb-json-config-{{ $.Release.Name }} + - name: initdb-sh-volume-{{ $.Release.Name }} + configMap: + name: initdb-sh-config-{{ $.Release.Name }} +{{- end }} \ No newline at end of file diff --git a/helm-chart/eoapi/templates/db/pvc.yaml b/helm-chart/eoapi/templates/db/pvc.yaml new file mode 100644 index 00000000..aaf14091 --- /dev/null +++ b/helm-chart/eoapi/templates/db/pvc.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pgstac-claim-{{ $.Release.Name }} +spec: + storageClassName: {{ .Values.db.storageClassName }} +{{- if .Values.db.accessModes }} + accessModes: +{{ toYaml .Values.db.accessModes | indent 4 }} +{{- end }} + resources: + requests: + storage: {{ .Values.db.settings.resources.requests.storage }} \ No newline at end of file diff --git a/helm-chart/eoapi/templates/db/secrets.yaml b/helm-chart/eoapi/templates/db/secrets.yaml new file mode 100644 index 00000000..fc48a9a1 --- /dev/null +++ b/helm-chart/eoapi/templates/db/secrets.yaml @@ -0,0 +1,20 @@ +{{- if .Values.db.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: pgstac-secrets-{{ $.Release.Name }} +type: "Opaque" +stringData: + {{- range $envKey, $envValue := .Values.db.settings.secrets }} + {{ upper $envKey }}: "{{ $envValue }}" + {{- /* stac-utils seems to require different environment variable for postgres so handle here via if/else to + avoid having to pass more arg secrets */ -}} + {{- if eq $envKey "PGPASSWORD" }} + POSTGRES_PASS: "{{ $envValue }}" + {{- end }} + {{- if eq $envKey "PGDATABASE" }} + POSTGRES_DBNAME: "{{ $envValue }}" + {{- end }} + {{- end }} +{{- end }} diff --git a/helm-chart/eoapi/templates/db/service.yaml b/helm-chart/eoapi/templates/db/service.yaml new file mode 100644 index 00000000..0ff5dfe6 --- /dev/null +++ b/helm-chart/eoapi/templates/db/service.yaml @@ -0,0 +1,16 @@ +{{- if .Values.db.enabled }} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: {{ include "eoapi.pgstacTempDbHostName" . | nindent 10 }} + name: {{ include "eoapi.pgstacTempDbHostName" . | nindent 8 }} +spec: + ports: + - name: "5432" + port: 5432 + targetPort: 5432 + selector: + app: {{ include "eoapi.pgstacTempDbHostName" . | nindent 10 }} +{{- end }} \ No newline at end of file diff --git a/helm-chart/eoapi/templates/pgstacboostrap/configmap.yaml b/helm-chart/eoapi/templates/pgstacboostrap/configmap.yaml index 0225d70e..b3f9777b 100644 --- a/helm-chart/eoapi/templates/pgstacboostrap/configmap.yaml +++ b/helm-chart/eoapi/templates/pgstacboostrap/configmap.yaml @@ -1,3 +1,4 @@ +{{- if .Values.pgstacBootstrap.enabled }} --- apiVersion: v1 kind: ConfigMap @@ -62,3 +63,4 @@ data: # let the k8's pod know we've completed successfully exit 0 +{{- end }} diff --git a/helm-chart/eoapi/templates/services/deployment.yaml b/helm-chart/eoapi/templates/services/deployment.yaml index c89ef6c1..6deba2bb 100644 --- a/helm-chart/eoapi/templates/services/deployment.yaml +++ b/helm-chart/eoapi/templates/services/deployment.yaml @@ -1,3 +1,4 @@ +{{ include "eoapi.validateTempDB" . }} {{- range $serviceName, $v := .Values -}} {{- if (or (eq $serviceName "raster") (eq $serviceName "stac") (eq $serviceName "vector")) }} {{- if index $v "enabled" }} @@ -50,14 +51,20 @@ spec: - containerPort: {{ $.Values.service.port }} resources: {{- toYaml (index $v "settings" "resources") | nindent 10 }} + {{- if $.Values.postgrescluster.enabled }} env: {{- include "eoapi.pgstacSecrets" $ | nindent 12 }} + {{- end }} envFrom: # NOTE: there's no reason we need to use a `ConfigMap` or `Secret` here to get os env vars into the pod. # we could just template them out here immediately with `value: $_` but this allows us # to store them in k8s intermediately and change them and then bounce deploys if needed - configMapRef: name: {{ $serviceName }}-envvar-configmap-{{ $.Release.Name }} + {{- if $.Values.db.enabled }} + - secretRef: + name: pgstac-secrets-{{ $.Release.Name }} + {{- end }} --- {{/* END: if index $v "enabled" */}} {{- end }} diff --git a/helm-chart/eoapi/values.yaml b/helm-chart/eoapi/values.yaml index 08052895..fa9e7155 100644 --- a/helm-chart/eoapi/values.yaml +++ b/helm-chart/eoapi/values.yaml @@ -56,8 +56,52 @@ comment_db: > The `postgrescluster` specs below are pass-through values to configure those separate charts. For more information read https://access.crunchydata.com/documentation/postgres-operator/latest +# temporary solution for EOEPCA until we figure out how to get NFS mounts +# working with PGO below. Since disabled by default most people SHOULD NOT +# use this option as it won't be talked about explicitly in the docs +db: + enabled: false + image: + name: ghcr.io/stac-utils/pgstac + tag: v0.8.2 + command: + - "postgres" + - "-N" + - "500" + # toggle to true||false if you want the db test fixtures loaded + enable_data_fixtures: true + storageClassName: "" + accessModes: + - ReadWriteOnce + settings: + resources: + requests: + storage: "100Mi" + cpu: "512m" + memory: "1024Mi" + limits: + cpu: "512m" + memory: "1024Mi" + secrets: + POSTGRES_DB: "postgis" + POSTGRES_USER: "" + POSTGRES_PASSWORD: "" + POSTGRES_PORT: "5432" + POSTGRES_HOST: "pgstac" + POSTGRES_HOST_READER: "pgstac" + POSTGRES_HOST_WRITER: "pgstac" + DB_MIN_CONN_SIZE: "1" + DB_MAX_CONN_SIZE: "15" + # default connect: https://www.postgresql.org/docs/current/libpq-envars.html + PGDATA: "/data/pgdata" + PGUSER: "" + PGPASSWORD: "" + PGDATABASE: "postgis" + + # this is declared as a dependency of eoapi in helm-chart/eoapi/Chart.yaml postgrescluster: + enabled: true postgresVersion: 16 postGISVersion: 3.4 pgBouncerReplicas: 1 From ac842f18402e28f44b3c89934b4158566e313f6a Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Fri, 5 Jul 2024 13:09:08 -0700 Subject: [PATCH 2/2] just use while --- .github/workflows/tests/test_vector.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/tests/test_vector.py b/.github/workflows/tests/test_vector.py index e2c66c50..d2e3b47b 100644 --- a/.github/workflows/tests/test_vector.py +++ b/.github/workflows/tests/test_vector.py @@ -1,5 +1,6 @@ import httpx import os +import time timeout = httpx.Timeout(15.0, connect=60.0) if bool(os.getenv("IGNORE_SSL_VERIFICATION", False)): @@ -37,6 +38,20 @@ def test_vector_api(vector_endpoint): "numberReturned", "collections", ] + + total_timeout = 60 * 5 + start_time = time.time() + while True: + if resp.json()["numberMatched"] == 7: + break + + if time.time() - start_time > total_timeout: + print("Timeout exceeded") + assert False + + time.sleep(20) + resp = client.get(f"{vector_endpoint}/collections") + assert resp.json()["numberMatched"] == 7 # one public table + 5 functions assert resp.json()["numberReturned"] == 7