diff --git a/.github/workflows/ci.yml b/.github/workflows/helm-tests.yml similarity index 97% rename from .github/workflows/ci.yml rename to .github/workflows/helm-tests.yml index 02264b1c..65f579c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/helm-tests.yml @@ -4,7 +4,7 @@ name: CI on: push: branches: - - main + - '*' tags: - '*' pull_request: diff --git a/helm-chart/eoapi/.gitignore b/helm-chart/eoapi/.gitignore new file mode 100644 index 00000000..c0363794 --- /dev/null +++ b/helm-chart/eoapi/.gitignore @@ -0,0 +1 @@ +tmp/ \ No newline at end of file diff --git a/helm-chart/eoapi/helm-template.sh b/helm-chart/eoapi/helm-template.sh old mode 100644 new mode 100755 index 46990d1b..d499bf26 --- a/helm-chart/eoapi/helm-template.sh +++ b/helm-chart/eoapi/helm-template.sh @@ -5,6 +5,7 @@ export PGPASSWORD=password export POSTGRES_PASSWORD=password export GITSHA=$(git rev-parse HEAD | cut -c1-10) +mkdir -p tmp MANIFEST=./tmp/eoapi-manfests.yaml echo "" > $MANIFEST diff --git a/.github/workflows/data/my_data.sql b/helm-chart/eoapi/initdb-data/my_data.sql similarity index 100% rename from .github/workflows/data/my_data.sql rename to helm-chart/eoapi/initdb-data/my_data.sql diff --git a/.github/workflows/data/noaa-emergency-response.json b/helm-chart/eoapi/initdb-data/noaa-emergency-response.json similarity index 100% rename from .github/workflows/data/noaa-emergency-response.json rename to helm-chart/eoapi/initdb-data/noaa-emergency-response.json diff --git a/.github/workflows/data/noaa-eri-nashville2020.json b/helm-chart/eoapi/initdb-data/noaa-eri-nashville2020.json similarity index 100% rename from .github/workflows/data/noaa-eri-nashville2020.json rename to helm-chart/eoapi/initdb-data/noaa-eri-nashville2020.json diff --git a/helm-chart/eoapi/templates/db/configmap.yaml b/helm-chart/eoapi/templates/db/configmap.yaml new file mode 100644 index 00000000..d7073f1e --- /dev/null +++ b/helm-chart/eoapi/templates/db/configmap.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: initdb-sql-config +data: + initdb.sql: | + {{- range $path, $bytes := $.Files.Glob "initdb-data/*.sql" -}} + {{ $.Files.Get $path | nindent 4 }} + {{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: initdb-json-config +data: + {{- range $path, $bytes := $.Files.Glob "initdb-data/*.json" -}} + {{- base $path | nindent 2 -}}: | {{- $.Files.Get $path | nindent 4 -}} + {{- end -}} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: initdb-sh-config +data: + load.sh: | + pypgstac pgready --dsn postgresql://username:password@pgstac/postgis + pypgstac load items /opt/initdb/json-data/noaa-eri-nashville2020.json --dsn postgresql://username:password@pgstac/postgis --method insert_ignore + pypgstac load collections /opt/initdb/json-data/noaa-emergency-response.json --dsn postgresql://username:password@pgstac/postgis --method insert_ignore + psql postgresql://username:password@pgstac/postgis -f /opt/initdb/sql-data/initdb.sql + # run it forever like a docker process should + tail -f /dev/null diff --git a/helm-chart/eoapi/templates/db/deployment.yaml b/helm-chart/eoapi/templates/db/deployment.yaml index 10970419..451cb449 100644 --- a/helm-chart/eoapi/templates/db/deployment.yaml +++ b/helm-chart/eoapi/templates/db/deployment.yaml @@ -25,9 +25,6 @@ spec: - "-N" - "500" envFrom: - # NOTE: there's no reason we need to use a `ConfigMap` or `Secret` here to get os env vars into the pod. - # we could just template them out here immediately with `value: $_` but this allows us - # to store them in k8s intermediately and change them and then bounce deploys if needed - secretRef: name: pgstac-secrets ports: @@ -35,7 +32,38 @@ spec: volumeMounts: - mountPath: /var/lib/postgresql/data name: pgstac-claim + {{- if .Values.db.enable_data_fixtures }} + - name: loader + image: ghcr.io/stac-utils/pgstac:v0.7.1 + command: + - "sh" + args: + - "/opt/initdb/load.sh" + envFrom: + - secretRef: + name: pgstac-secrets + ports: + - containerPort: 6543 + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: pgstac-claim + - mountPath: /opt/initdb/sql-data + name: initdb-sql-volume + - mountPath: /opt/initdb/json-data + name: initdb-json-volume + - mountPath: /opt/initdb/ + name: initdb-sh-volume + {{- end }} volumes: - name: pgstac-claim persistentVolumeClaim: claimName: pgstac-claim + - name: initdb-sql-volume + configMap: + name: initdb-sql-config + - name: initdb-json-volume + configMap: + name: initdb-json-config + - name: initdb-sh-volume + configMap: + name: initdb-sh-config diff --git a/helm-chart/eoapi/templates/db/secrets.yaml b/helm-chart/eoapi/templates/db/secrets.yaml index b53d9e94..347c5f91 100644 --- a/helm-chart/eoapi/templates/db/secrets.yaml +++ b/helm-chart/eoapi/templates/db/secrets.yaml @@ -8,7 +8,7 @@ stringData: {{- range $envKey, $envValue := .Values.db.settings.secrets }} {{ upper $envKey }}: "{{ $envValue }}" {{- /* stac-utils seems to require different environment variable for postgres so handle here via if/else to - avoid having to pass more arg secrets passed */ -}} + avoid having to pass more arg secrets */ -}} {{- if eq $envKey "PGPASSWORD" }} POSTGRES_PASS: "{{ $envValue }}" {{- end }} diff --git a/helm-chart/eoapi/tests/db_deploy_tests.yaml b/helm-chart/eoapi/tests/db_deploy_tests.yaml new file mode 100644 index 00000000..7578e0ec --- /dev/null +++ b/helm-chart/eoapi/tests/db_deploy_tests.yaml @@ -0,0 +1,41 @@ +suite: db defaults deployment +templates: + - templates/db/deployment.yaml +tests: + - it: "db deploy defaults" + set: + db.enabled: true + db.enable_data_fixtures: false + asserts: + - isKind: + of: Deployment + - matchRegex: + path: metadata.name + pattern: ^pgstac$ + - equal: + path: spec.strategy.type + value: "Recreate" + - equal: + path: spec.template.spec.containers[0].name + value: "pgstac" + - isNull: + path: spec.template.spec.containers[1].name + - it: "db deploy with fixtures" + set: + db.enabled: true + db.enable_data_fixtures: true + asserts: + - isKind: + of: Deployment + - matchRegex: + path: metadata.name + pattern: ^pgstac$ + - equal: + path: spec.strategy.type + value: "Recreate" + - equal: + path: spec.template.spec.containers[0].name + value: "pgstac" + - equal: + path: spec.template.spec.containers[1].name + value: "loader" \ No newline at end of file diff --git a/helm-chart/eoapi/tests/deploy_tests.yaml b/helm-chart/eoapi/tests/deploy_tests.yaml index fa431ba0..90d23237 100644 --- a/helm-chart/eoapi/tests/deploy_tests.yaml +++ b/helm-chart/eoapi/tests/deploy_tests.yaml @@ -7,7 +7,6 @@ tests: raster.enabled: false stac.enabled: false vector.enabled: true - gitSha: "ABC123" asserts: - isKind: of: Deployment @@ -37,7 +36,6 @@ tests: raster.enabled: true stac.enabled: false vector.enabled: false - gitSha: "ABC123" asserts: - isKind: of: Deployment @@ -67,7 +65,6 @@ tests: raster.enabled: false stac.enabled: true vector.enabled: false - gitSha: "ABC123" asserts: - isKind: of: Deployment diff --git a/helm-chart/eoapi/tmp/eoapi-manfests.yaml b/helm-chart/eoapi/tmp/eoapi-manfests.yaml deleted file mode 100644 index 178e3fbf..00000000 --- a/helm-chart/eoapi/tmp/eoapi-manfests.yaml +++ /dev/null @@ -1,383 +0,0 @@ - ---- -# Source: eoapi/templates/db/manifest.yaml -apiVersion: v1 -kind: Secret -metadata: - name: pgstac-secrets -type: "Opaque" -stringData: - DB_MAX_CONN_SIZE: "10" - DB_MIN_CONN_SIZE: "1" - PGDATA: "/var/lib/postgresql/data/pgdata" - PGDATABASE: "postgis" - POSTGRES_DBNAME: "postgis" - PGPASSWORD: "password" - POSTGRES_PASS: "password" - PGUSER: "username" - POSTGRES_DB: "postgis" - POSTGRES_DBNAME: "postgis" - POSTGRES_HOST: "pgstac" - POSTGRES_HOST_READER: "pgstac" - POSTGRES_HOST_WRITER: "pgstac" - POSTGRES_PASS: "password" - POSTGRES_PASSWORD: "password" - POSTGRES_PORT: "5432" - POSTGRES_USER: "username" ---- -# Source: eoapi/templates/db/manifest.yaml -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: pgstac-claim -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Mi ---- -# Source: eoapi/templates/db/manifest.yaml -apiVersion: v1 -kind: Service -metadata: - labels: - app: pgstac - name: pgstac -spec: - ports: - - name: "5432" - port: 5432 - targetPort: 5432 - selector: - app: pgstac ---- -# Source: eoapi/templates/db/manifest.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: pgstac - labels: - app: pgstac -spec: - selector: - matchLabels: - app: pgstac - strategy: - type: Recreate - template: - metadata: - labels: - app: pgstac - spec: - restartPolicy: Always - containers: - - name: pgstac - image: ghcr.io/stac-utils/pgstac:v0.7.1 - args: - - "postgres" - - "-N" - - "500" - envFrom: - # NOTE: there's no reason we need to use a `ConfigMap` or `Secret` here to get os env vars into the pod. - # we could just template them out here immediately with `value: $_` but this allows us - # to store them in k8s intermediately and change them and then bounce deploys if needed - - secretRef: - name: pgstac-secrets - ports: - - containerPort: 5432 - volumeMounts: - - mountPath: /var/lib/postgresql/data - name: pgstac-claim - volumes: - - name: pgstac-claim - persistentVolumeClaim: - claimName: pgstac-claim ---- -# Source: eoapi/templates/services/manifest.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: raster-envvar-configmap -data: - CPL_VSIL_CURL_ALLOWED_EXTENSIONS: ".tif,.TIF,.tiff" - GDAL_CACHEMAX: "200" - GDAL_DISABLE_READDIR_ON_OPEN: "EMPTY_DIR" - GDAL_HTTP_MERGE_CONSECUTIVE_RANGES: "YES" - GDAL_HTTP_MULTIPLEX: "YES" - GDAL_HTTP_VERSION: "2" - GDAL_INGESTED_BYTES_AT_OPEN: "32768" - HOST: "0.0.0.0" - PORT: "8080" - PYTHONWARNINGS: "ignore" - VSI_CACHE: "TRUE" - VSI_CACHE_SIZE: "5000000" - WEB_CONCURRENCY: "10" ---- -# Source: eoapi/templates/services/manifest.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: stac-envvar-configmap -data: - HOST: "0.0.0.0" - PORT: "8080" - TITILER_ENDPOINT: "raster:8080" - WEB_CONCURRENCY: "10" ---- -# Source: eoapi/templates/services/manifest.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: vector-envvar-configmap -data: - HOST: "0.0.0.0" - PORT: "8080" - TIPG_CATALOG_TTL: "0" - WEB_CONCURRENCY: "10" ---- -# Source: eoapi/templates/services/manifest.yaml -apiVersion: v1 -kind: Service -metadata: - labels: - app: raster - name: raster -spec: - type: NodePort - ports: - - name: '8080' - port: 8080 - targetPort: 8080 - selector: - app: raster ---- -# Source: eoapi/templates/services/manifest.yaml -apiVersion: v1 -kind: Service -metadata: - labels: - app: stac - name: stac -spec: - type: NodePort - ports: - - name: '8080' - port: 8080 - targetPort: 8080 - selector: - app: stac ---- -# Source: eoapi/templates/services/manifest.yaml -apiVersion: v1 -kind: Service -metadata: - labels: - app: vector - name: vector -spec: - type: NodePort - ports: - - name: '8080' - port: 8080 - targetPort: 8080 - selector: - app: vector ---- -# Source: eoapi/templates/services/manifest.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: raster - gitsha: a5ba4b476b - name: raster -spec: - progressDeadlineSeconds: 600 - revisionHistoryLimit: 5 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 50% - maxUnavailable: 0 - selector: - matchLabels: - app: raster - template: - metadata: - labels: - app: raster - spec: - containers: - - image: ghcr.io/developmentseed/eoapi-raster:latest - name: raster - ports: - - containerPort: 8080 - resources: - limits: - cpu: 512m - memory: 4096M - requests: - cpu: 256m - memory: 3072M - envFrom: - # NOTE: there's no reason we need to use a `ConfigMap` or `Secret` here to get os env vars into the pod. - # we could just template them out here immediately with `value: $_` but this allows us - # to store them in k8s intermediately and change them and then bounce deploys if needed - - secretRef: - name: pgstac-secrets - - configMapRef: - name: raster-envvar-configmap ---- -# Source: eoapi/templates/services/manifest.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: stac - gitsha: a5ba4b476b - name: stac -spec: - progressDeadlineSeconds: 600 - revisionHistoryLimit: 5 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 50% - maxUnavailable: 0 - selector: - matchLabels: - app: stac - template: - metadata: - labels: - app: stac - spec: - containers: - - image: ghcr.io/developmentseed/eoapi-stac:latest - name: stac - ports: - - containerPort: 8080 - resources: - limits: - cpu: 512m - memory: 1024Mi - requests: - cpu: 256m - memory: 256Mi - envFrom: - # NOTE: there's no reason we need to use a `ConfigMap` or `Secret` here to get os env vars into the pod. - # we could just template them out here immediately with `value: $_` but this allows us - # to store them in k8s intermediately and change them and then bounce deploys if needed - - secretRef: - name: pgstac-secrets - - configMapRef: - name: stac-envvar-configmap ---- -# Source: eoapi/templates/services/manifest.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: vector - gitsha: a5ba4b476b - name: vector -spec: - progressDeadlineSeconds: 600 - revisionHistoryLimit: 5 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 50% - maxUnavailable: 0 - selector: - matchLabels: - app: vector - template: - metadata: - labels: - app: vector - spec: - containers: - - image: ghcr.io/developmentseed/eoapi-vector:latest - name: vector - ports: - - containerPort: 8080 - resources: - limits: - cpu: 512m - memory: 1024Mi - requests: - cpu: 256m - memory: 256Mi - envFrom: - # NOTE: there's no reason we need to use a `ConfigMap` or `Secret` here to get os env vars into the pod. - # we could just template them out here immediately with `value: $_` but this allows us - # to store them in k8s intermediately and change them and then bounce deploys if needed - - secretRef: - name: pgstac-secrets - - configMapRef: - name: vector-envvar-configmap ---- -# Source: eoapi/templates/services/manifest.yaml -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: raster - annotations: - alb.ingress.kubernetes.io/scheme: internet-facing - alb.ingress.kubernetes.io/target-type: instance -spec: - ingressClassName: alb - rules: - - http: - paths: - - pathType: Prefix - path: "/" - backend: - service: - name: raster - port: - number: 8080 ---- -# Source: eoapi/templates/services/manifest.yaml -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: stac - annotations: - alb.ingress.kubernetes.io/scheme: internet-facing - alb.ingress.kubernetes.io/target-type: instance -spec: - ingressClassName: alb - rules: - - http: - paths: - - pathType: Prefix - path: "/" - backend: - service: - name: stac - port: - number: 8080 ---- -# Source: eoapi/templates/services/manifest.yaml -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: vector - annotations: - alb.ingress.kubernetes.io/scheme: internet-facing - alb.ingress.kubernetes.io/target-type: instance -spec: - ingressClassName: alb - rules: - - http: - paths: - - pathType: Prefix - path: "/" - backend: - service: - name: vector - port: - number: 8080 diff --git a/helm-chart/eoapi/values.yaml b/helm-chart/eoapi/values.yaml index c676b9a8..f896a4cb 100644 --- a/helm-chart/eoapi/values.yaml +++ b/helm-chart/eoapi/values.yaml @@ -34,6 +34,8 @@ db: # environment options: "rds" || "k8s" environment: "k8s" enabled: true + # toggle to true if you want the db test fixtures loaded + enable_data_fixtures: false settings: resources: requests: @@ -79,7 +81,6 @@ raster: image: name: ghcr.io/developmentseed/eoapi-raster tag: latest - # nothing definitive about this, just plumbed in after quick survey of `eoAPI/infrastructure/aws/cdk/config.py` settings: timeout: 10 # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ @@ -117,7 +118,6 @@ stac: image: name: ghcr.io/developmentseed/eoapi-stac tag: latest - # nothing definitive about this, just plumbed in after quick survey of `eoAPI/infrastructure/aws/cdk/config.py` settings: timeout: 10 # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ @@ -147,7 +147,6 @@ vector: image: name: ghcr.io/developmentseed/eoapi-vector tag: latest - # nothing definitive about this, just plumbed in after quick survey of `eoAPI/infrastructure/aws/cdk/config.py` settings: timeout: 10 # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/