diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 2b2c4d3a..20045456 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -25,6 +25,9 @@ jobs: cd helm-chart helm unittest eoapi -f 'tests/*.yaml' -v eoapi/test-helm-values.yaml unit-tests: + permissions: + contents: 'read' + id-token: 'write' needs: helm-tests runs-on: ubuntu-latest steps: @@ -35,22 +38,32 @@ jobs: helm-version: v3.8.2 github-token: ${{ secrets.GITHUB_TOKEN }} - - name: start minikube - uses: medyagh/setup-minikube@master + - name: set k8s namespace name for parallel tests + run: | + random_number="${RANDOM}${RANDOM}${RANDOM}" + # Limit it to 10 digits + RANDOMNESS=$(echo $random_number | cut -c 1-10) + #echo "NS_NAME=unittest-$RANDOMNESS" >> $GITHUB_ENV + echo "NS_NAME=eoapitest" >> $GITHUB_ENV + + - id: 'auth' + uses: 'google-github-actions/auth@v1' + with: + service_account: 'k8seed-deploy@devseed-labs.iam.gserviceaccount.com' + credentials_json: ${{ secrets.GH_ACTIONS_SA_JSON }} - - name: test the cluster - run: kubectl get pods -A + - name: setup gcloud sdk + uses: google-github-actions/setup-gcloud@v1 + with: + version: '>= 363.0.0' + project_id: 'devseed-labs' + service_account_key: ${{ secrets.GH_ACTIONS_SA_TOKEN }} + export_default_credentials: true - - name: build images for unit tests + - name: configure kubectl context run: | - export SHELL=/bin/bash - eval $(minikube -p minikube docker-env) - docker build -f ./dockerfiles/Dockerfile.pgstac -t local/pgstac . - docker build -f ./dockerfiles/Dockerfile.stac -t local/stac . - docker build -f ./dockerfiles/Dockerfile.titiler -t local/titiler . - docker build -f ./dockerfiles/Dockerfile.tipg -t local/tipg . - echo -n "verifying images:" - docker images + gcloud components install gke-gcloud-auth-plugin + gcloud container clusters get-credentials k8seed-labs-cluster --zone us-central1-f - name: helm render/install eoapi templates run: | @@ -58,12 +71,12 @@ jobs: export POSTGRES_USER=username export PGPASSWORD=password export POSTGRES_PASSWORD=password - export GITSHA='A12345' + export GITSHA='${{github.sha}}' cd helm-chart helm install \ - --namespace default \ + --namespace $NS_NAME \ --create-namespace \ --set gitSha=$GITSHA \ --set db.settings.secrets.PGUSER=$PGUSER \ @@ -81,15 +94,7 @@ jobs: - name: restart the services run: | - # usually if a pod fails b/c the DB isn't up yet it will go into CrashLoopBackOff state and then restart - # and then the service is fine but minikube doesn't seem to restart things for some reason, could be - # that the underlying CMD(s) don't fully exit so we need to restart the services here - kubectl rollout restart deploy/vector - kubectl rollout restart deploy/stac - kubectl rollout restart deploy/raster - - sleep 10s - + kubectl config set-context --current --namespace=$NS_NAME while [[ -z "$(kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "startup complete" | head -n 1)" ]]; do echo "still waiting for raster service to start..." sleep 1 @@ -106,16 +111,6 @@ jobs: done echo "all services have started, moving on..." - sleep 10s - - - name: test service urls - run: | - minikube service list -# echo "------------------opening the service------------------" -# curl -XGET "$(minikube service vector --url)" -# curl -XGET "$(minikube service stac --url)" -# curl -XGET "$(minikube service raster --url)" - - name: install python unit-test dependencies run: | python -m pip install pytest httpx @@ -125,20 +120,27 @@ jobs: - name: run the tests run: | + kubectl config set-context --current --namespace=$NS_NAME + PUBLICIP='http://'$(kubectl -n ingress-nginx get svc/ingress-nginx-controller -o jsonpath='{.spec.loadBalancerIP}') + echo '#################################' + echo $PUBLICIP + echo '#################################' + # first substitute test endpoints in the test files # TODO: pytest should be able to have a pattern for injection here but moving fast - URL=$(minikube service vector --url) - sed -i "s|vector_endpoint\=.*$|vector_endpoint\='$URL'|g" .github/workflows/tests/test_vector.py + sed -i "s|vector_endpoint\=.*$|vector_endpoint\='$PUBLICIP/vector'|g" .github/workflows/tests/test_vector.py head -n 5 .github/workflows/tests/test_vector.py pytest .github/workflows/tests/test_vector.py - URL=$(minikube service stac --url) - sed -i "s|stac_endpoint\=.*$|stac_endpoint\='$URL'|g" .github/workflows/tests/test_stac.py + sed -i "s|stac_endpoint\=.*$|stac_endpoint\='$PUBLICIP/stac'|g" .github/workflows/tests/test_stac.py head -n 5 .github/workflows/tests/test_stac.py pytest .github/workflows/tests/test_stac.py - URL=$(minikube service raster --url) - sed -i "s|raster_endpoint\=.*$|raster_endpoint\='$URL'|g" .github/workflows/tests/test_raster.py + sed -i "s|raster_endpoint\=.*$|raster_endpoint\='$PUBLICIP/raster'|g" .github/workflows/tests/test_raster.py head -n 5 .github/workflows/tests/test_raster.py pytest .github/workflows/tests/test_raster.py + - name: helm uinstall eoapi templates + run: | + helm uninstall eoapi + kubectl delete ns $NS_NAME diff --git a/README.md b/README.md index 913ae6bb..921b27c3 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ If you don't have a k8s cluster set up on AWS or GCP then follow an IaC guide be ## Helm Installation -Once you have a k8s cluster set up you can `helm install` eoAPI as follows +Once you have a k8s cluster set up you can `helm install` eoAPI as follows: 1. `helm install` from https://devseed.com/eoapi-k8s/: @@ -79,3 +79,7 @@ Once you have a k8s cluster set up you can `helm install` eoAPI as follows eoapi \ ./eoapi ``` + +## Configuration Options and Defaults +Read about [Default Configuration](./docs/configuration.md#default-configuration) and +other [Configuration Options](./docs/configuration.md#additional-options) in the documentation diff --git a/dockerfiles/Dockerfile.pgstac b/dockerfiles/Dockerfile.pgstac deleted file mode 100644 index c5d020f1..00000000 --- a/dockerfiles/Dockerfile.pgstac +++ /dev/null @@ -1 +0,0 @@ -FROM ghcr.io/stac-utils/pgstac:v0.7.1 \ No newline at end of file diff --git a/dockerfiles/Dockerfile.stac b/dockerfiles/Dockerfile.stac deleted file mode 100644 index ab4a8f17..00000000 --- a/dockerfiles/Dockerfile.stac +++ /dev/null @@ -1 +0,0 @@ -FROM ghcr.io/stac-utils/stac-fastapi-pgstac:2.4.9 \ No newline at end of file diff --git a/dockerfiles/Dockerfile.tipg b/dockerfiles/Dockerfile.tipg deleted file mode 100644 index 5fe673b3..00000000 --- a/dockerfiles/Dockerfile.tipg +++ /dev/null @@ -1 +0,0 @@ -FROM ghcr.io/developmentseed/tipg:uvicorn-0.2.0 \ No newline at end of file diff --git a/dockerfiles/Dockerfile.titiler b/dockerfiles/Dockerfile.titiler deleted file mode 100644 index 2deefc0b..00000000 --- a/dockerfiles/Dockerfile.titiler +++ /dev/null @@ -1 +0,0 @@ -FROM ghcr.io/stac-utils/titiler-pgstac:uvicorn-0.4.1 \ No newline at end of file diff --git a/dockerfiles/README.md b/dockerfiles/README.md deleted file mode 100644 index aef17739..00000000 --- a/dockerfiles/README.md +++ /dev/null @@ -1 +0,0 @@ -these Dockerfile. are used in tests but should match default versions in `helm-chart/eoapi/values.yaml` \ No newline at end of file diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 00000000..fc275f0c --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,79 @@ +# Configuration Options + +## Required Values + +The required values to pass to `helm install` or `helm template` commands can be found by showing what is validated: + +```bash +$ head -n 9 /values.schema.json +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "required": [ + "providerContext", + "db", + "service", + "gitSha" + ], +``` + +Most of the required fields have common-sense defaults except traditional username and password secrets under `db`. +The table below and the `values.yaml` comments should explain what the options and defaults are: + +| **Values Key** | **Description** | **Default** | **Choices** | +|:-------------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------:|:------------:|------------------------| +| db.settings.secrets.PGUSER
db.settings.secrets.PGPASSWORD | username and password used by application for connections
https://www.postgresql.org/docs/current/libpq-envars.html | | | +| db.settings.secrets.POSTGRES_USER
db.settings.secrets.POSTGRES_PASSWORD | username and password used by
base postgresl image for admin purposes
see https://www.postgresql.org/docs/current/libpq-envars.html | | | +| providerContext | deprecated: used as a switch in helm templates for
provider-specific logic if needed | minikube | minikube | +| service.port | the port that all vector/raster/stac services run on
used in `kind: Service` and `kind: Ingress` | 8080 | your favorite port | +| gitSha | sha attached to a `kind: Deployment` key `metadata.labels` | gitshaABC123 | your favorite sha | + + +--- + +## Default Configuration + +Running `helm install` from https://devseed.com/eoapi-k8s/ with this simple `config.yml` overrides below +should spin up similar infrastructure in EKS or GKE: + +```python +$ cat config.yaml +db: + settings: + secrets: + PGUSER: "username" + POSTGRES_USER: "username" + PGPASSWORD: "password" + POSTGRES_PASSWORD: "password" +``` + +In EKS or GKE you'll by default get: + +* a pgstac PostgreSQL database deployment and service +* the same vector and raster data fixtures used for testing loaded into the DB +* a load balancer and nginx-compatible ingress with the following path rewrites: + * a `/stac` service for `stac_fastapi.pgstac` + * a `/raster` service for `titler.pgstac` + * a `/vector` service for `tipg.pgstac` + +Here's a simplified high-level diagram to grok: +![](./images/default_architecture.png) + +--- + +## Additional Options + +### Key `ingress.className` + +| **Values Key** | **Description** | **Default** | **Choices** | +|:-----------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------:|:-----------:|--------------| +| ingress.className | used as switch in helm templates for specific
functionality regarding `kind: Ingress` and cloud-provider
specific load balancing options | nginx | nginx
alb
gce
| + +#### Given `ingress.className=nginx` +![](./images/default_architecture.png) + + +#### Given `ingress.className=alb||gce` +![](./images/alb_architecture.png) + + diff --git a/docs/images/alb_architecture.png b/docs/images/alb_architecture.png new file mode 100644 index 00000000..8126876d Binary files /dev/null and b/docs/images/alb_architecture.png differ diff --git a/docs/images/default_architecture.png b/docs/images/default_architecture.png new file mode 100644 index 00000000..3d5413bc Binary files /dev/null and b/docs/images/default_architecture.png differ diff --git a/helm-chart/eoapi/CHANGELOG.md b/helm-chart/eoapi/CHANGELOG.md index f2ef2921..70d8d070 100644 --- a/helm-chart/eoapi/CHANGELOG.md +++ b/helm-chart/eoapi/CHANGELOG.md @@ -1,5 +1,11 @@ version numbers below correspond to helm chart `appVersion`: see ./helm-chart/eoapi/Chart.yaml --- +# 0.1.3 (2023-09-05) + +* test on GKE and add documentation where needed for [GKE template changes](https://github.com/developmentseed/eoapi-k8s/issues/29) +* CI/CD should run on GKE so we debug less test failures on minikube for [move CI/CD away from minikube](https://github.com/developmentseed/eoapi-k8s/issues/36) +* documentation about default configuration and additional options for [documentation](https://github.com/developmentseed/eoapi-k8s/issues/19) + # 0.1.2 (2023-08-31) * move `command` blocks out to `values.yml` for [generalizing ticket](https://github.com/developmentseed/eoapi-k8s/issues/31) diff --git a/helm-chart/eoapi/Chart.yaml b/helm-chart/eoapi/Chart.yaml index eba19743..f976408c 100644 --- a/helm-chart/eoapi/Chart.yaml +++ b/helm-chart/eoapi/Chart.yaml @@ -17,10 +17,10 @@ kubeVersion: ">=1.23.0-0" # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.1.4" +version: "0.1.5" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.1.2" +appVersion: "0.1.3" diff --git a/helm-chart/eoapi/templates/services/ingress-gce.yaml b/helm-chart/eoapi/templates/services/ingress-gce.yaml new file mode 100644 index 00000000..e0da898a --- /dev/null +++ b/helm-chart/eoapi/templates/services/ingress-gce.yaml @@ -0,0 +1,42 @@ +{{- if (and (.Values.ingress.enabled) (eq .Values.ingress.className "gce")) }} +{{- range $serviceName, $v := .Values -}} +{{- if (or (eq $serviceName "raster") (eq $serviceName "stac") (eq $serviceName "vector")) }} +{{- if index $v "enabled" }} +{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" $.Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $serviceName }} + # AWS EKS: + # https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.4/guide/ingress/annotations/ + # GCP GKE: + # https://cloud.google.com/kubernetes-engine/docs/concepts/ingress + annotations: + {{- if (and ($.Values.ingress.className) (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion)) }} + kubernetes.io/ingress.class: "{{ $.Values.ingress.className }}" + {{- end }} +spec: + rules: + - http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: {{ $serviceName }} + port: + number: {{ $.Values.service.port }} +--- +{{/* END: if index $v "enabled" */}} +{{- end }} +{{/* END: if (or (eq $serviceName "raster") (eq $serviceName "stac") (eq $serviceName "vector")) */}} +{{- end }} +{{/* END: range $serviceName, $v := .Values*/}} +{{- end }} +{{/* END: if .Values.ingress.className "alb" */}} +{{- end }} diff --git a/helm-chart/eoapi/test-unittest-values.yaml b/helm-chart/eoapi/test-unittest-values.yaml index c71dfe07..bfa1baa1 100644 --- a/helm-chart/eoapi/test-unittest-values.yaml +++ b/helm-chart/eoapi/test-unittest-values.yaml @@ -1,29 +1,21 @@ # used in GH Actions `unit-tests.yml` -providerContext: "minikube" ingress: - enabled: false - # used to set up NodePort in CI unittests on minikube - className: "testing123" + enabled: true + className: "nginx" db: enabled: true raster: enabled: true - image: - name: local/titiler - tag: latest settings: resources: limits: cpu: "512m" - memory: "1024Mi" + memory: "4096Mi" requests: - cpu: "512m" - memory: "1024Mi" + cpu: "256m" + memory: "1048Mi" stac: enabled: true - image: - name: local/stac - tag: latest settings: resources: limits: @@ -34,9 +26,6 @@ stac: memory: "512Mi" vector: enabled: true - image: - name: local/tipg - tag: latest settings: resources: limits: diff --git a/helm-chart/eoapi/values.schema.json b/helm-chart/eoapi/values.schema.json index dc7708bb..05f5c6a2 100644 --- a/helm-chart/eoapi/values.schema.json +++ b/helm-chart/eoapi/values.schema.json @@ -52,7 +52,7 @@ }, "environment": { "type": "string", - "pattern": "^(k8s|rds)$" + "pattern": "^(k8s|rds|cloudsql)$" } } }, @@ -76,7 +76,7 @@ "properties": { "className": { "type": "string", - "pattern": "^(alb|nginx|testing123)$" + "pattern": "^(alb|nginx|gce|testing123)$" } } }, @@ -86,7 +86,7 @@ }, "providerContext": { "type": "string", - "pattern": "^(aws|minikube)$" + "pattern": "^(aws|gcp|minikube|ignore)$" } } } diff --git a/helm-chart/eoapi/values.yaml b/helm-chart/eoapi/values.yaml index 85defa06..9df10a26 100644 --- a/helm-chart/eoapi/values.yaml +++ b/helm-chart/eoapi/values.yaml @@ -1,7 +1,7 @@ # default values for eoapi comment: > - some postgres variables are basically secrets that should be passed in via os environment variables. + some postgres variables are secrets that should be passed in via os environment variables. for example, to pass username or password information during `helm template` or `helm install` do: $ export GITSHA=$(git rev-parse HEAD | cut -c1-10) @@ -63,21 +63,18 @@ db: PGPASSWORD: "" PGDATABASE: "postgis" -# `providerContext`: "aws" || "minikube" -# currently only support for "aws" even though CI tests use "minikube" -# "gcp" will be an option available in the future -providerContext: "aws" +# deprecated: helm templates are only sniffing for "minikube" as of now +providerContext: "ignore" -# if running `helm install` from repo directory you'll have to provide this -# otherwise the chart on the gh-pages branch will provide the correct updated value +# the chart on the gh-pages branch will provide the correct updated value otherwise it's defaulted gitSha: "gitshaABC123" service: port: 8080 ingress: - # `ingress.className`: "alb" || "nginx" - # `"alb"` will create a `kind:Service` with `spec.port:'NodePort'` and ALBs for each service + # `ingress.className`: "alb" || "gce" || "nginx" + # `"alb || gce"` will create a `kind:Service` with `spec.port:'NodePort'` and ALBs for each service # `"nginx"` will create a `kind:Service` with a `spec.port:ClusterIP` and a single NLB and path rewrites for /vector, /stac, /raster enabled: true className: "nginx"