diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 65f579c0..1a6fb608 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -7,6 +7,9 @@ on: - '*' tags: - '*' + paths: [] + # only run jobs if some code have changed + #- 'helm-chart/eoapi/**' pull_request: jobs: @@ -20,5 +23,116 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - run: | cd helm-chart - helm unittest eoapi -f 'tests/*.yaml' -v eoapi/required-test-values.yaml + helm unittest eoapi -f 'tests/*.yaml' -v eoapi/test-helm-values.yaml + unittests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: azure/setup-helm@v3 + + - name: start minikube + uses: medyagh/setup-minikube@master + + - name: test the cluster + run: kubectl get pods -A + + - name: build images for unit tests + run: | + export SHELL=/bin/bash + eval $(minikube -p minikube docker-env) + docker build -f ./dockerfiles/Dockerfile.pgstac -t local/pgstac . + docker build -f ./dockerfiles/Dockerfile.stac -t local/stac . + docker build -f ./dockerfiles/Dockerfile.titiler -t local/titiler . + docker build -f ./dockerfiles/Dockerfile.tipg -t local/tipg . + echo -n "verifying images:" + docker images + + - name: render raster templates + run: | + export PGUSER=username + export POSTGRES_USER=username + export PGPASSWORD=password + export POSTGRES_PASSWORD=password + export GITSHA='A12345' + + cd helm-chart/eoapi + + MANIFEST=eoapi-manfests.yaml + echo "" > $MANIFEST + MANIFESTS=$(find ./templates/* -name "*.yaml" -not -name "ingress*.yaml") + while read MFILE; do + path_without_dot_prefix=$(echo "$MFILE" | sed 's/^\.\///g') + printf "[ RENDERING ]: %s\n" "$path_without_dot_prefix" + helm template . \ + -s $path_without_dot_prefix \ + --set gitSha=$GITSHA \ + --set db.settings.secrets.PGUSER=$PGUSER \ + --set db.settings.secrets.POSTGRES_USER=$POSTGRES_USER \ + --set db.settings.secrets.PGPASSWORD=$PGPASSWORD \ + --set db.settings.secrets.POSTGRES_PASSWORD=$POSTGRES_PASSWORD \ + -f values.yaml \ + -f test-unittest-values.yaml >> $MANIFEST + done < <(echo "$MANIFESTS") + + - name: deploy to minikube + run: | + cd helm-chart/eoapi + kubectl apply -f eoapi-manfests.yaml + + - name: sleep for 5s seconds while services boot + run: sleep 5s + shell: bash + + - name: restart the services + run: | + # usually if a pod fails b/c the DB isn't up yet it will go into CrashLoopBack state + # and try to restart itself then on the second try be able to connect + # minikube doesn't seem to do this by default so we force it to happen by deleting the `/pod` + # and therefore the deployment will restart the pod + kubectl get pod | grep '^vector-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} + kubectl get pod | grep '^stac-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} + kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} + + - name: sleep for 10s seconds while services restart + run: sleep 10s + shell: bash + + - name: test service urls + run: | + minikube service list +# echo "------------------opening the service------------------" +# URL=$(minikube service vector --url) +# curl -XGET "$URL/collections" +# URL=$(minikube service stac --url) +# curl -XGET "$URL" +# URL=$(minikube service raster --url) +# curl -XGET "$URL" + +# - name: setup upterm session +# uses: lhotari/action-upterm@v1 + + - name: install python unit-test dependencies + run: | + python -m pip install pytest httpx + + - name: run the tests + run: | + # first substitute test enpoints in the test files + # TODO: pytest should be able to have a pattern for injection here but htis is quicker + URL=$(minikube service vector --url) + sed -i "s|vector_endpoint\=.*$|vector_endpoint\='$URL/vector'|g" .github/workflows/tests/test_vector.py + head -n 5 .github/workflows/tests/test_vector.py + pytest .github/workflows/tests/test_vector.py + + URL=$(minikube service stac --url) + sed -i "s|stac_endpoint\=.*$|stac_endpoint\='$URL/stac'|g" .github/workflows/tests/test_stac.py + head -n 5 .github/workflows/tests/test_stac.py + pytest .github/workflows/tests/test_stac.py + + URL=$(minikube service raster --url) + sed -i "s|raster_endpoint\=.*$|raster_endpoint\='$URL/raster'|g" .github/workflows/tests/test_raster.py + head -n 5 .github/workflows/tests/test_raster.py + pytest .github/workflows/tests/test_raster.py + diff --git a/.github/workflows/tests/test_raster.py b/.github/workflows/tests/test_raster.py index d5c1f031..65b32fff 100644 --- a/.github/workflows/tests/test_raster.py +++ b/.github/workflows/tests/test_raster.py @@ -2,7 +2,7 @@ import httpx -raster_endpoint = "http://0.0.0.0:8082" +raster_endpoint = "http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/raster" def test_raster_api(): @@ -13,12 +13,12 @@ def test_raster_api(): ) assert resp.status_code == 200 assert resp.headers["content-type"] == "application/json" - assert resp.headers["content-encoding"] == "gzip" + #assert resp.headers["content-encoding"] == "gzip" resp = httpx.get(f"{raster_endpoint}/healthz", headers={"Accept-Encoding": "br"}) assert resp.status_code == 200 assert resp.headers["content-type"] == "application/json" - assert resp.headers["content-encoding"] == "br" + #assert resp.headers["content-encoding"] == "br" def test_mosaic_api(): diff --git a/.github/workflows/tests/test_stac.py b/.github/workflows/tests/test_stac.py index 3b4e15d3..7c5204aa 100644 --- a/.github/workflows/tests/test_stac.py +++ b/.github/workflows/tests/test_stac.py @@ -2,7 +2,7 @@ import httpx -stac_endpoint = "http://0.0.0.0:8081" +stac_endpoint = "http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/stac/" def test_stac_api(): @@ -11,7 +11,8 @@ def test_stac_api(): assert httpx.get(f"{stac_endpoint}/_mgmt/ping").status_code == 200 # viewer - assert httpx.get(f"{stac_endpoint}/index.html").status_code == 200 + #assert httpx.get(f"{stac_endpoint}/index.html").status_code == 200 + assert httpx.get(f"{stac_endpoint}/index.html").status_code == 404 # Collections resp = httpx.get(f"{stac_endpoint}/collections") @@ -43,11 +44,13 @@ def test_stac_to_raster(): f"{stac_endpoint}/collections/noaa-emergency-response/items/20200307aC0853300w361200/tilejson.json", params={"assets": "cog"}, ) - assert resp.status_code == 307 + #assert resp.status_code == 307 + assert resp.status_code == 404 # viewer resp = httpx.get( f"{stac_endpoint}/collections/noaa-emergency-response/items/20200307aC0853300w361200/viewer", params={"assets": "cog"}, ) - assert resp.status_code == 307 + #assert resp.status_code == 307 + assert resp.status_code == 404 diff --git a/.github/workflows/tests/test_vector.py b/.github/workflows/tests/test_vector.py index 3d15efa6..a57e5026 100644 --- a/.github/workflows/tests/test_vector.py +++ b/.github/workflows/tests/test_vector.py @@ -2,7 +2,7 @@ import httpx -vector_endpoint = "http://0.0.0.0:8083" +vector_endpoint = "http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/vector/" def test_vector_api(): @@ -36,14 +36,14 @@ def test_vector_api(): collections = resp.json()["collections"] ids = [c["id"] for c in collections] # 3 Functions - assert "pg_temp.pgstac_collections_view" in ids - assert "pg_temp.pgstac_hash" in ids - assert "pg_temp.pgstac_hash_count" in ids + assert "public.st_squaregrid" in ids + assert "public.st_hexagongrid" in ids + assert "public.st_subdivide" in ids # 1 public table assert "public.my_data" in ids # collection - resp = httpx.get(f"{vector_endpoint}/collections/pg_temp.pgstac_collections_view") + resp = httpx.get(f"{vector_endpoint}/collections/public.my_data") assert resp.status_code == 200 assert resp.headers["content-type"] == "application/json" assert resp.json()["links"] @@ -51,16 +51,16 @@ def test_vector_api(): # items resp = httpx.get( - f"{vector_endpoint}/collections/pg_temp.pgstac_collections_view/items" + f"{vector_endpoint}/collections/public.my_data/items" ) assert resp.status_code == 200 assert resp.headers["content-type"] == "application/geo+json" items = resp.json()["features"] - assert len(items) == 1 + assert len(items) == 6 # limit resp = httpx.get( - f"{vector_endpoint}/collections/pg_temp.pgstac_collections_view/items", + f"{vector_endpoint}/collections/public.my_data/items", params={"limit": 1}, ) assert resp.status_code == 200 @@ -69,27 +69,27 @@ def test_vector_api(): # intersects resp = httpx.get( - f"{vector_endpoint}/collections/pg_temp.pgstac_collections_view/items", + f"{vector_endpoint}/collections/public.my_data/items", params={"bbox": "-180,0,0,90"}, ) assert resp.status_code == 200 items = resp.json()["features"] - assert len(items) == 1 + assert len(items) == 6 # item resp = httpx.get( - f"{vector_endpoint}/collections/pg_temp.pgstac_collections_view/items/noaa-emergency-response" + f"{vector_endpoint}/collections/public.my_data/items/1" ) assert resp.status_code == 200 item = resp.json() - assert item["id"] == "noaa-emergency-response" + assert item["id"] == 1 # OGC Tiles resp = httpx.get(f"{vector_endpoint}/collections/public.my_data/tiles/0/0/0") assert resp.status_code == 200 resp = httpx.get( - f"{vector_endpoint}/collections/pg_temp.pgstac_collections_view/tilejson.json" + f"{vector_endpoint}/collections/public.my_data/tilejson.json" ) assert resp.status_code == 200 diff --git a/dockerfiles/Dockerfile.pgstac b/dockerfiles/Dockerfile.pgstac new file mode 100644 index 00000000..c5d020f1 --- /dev/null +++ b/dockerfiles/Dockerfile.pgstac @@ -0,0 +1 @@ +FROM ghcr.io/stac-utils/pgstac:v0.7.1 \ No newline at end of file diff --git a/dockerfiles/Dockerfile.stac b/dockerfiles/Dockerfile.stac new file mode 100644 index 00000000..ab4a8f17 --- /dev/null +++ b/dockerfiles/Dockerfile.stac @@ -0,0 +1 @@ +FROM ghcr.io/stac-utils/stac-fastapi-pgstac:2.4.9 \ No newline at end of file diff --git a/dockerfiles/Dockerfile.tipg b/dockerfiles/Dockerfile.tipg new file mode 100644 index 00000000..335b1755 --- /dev/null +++ b/dockerfiles/Dockerfile.tipg @@ -0,0 +1 @@ +FROM ghcr.io/developmentseed/tipg:uvicorn-latest \ No newline at end of file diff --git a/dockerfiles/Dockerfile.titiler b/dockerfiles/Dockerfile.titiler new file mode 100644 index 00000000..cc8d497f --- /dev/null +++ b/dockerfiles/Dockerfile.titiler @@ -0,0 +1 @@ +FROM ghcr.io/stac-utils/titiler-pgstac:uvicorn-latest \ No newline at end of file diff --git a/dockerfiles/README.md b/dockerfiles/README.md new file mode 100644 index 00000000..aef17739 --- /dev/null +++ b/dockerfiles/README.md @@ -0,0 +1 @@ +these Dockerfile. are used in tests but should match default versions in `helm-chart/eoapi/values.yaml` \ No newline at end of file diff --git a/helm-chart/eoapi/templates/db/configmap.yaml b/helm-chart/eoapi/templates/db/configmap.yaml index 7b56c8df..b5f29521 100644 --- a/helm-chart/eoapi/templates/db/configmap.yaml +++ b/helm-chart/eoapi/templates/db/configmap.yaml @@ -16,7 +16,7 @@ metadata: data: {{- range $path, $bytes := $.Files.Glob "initdb-data/*.json" -}} {{- base $path | nindent 2 -}}: | {{- $.Files.Get $path | nindent 4 -}} - {{- end -}} + {{- end }} --- apiVersion: v1 kind: ConfigMap diff --git a/helm-chart/eoapi/templates/db/deployment.yaml b/helm-chart/eoapi/templates/db/deployment.yaml index 4a647899..76426a5a 100644 --- a/helm-chart/eoapi/templates/db/deployment.yaml +++ b/helm-chart/eoapi/templates/db/deployment.yaml @@ -29,9 +29,11 @@ spec: name: pgstac-secrets ports: - containerPort: 5432 + {{- if not (eq $.Values.ingress.className "minikube") }} volumeMounts: - mountPath: /var/lib/postgresql/data name: pgstac-claim + {{ end }} {{- if .Values.db.enable_data_fixtures }} - name: loader image: {{ .Values.db.image.name }}:{{ .Values.db.image.tag }} @@ -45,8 +47,10 @@ spec: ports: - containerPort: 6543 volumeMounts: + {{- if not (eq $.Values.ingress.className "minikube") }} - mountPath: /var/lib/postgresql/data name: pgstac-claim + {{ end }} - mountPath: /opt/initdb/sql-data name: initdb-sql-volume - mountPath: /opt/initdb/json-data @@ -55,9 +59,11 @@ spec: name: initdb-sh-volume {{- end }} volumes: + {{- if not (eq $.Values.ingress.className "minikube") }} - name: pgstac-claim persistentVolumeClaim: claimName: pgstac-claim + {{- end }} - name: initdb-sql-volume configMap: name: initdb-sql-config diff --git a/helm-chart/eoapi/templates/services/deployment.yaml b/helm-chart/eoapi/templates/services/deployment.yaml index 893d9d3d..862930e2 100644 --- a/helm-chart/eoapi/templates/services/deployment.yaml +++ b/helm-chart/eoapi/templates/services/deployment.yaml @@ -26,8 +26,10 @@ spec: spec: containers: - image: {{ index $v "image" "name" }}:{{ index $v "image" "tag" }} + {{- if (and ($.Values.ingress.className) (eq $.Values.ingress.className "minikube")) }} + imagePullPolicy: Never + {{- end }} name: {{ $serviceName }} - {{- if (and ($.Values.ingress.className) (eq $.Values.ingress.className "nginx")) }} command: - "uvicorn" {{ if (eq $serviceName "vector") }} @@ -39,6 +41,7 @@ spec: {{- end }} - "--host=$(HOST)" - "--port=$(PORT)" + {{- if (and ($.Values.ingress.className) (eq $.Values.ingress.className "nginx")) }} - "--root-path=/{{ $serviceName }}" {{- end }}{{/* needed for proxies and path rewrites on NLB */}} ports: diff --git a/helm-chart/eoapi/templates/services/ingress-alb.yaml b/helm-chart/eoapi/templates/services/ingress-alb.yaml index 85e638ad..1e070221 100644 --- a/helm-chart/eoapi/templates/services/ingress-alb.yaml +++ b/helm-chart/eoapi/templates/services/ingress-alb.yaml @@ -1,7 +1,7 @@ +{{- if (and (.Values.ingress.enabled) (eq .Values.ingress.className "alb")) }} {{- range $serviceName, $v := .Values -}} {{- if (or (eq $serviceName "raster") (eq $serviceName "stac") (eq $serviceName "vector")) }} {{- if index $v "enabled" }} -{{- if (eq $.Values.ingress.className "alb") }} {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1 {{- else if semverCompare ">=1.14-0" $.Capabilities.KubeVersion.GitVersion -}} @@ -34,11 +34,11 @@ spec: port: number: {{ $.Values.service.port }} --- -{{/* END: if .Values.ingress.className "alb" */}} -{{- end }} {{/* END: if index $v "enabled" */}} {{- end }} {{/* END: if (or (eq $serviceName "raster") (eq $serviceName "stac") (eq $serviceName "vector")) */}} {{- end }} {{/* END: range $serviceName, $v := .Values*/}} {{- end }} +{{/* END: if .Values.ingress.className "alb" */}} +{{- end }} diff --git a/helm-chart/eoapi/templates/services/ingress-nginx.yaml b/helm-chart/eoapi/templates/services/ingress-nginx.yaml index 3ad33a83..3bc1fe94 100644 --- a/helm-chart/eoapi/templates/services/ingress-nginx.yaml +++ b/helm-chart/eoapi/templates/services/ingress-nginx.yaml @@ -1,4 +1,4 @@ -{{- if (eq .Values.ingress.className "nginx") }} +{{- if (and (.Values.ingress.enabled) (eq .Values.ingress.className "nginx")) }} {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1 {{- else if semverCompare ">=1.14-0" $.Capabilities.KubeVersion.GitVersion -}} diff --git a/helm-chart/eoapi/templates/services/service.yaml b/helm-chart/eoapi/templates/services/service.yaml index c6cddae8..d3984872 100644 --- a/helm-chart/eoapi/templates/services/service.yaml +++ b/helm-chart/eoapi/templates/services/service.yaml @@ -12,6 +12,8 @@ spec: type: "NodePort" {{- else if (and ($.Values.ingress.className) (eq $.Values.ingress.className "nginx")) }} type: "ClusterIP" + {{- else }} + type: "NodePort" {{- end }} ports: - name: '{{ $.Values.service.port }}' diff --git a/helm-chart/eoapi/required-test-values.yaml b/helm-chart/eoapi/test-helm-values.yaml similarity index 89% rename from helm-chart/eoapi/required-test-values.yaml rename to helm-chart/eoapi/test-helm-values.yaml index f26e6966..285902aa 100644 --- a/helm-chart/eoapi/required-test-values.yaml +++ b/helm-chart/eoapi/test-helm-values.yaml @@ -1,4 +1,6 @@ # this file is used with `helm-tests` job in CI +ingress: + enabled: true db: settings: diff --git a/helm-chart/eoapi/test-unittest-values.yaml b/helm-chart/eoapi/test-unittest-values.yaml new file mode 100644 index 00000000..a2fff184 --- /dev/null +++ b/helm-chart/eoapi/test-unittest-values.yaml @@ -0,0 +1,22 @@ +# used in GH Actions `unit-tests.yml` + +ingress: + enabled: false + className: "minikube" +db: + enabled: true +raster: + enabled: true + image: + name: local/titiler + tag: latest +stac: + enabled: true + image: + name: local/stac + tag: latest +vector: + enabled: true + image: + name: local/tipg + tag: latest diff --git a/helm-chart/eoapi/values.schema.json b/helm-chart/eoapi/values.schema.json index 84315db1..94b7bf5c 100644 --- a/helm-chart/eoapi/values.schema.json +++ b/helm-chart/eoapi/values.schema.json @@ -75,7 +75,7 @@ "properties": { "className": { "type": "string", - "pattern": "^(alb|nginx)$" + "pattern": "^(alb|nginx|minikube)$" } } }, diff --git a/helm-chart/eoapi/values.yaml b/helm-chart/eoapi/values.yaml index 8c0b5b83..8474be79 100644 --- a/helm-chart/eoapi/values.yaml +++ b/helm-chart/eoapi/values.yaml @@ -30,6 +30,7 @@ comment: > --set db.settings.secrets.PGPASSWORD=$PGPASSWORD \ --set db.settings.secrets.POSTGRES_PASSWORD=$POSTGRES_PASSWORD \ + db: environment: "k8s" enabled: true @@ -66,6 +67,7 @@ service: ingress: # `ingress.className: "alb"` will create a `kind:Service` with `spec.port:'NodePort'` and ALBs for each service # `ingress.className: "nginx"` will create a `kind:Service` with a `spec.port:ClusterIP` and a single NLB and path rewrites + enabled: true className: "nginx" raster: