Feature/unit tests

.github/workflows/helm-tests.yml

@@ -7,6 +7,9 @@ on:
     - '*'
     tags:
     - '*'
+    paths: []
+    # only run jobs if some code have changed
+    #- 'helm-chart/eoapi/**'
   pull_request:
 
 jobs:
@@ -20,5 +23,116 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
       - run: |
           cd helm-chart
-          helm unittest eoapi -f 'tests/*.yaml' -v eoapi/required-test-values.yaml
+          helm unittest eoapi -f 'tests/*.yaml' -v eoapi/test-helm-values.yaml
+  unittests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: azure/setup-helm@v3
+
+      - name: start minikube
+        uses: medyagh/setup-minikube@master
+
+      - name: test the cluster
+        run: kubectl get pods -A
+
+      - name: build images for unit tests
+        run: |
+          export SHELL=/bin/bash
+          eval $(minikube -p minikube docker-env)
+          docker build -f ./dockerfiles/Dockerfile.pgstac -t local/pgstac .
+          docker build -f ./dockerfiles/Dockerfile.stac -t local/stac .
+          docker build -f ./dockerfiles/Dockerfile.titiler -t local/titiler .
+          docker build -f ./dockerfiles/Dockerfile.tipg -t local/tipg .
+          echo -n "verifying images:"
+          docker images
+
+      - name: render raster templates
+        run: |
+          export PGUSER=username
+          export POSTGRES_USER=username
+          export PGPASSWORD=password
+          export POSTGRES_PASSWORD=password
+          export GITSHA='A12345'
+
+          cd helm-chart/eoapi
+
+          MANIFEST=eoapi-manfests.yaml
+          echo "" > $MANIFEST
+          MANIFESTS=$(find ./templates/* -name "*.yaml" -not -name "ingress*.yaml")
+          while read MFILE; do
+            path_without_dot_prefix=$(echo "$MFILE" | sed 's/^\.\///g')
+            printf "[ RENDERING ]: %s\n" "$path_without_dot_prefix"
+            helm template . \
+              -s $path_without_dot_prefix \
+              --set gitSha=$GITSHA \
+              --set db.settings.secrets.PGUSER=$PGUSER \
+              --set db.settings.secrets.POSTGRES_USER=$POSTGRES_USER \
+              --set db.settings.secrets.PGPASSWORD=$PGPASSWORD \
+              --set db.settings.secrets.POSTGRES_PASSWORD=$POSTGRES_PASSWORD \
+              -f values.yaml \
+              -f test-unittest-values.yaml >> $MANIFEST
+          done < <(echo "$MANIFESTS")
+
+      - name: deploy to minikube
+        run: |
+          cd helm-chart/eoapi
+          kubectl apply -f eoapi-manfests.yaml
+
+      - name: sleep for 5s seconds while services boot
+        run: sleep 5s
+        shell: bash
+
+      - name: restart the services
+        run: |
+          # usually if a pod fails b/c the DB isn't up yet it will go into CrashLoopBack state
+          # and try to restart itself then on the second try be able to connect
+          # minikube doesn't seem to do this by default so we force it to happen by deleting the `/pod`
+          # and therefore the deployment will restart the pod
+          kubectl get pod  | grep '^vector-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{}
+          kubectl get pod  | grep '^stac-.*$'  | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{}
+          kubectl get pod  | grep '^raster-.*$'  | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{}
+
+      - name: sleep for 10s seconds while services restart
+        run: sleep 10s
+        shell: bash
+
+      - name: test service urls
+        run: |
+          minikube service list
+#          echo "------------------opening the service------------------"
+#          URL=$(minikube service vector --url)
+#          curl -XGET "$URL/collections"
+#          URL=$(minikube service stac --url)
+#          curl -XGET "$URL"
+#          URL=$(minikube service raster --url)
+#          curl -XGET "$URL"
+
+#      - name: setup upterm session
+#        uses: lhotari/action-upterm@v1
+
+      - name: install python unit-test dependencies
+        run: |
+          python -m pip install pytest httpx
+
+      - name: run the tests
+        run: |
+          # first substitute test enpoints in the test files
+          # TODO: pytest should be able to have a pattern for injection here but htis is quicker
+          URL=$(minikube service vector --url)
+          sed -i "s|vector_endpoint\=.*$|vector_endpoint\='$URL/vector'|g" .github/workflows/tests/test_vector.py
+          head -n 5 .github/workflows/tests/test_vector.py
+          pytest .github/workflows/tests/test_vector.py
+
+          URL=$(minikube service stac --url)
+          sed -i "s|stac_endpoint\=.*$|stac_endpoint\='$URL/stac'|g" .github/workflows/tests/test_stac.py
+          head -n 5 .github/workflows/tests/test_stac.py
+          pytest .github/workflows/tests/test_stac.py
+
+          URL=$(minikube service raster --url)
+          sed -i "s|raster_endpoint\=.*$|raster_endpoint\='$URL/raster'|g" .github/workflows/tests/test_raster.py
+          head -n 5 .github/workflows/tests/test_raster.py
+          pytest .github/workflows/tests/test_raster.py
+
 

.github/workflows/tests/test_raster.py

@@ -2,7 +2,7 @@
 
 import httpx
 
-raster_endpoint = "http://0.0.0.0:8082"
+raster_endpoint = "http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/raster"
 
 
 def test_raster_api():
@@ -13,12 +13,12 @@ def test_raster_api():
     )
     assert resp.status_code == 200
     assert resp.headers["content-type"] == "application/json"
-    assert resp.headers["content-encoding"] == "gzip"
+    #assert resp.headers["content-encoding"] == "gzip"
 
     resp = httpx.get(f"{raster_endpoint}/healthz", headers={"Accept-Encoding": "br"})
     assert resp.status_code == 200
     assert resp.headers["content-type"] == "application/json"
-    assert resp.headers["content-encoding"] == "br"
+    #assert resp.headers["content-encoding"] == "br"
 
 
 def test_mosaic_api():

.github/workflows/tests/test_stac.py

@@ -2,7 +2,7 @@
 
 import httpx
 
-stac_endpoint = "http://0.0.0.0:8081"
+stac_endpoint = "http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/stac/"
 
 
 def test_stac_api():
@@ -11,7 +11,8 @@ def test_stac_api():
     assert httpx.get(f"{stac_endpoint}/_mgmt/ping").status_code == 200
 
     # viewer
-    assert httpx.get(f"{stac_endpoint}/index.html").status_code == 200
+    #assert httpx.get(f"{stac_endpoint}/index.html").status_code == 200
+    assert httpx.get(f"{stac_endpoint}/index.html").status_code == 404
 
     # Collections
     resp = httpx.get(f"{stac_endpoint}/collections")
@@ -43,11 +44,13 @@ def test_stac_to_raster():
         f"{stac_endpoint}/collections/noaa-emergency-response/items/20200307aC0853300w361200/tilejson.json",
         params={"assets": "cog"},
     )
-    assert resp.status_code == 307
+    #assert resp.status_code == 307
+    assert resp.status_code == 404
 
     # viewer
     resp = httpx.get(
         f"{stac_endpoint}/collections/noaa-emergency-response/items/20200307aC0853300w361200/viewer",
         params={"assets": "cog"},
     )
-    assert resp.status_code == 307
+    #assert resp.status_code == 307
+    assert resp.status_code == 404

.github/workflows/tests/test_vector.py

@@ -2,7 +2,7 @@
 
 import httpx
 
-vector_endpoint = "http://0.0.0.0:8083"
+vector_endpoint = "http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/vector/"
 
 
 def test_vector_api():
@@ -36,31 +36,31 @@ def test_vector_api():
     collections = resp.json()["collections"]
     ids = [c["id"] for c in collections]
     # 3 Functions
-    assert "pg_temp.pgstac_collections_view" in ids
-    assert "pg_temp.pgstac_hash" in ids
-    assert "pg_temp.pgstac_hash_count" in ids
+    assert "public.st_squaregrid" in ids
+    assert "public.st_hexagongrid" in ids
+    assert "public.st_subdivide" in ids
     # 1 public table
     assert "public.my_data" in ids
 
     # collection
-    resp = httpx.get(f"{vector_endpoint}/collections/pg_temp.pgstac_collections_view")
+    resp = httpx.get(f"{vector_endpoint}/collections/public.my_data")
     assert resp.status_code == 200
     assert resp.headers["content-type"] == "application/json"
     assert resp.json()["links"]
     assert resp.json()["itemType"] == "feature"
 
     # items
     resp = httpx.get(
-        f"{vector_endpoint}/collections/pg_temp.pgstac_collections_view/items"
+        f"{vector_endpoint}/collections/public.my_data/items"
     )
     assert resp.status_code == 200
     assert resp.headers["content-type"] == "application/geo+json"
     items = resp.json()["features"]
-    assert len(items) == 1
+    assert len(items) == 6
 
     # limit
     resp = httpx.get(
-        f"{vector_endpoint}/collections/pg_temp.pgstac_collections_view/items",
+        f"{vector_endpoint}/collections/public.my_data/items",
         params={"limit": 1},
     )
     assert resp.status_code == 200
@@ -69,27 +69,27 @@ def test_vector_api():
 
     # intersects
     resp = httpx.get(
-        f"{vector_endpoint}/collections/pg_temp.pgstac_collections_view/items",
+        f"{vector_endpoint}/collections/public.my_data/items",
         params={"bbox": "-180,0,0,90"},
     )
     assert resp.status_code == 200
     items = resp.json()["features"]
-    assert len(items) == 1
+    assert len(items) == 6
 
     # item
     resp = httpx.get(
-        f"{vector_endpoint}/collections/pg_temp.pgstac_collections_view/items/noaa-emergency-response"
+        f"{vector_endpoint}/collections/public.my_data/items/1"
     )
     assert resp.status_code == 200
     item = resp.json()
-    assert item["id"] == "noaa-emergency-response"
+    assert item["id"] == 1
 
     # OGC Tiles
     resp = httpx.get(f"{vector_endpoint}/collections/public.my_data/tiles/0/0/0")
     assert resp.status_code == 200
 
     resp = httpx.get(
-        f"{vector_endpoint}/collections/pg_temp.pgstac_collections_view/tilejson.json"
+        f"{vector_endpoint}/collections/public.my_data/tilejson.json"
     )
     assert resp.status_code == 200
 

dockerfiles/Dockerfile.pgstac

@@ -0,0 +1 @@
+FROM ghcr.io/stac-utils/pgstac:v0.7.1

dockerfiles/Dockerfile.stac

@@ -0,0 +1 @@
+FROM ghcr.io/stac-utils/stac-fastapi-pgstac:2.4.9

dockerfiles/Dockerfile.tipg

@@ -0,0 +1 @@
+FROM ghcr.io/developmentseed/tipg:uvicorn-latest

dockerfiles/Dockerfile.titiler

@@ -0,0 +1 @@
+FROM ghcr.io/stac-utils/titiler-pgstac:uvicorn-latest

dockerfiles/README.md

@@ -0,0 +1 @@
+these Dockerfile.<package> are used in tests but should match default versions in `helm-chart/eoapi/values.yaml`

helm-chart/eoapi/templates/db/configmap.yaml

@@ -16,7 +16,7 @@ metadata:
 data:
   {{- range $path, $bytes := $.Files.Glob "initdb-data/*.json"  -}}
   {{- base $path | nindent 2 -}}: | {{- $.Files.Get $path | nindent 4 -}}
-  {{- end -}}
+  {{- end }}
 ---
 apiVersion: v1
 kind: ConfigMap

helm-chart/eoapi/templates/db/deployment.yaml

@@ -29,9 +29,11 @@ spec:
               name: pgstac-secrets
           ports:
             - containerPort: 5432
+          {{- if not (eq $.Values.ingress.className "minikube") }}
           volumeMounts:
             - mountPath: /var/lib/postgresql/data
               name: pgstac-claim
+          {{ end }}
         {{- if .Values.db.enable_data_fixtures }}
         - name: loader
           image: {{ .Values.db.image.name }}:{{ .Values.db.image.tag }}
@@ -45,8 +47,10 @@ spec:
           ports:
             - containerPort: 6543
           volumeMounts:
+            {{- if not (eq $.Values.ingress.className "minikube") }}
             - mountPath: /var/lib/postgresql/data
               name: pgstac-claim
+            {{ end }}
             - mountPath: /opt/initdb/sql-data
               name: initdb-sql-volume
             - mountPath: /opt/initdb/json-data
@@ -55,9 +59,11 @@ spec:
               name: initdb-sh-volume
         {{- end }}
       volumes:
+        {{- if not (eq $.Values.ingress.className "minikube") }}
         - name: pgstac-claim
           persistentVolumeClaim:
             claimName: pgstac-claim
+        {{- end }}
         - name: initdb-sql-volume
           configMap:
             name: initdb-sql-config

helm-chart/eoapi/templates/services/deployment.yaml

@@ -26,8 +26,10 @@ spec:
     spec:
       containers:
       - image: {{ index $v "image" "name" }}:{{ index $v "image" "tag" }}
+        {{- if (and ($.Values.ingress.className) (eq $.Values.ingress.className "minikube")) }}
+        imagePullPolicy: Never
+        {{- end }}
         name: {{ $serviceName }}
-        {{- if (and ($.Values.ingress.className) (eq $.Values.ingress.className "nginx")) }}
         command:
           - "uvicorn"
         {{ if (eq $serviceName "vector") }}
@@ -39,6 +41,7 @@ spec:
         {{- end }}
           - "--host=$(HOST)"
           - "--port=$(PORT)"
+        {{- if (and ($.Values.ingress.className) (eq $.Values.ingress.className "nginx")) }}
           - "--root-path=/{{ $serviceName }}"
         {{- end }}{{/* needed for proxies and path rewrites on NLB */}}
         ports:

helm-chart/eoapi/templates/services/ingress-alb.yaml

@@ -1,7 +1,7 @@
+{{- if (and (.Values.ingress.enabled) (eq .Values.ingress.className "alb")) }}
 {{- range $serviceName, $v := .Values -}}
 {{- if (or (eq $serviceName "raster") (eq $serviceName "stac") (eq $serviceName "vector")) }}
 {{- if index $v "enabled" }}
-{{- if (eq $.Values.ingress.className "alb") }}
 {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion -}}
 apiVersion: networking.k8s.io/v1
 {{- else if semverCompare ">=1.14-0" $.Capabilities.KubeVersion.GitVersion -}}
@@ -34,11 +34,11 @@ spec:
                 port:
                   number: {{ $.Values.service.port }}
 ---
-{{/* END: if .Values.ingress.className "alb" */}}
-{{- end }}
 {{/* END: if index $v "enabled" */}}
 {{- end }}
 {{/* END: if (or (eq $serviceName "raster") (eq $serviceName "stac") (eq $serviceName "vector")) */}}
 {{- end }}
 {{/* END: range $serviceName, $v := .Values*/}}
 {{- end }}
+{{/* END: if .Values.ingress.className "alb" */}}
+{{- end }}

helm-chart/eoapi/templates/services/ingress-nginx.yaml

@@ -1,4 +1,4 @@
-{{- if (eq .Values.ingress.className "nginx") }}
+{{- if (and (.Values.ingress.enabled) (eq .Values.ingress.className "nginx")) }}
 {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion -}}
 apiVersion: networking.k8s.io/v1
 {{- else if semverCompare ">=1.14-0" $.Capabilities.KubeVersion.GitVersion -}}

helm-chart/eoapi/templates/services/service.yaml

@@ -12,6 +12,8 @@ spec:
   type: "NodePort"
   {{- else if (and ($.Values.ingress.className) (eq $.Values.ingress.className "nginx")) }}
   type: "ClusterIP"
+  {{- else }}
+  type: "NodePort"
   {{- end }}
   ports:
   - name: '{{ $.Values.service.port }}'

helm-chart/eoapi/required-test-values.yaml → helm-chart/eoapi/test-helm-values.yaml

@@ -1,4 +1,6 @@
 # this file is used with `helm-tests` job in CI
+ingress:
+  enabled: true
 
 db:
   settings: