From 3238fc653a832a0850f763205a7f8f7662a144a0 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 07:56:57 -0700 Subject: [PATCH 01/22] use GITHUB_TOKEN for helm install --- .github/workflows/helm-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 1a6fb608..b9010f55 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -30,6 +30,9 @@ jobs: - uses: actions/checkout@v3 - uses: azure/setup-helm@v3 + - with: + helm-version: v3.8.2 + github-token: ${{ secrets.GITHUB_TOKEN }} - name: start minikube uses: medyagh/setup-minikube@master From c54fe5b5b0d58847cc0f03906843498350f44626 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 07:57:42 -0700 Subject: [PATCH 02/22] uses key not array --- .github/workflows/helm-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index b9010f55..68da9700 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -30,7 +30,7 @@ jobs: - uses: actions/checkout@v3 - uses: azure/setup-helm@v3 - - with: + with: helm-version: v3.8.2 github-token: ${{ secrets.GITHUB_TOKEN }} From 822d41a8ec9350ee433654a3b089e8f0b30f4ded Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 08:01:50 -0700 Subject: [PATCH 03/22] fix README and hopefully trigger job --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 55d8c372..1df72fd9 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,7 @@ ## Getting Started -If you don't have a k8s cluster set up on AWS or GCP then follow an IAC guide below that is relevant to you. This helm chart hasn't been -tested on [minikube](https://github.com/kubernetes/minikube) yet: +If you don't have a k8s cluster set up on AWS or GCP then follow an IAC guide below that is relevant to you > ⓘ The helm chart in this repo assumes your cluster has a few third-party add-ons and controllers installed. So > it's in your best interest to read through the IAC guides to understand what those defaults are From 8bad1f44f8f779c358362a67acd39defc3ba75d1 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 08:02:36 -0700 Subject: [PATCH 04/22] this work? --- .github/workflows/helm-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 68da9700..0f1c9f6e 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -31,7 +31,6 @@ jobs: - uses: azure/setup-helm@v3 with: - helm-version: v3.8.2 github-token: ${{ secrets.GITHUB_TOKEN }} - name: start minikube From 3e5142f6a2741a243c055743d834a6ffb74a24b4 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 08:03:14 -0700 Subject: [PATCH 05/22] try this now? --- .github/workflows/helm-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 0f1c9f6e..621f0c97 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -30,8 +30,8 @@ jobs: - uses: actions/checkout@v3 - uses: azure/setup-helm@v3 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} +# with: +# github-token: ${{ secrets.GITHUB_TOKEN }} - name: start minikube uses: medyagh/setup-minikube@master From 9903f159483b8c5c78f77ff126b4700612aa93e9 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 08:14:46 -0700 Subject: [PATCH 06/22] add 'providerContext' key --- .github/workflows/ci.bkup | 126 ------------------ .github/workflows/deploy_mkdocs.bkup | 34 ----- helm-chart/eoapi/templates/db/deployment.yaml | 6 +- .../eoapi/templates/services/deployment.yaml | 2 +- helm-chart/eoapi/test-helm-values.yaml | 2 + helm-chart/eoapi/test-unittest-values.yaml | 3 +- helm-chart/eoapi/values.schema.json | 5 + helm-chart/eoapi/values.yaml | 5 + 8 files changed, 17 insertions(+), 166 deletions(-) delete mode 100644 .github/workflows/ci.bkup delete mode 100644 .github/workflows/deploy_mkdocs.bkup diff --git a/.github/workflows/ci.bkup b/.github/workflows/ci.bkup deleted file mode 100644 index 8fa11198..00000000 --- a/.github/workflows/ci.bkup +++ /dev/null @@ -1,126 +0,0 @@ -name: CI - -# On every pull request, but only on push to main -on: - push: - branches: - - main - tags: - - '*' - pull_request: - -jobs: - helm-tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: d3adb5/helm-unittest-action@v2 - with: - helm-version: v3.8.2 - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: | - cd infrastructure/k8s/helm-chart - helm unittest eoapi -f 'tests/*.yaml' -v eoapi/required-test-values.yaml - - - publish-docker: - needs: [helm-tests] - if: github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to Github - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set tag version - id: tag - run: | - echo "version=${GITHUB_REF#refs/*/}" - echo "version=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - - ############################################################################# - # RASTER - - name: RASTER - Build and push latest - if: github.ref == 'refs/heads/main' - uses: docker/build-push-action@v2 - with: - platforms: linux/amd64 - context: . - file: dockerfiles/Dockerfile.raster-uvicorn - push: true - tags: | - ghcr.io/developmentseed/eoapi-raster:latest - - - name: RASTER - Build and push tags - if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' - uses: docker/build-push-action@v2 - with: - platforms: linux/amd64 - context: . - file: dockerfiles/Dockerfile.raster-uvicorn - push: true - tags: | - ghcr.io/developmentseed/eoapi-raster:${{ steps.tag.outputs.tag }} - - ############################################################################# - # STAC - - name: STAC - Build and push latest - if: github.ref == 'refs/heads/main' - uses: docker/build-push-action@v2 - with: - context: . - file: dockerfiles/Dockerfile.stac-uvicorn - push: true - tags: | - ghcr.io/developmentseed/eoapi-stac:latest - - - name: STAC - Build and push tags - if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' - uses: docker/build-push-action@v2 - with: - context: . - file: dockerfiles/Dockerfile.stac-uvicorn - push: true - tags: | - ghcr.io/developmentseed/eoapi-stac:${{ steps.tag.outputs.tag }} - - ############################################################################# - # VECTOR - - name: VECTOR - Build and push latest - if: github.ref == 'refs/heads/main' - uses: docker/build-push-action@v2 - with: - context: . - file: dockerfiles/Dockerfile.vector-uvicorn - push: true - tags: | - ghcr.io/developmentseed/eoapi-vector:latest - - - name: VECTOR - Build and push tags - if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' - uses: docker/build-push-action@v2 - with: - context: . - file: dockerfiles/Dockerfile.vector-uvicorn - push: true - tags: | - ghcr.io/developmentseed/eoapi-vector:${{ steps.tag.outputs.tag }} - diff --git a/.github/workflows/deploy_mkdocs.bkup b/.github/workflows/deploy_mkdocs.bkup deleted file mode 100644 index 159aae18..00000000 --- a/.github/workflows/deploy_mkdocs.bkup +++ /dev/null @@ -1,34 +0,0 @@ -name: Publish docs via GitHub Pages - -on: - push: - branches: - - main - paths: - # Only rebuild website when docs have changed - - 'README.md' - - 'CHANGES.md' - - 'CONTRIBUTING.md' - - 'docs/**' - - '.github/workflows/deploy_mkdocs.yml' - -jobs: - build: - name: Deploy docs - runs-on: ubuntu-latest - steps: - - name: Checkout main - uses: actions/checkout@v3 - - - name: Set up Python 3.8 - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install mkdocs mkdocs-material pygments - - - name: Deploy docs - run: mkdocs gh-deploy --force -f docs/mkdocs.yml diff --git a/helm-chart/eoapi/templates/db/deployment.yaml b/helm-chart/eoapi/templates/db/deployment.yaml index 76426a5a..bff11541 100644 --- a/helm-chart/eoapi/templates/db/deployment.yaml +++ b/helm-chart/eoapi/templates/db/deployment.yaml @@ -29,7 +29,7 @@ spec: name: pgstac-secrets ports: - containerPort: 5432 - {{- if not (eq $.Values.ingress.className "minikube") }} + {{- if not (eq .Values.providerContext "minikube") }} volumeMounts: - mountPath: /var/lib/postgresql/data name: pgstac-claim @@ -47,7 +47,7 @@ spec: ports: - containerPort: 6543 volumeMounts: - {{- if not (eq $.Values.ingress.className "minikube") }} + {{- if not (eq .Values.providerContext "minikube") }} - mountPath: /var/lib/postgresql/data name: pgstac-claim {{ end }} @@ -59,7 +59,7 @@ spec: name: initdb-sh-volume {{- end }} volumes: - {{- if not (eq $.Values.ingress.className "minikube") }} + {{- if not (eq .Values.providerContext "minikube") }} - name: pgstac-claim persistentVolumeClaim: claimName: pgstac-claim diff --git a/helm-chart/eoapi/templates/services/deployment.yaml b/helm-chart/eoapi/templates/services/deployment.yaml index 862930e2..3264617b 100644 --- a/helm-chart/eoapi/templates/services/deployment.yaml +++ b/helm-chart/eoapi/templates/services/deployment.yaml @@ -26,7 +26,7 @@ spec: spec: containers: - image: {{ index $v "image" "name" }}:{{ index $v "image" "tag" }} - {{- if (and ($.Values.ingress.className) (eq $.Values.ingress.className "minikube")) }} + {{- if (eq $.Values.providerContext "minikube") }} imagePullPolicy: Never {{- end }} name: {{ $serviceName }} diff --git a/helm-chart/eoapi/test-helm-values.yaml b/helm-chart/eoapi/test-helm-values.yaml index 285902aa..71ee356e 100644 --- a/helm-chart/eoapi/test-helm-values.yaml +++ b/helm-chart/eoapi/test-helm-values.yaml @@ -1,4 +1,6 @@ # this file is used with `helm-tests` job in CI +providerContext: "aws" + ingress: enabled: true diff --git a/helm-chart/eoapi/test-unittest-values.yaml b/helm-chart/eoapi/test-unittest-values.yaml index a2fff184..cba5ad93 100644 --- a/helm-chart/eoapi/test-unittest-values.yaml +++ b/helm-chart/eoapi/test-unittest-values.yaml @@ -1,8 +1,7 @@ # used in GH Actions `unit-tests.yml` - +providerContext: "minikube" ingress: enabled: false - className: "minikube" db: enabled: true raster: diff --git a/helm-chart/eoapi/values.schema.json b/helm-chart/eoapi/values.schema.json index 94b7bf5c..6cb432f0 100644 --- a/helm-chart/eoapi/values.schema.json +++ b/helm-chart/eoapi/values.schema.json @@ -2,6 +2,7 @@ "$schema": "http://json-schema.org/schema#", "type": "object", "required": [ + "providerContext", "db", "service", "gitSha" @@ -82,6 +83,10 @@ "gitSha": { "type": "string", "pattern": "^.+$" + }, + "providerContext": { + "type": "string", + "pattern": "^(aws|minikube)$" } } } diff --git a/helm-chart/eoapi/values.yaml b/helm-chart/eoapi/values.yaml index 8474be79..e299285a 100644 --- a/helm-chart/eoapi/values.yaml +++ b/helm-chart/eoapi/values.yaml @@ -59,6 +59,11 @@ db: PGPASSWORD: "" PGDATABASE: "postgis" +providerContext: "aws" +# 'aws' is only supported at the moment. +# our CI pipelines test against 'minikube' yet it hasn't been fully evaluated. +# 'gcp' will be coming shortly in the future + gitSha: "" service: From 5a5787fe54c13f6063b8e84ccbb61a017bca7243 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 08:29:43 -0700 Subject: [PATCH 07/22] pin image versions in values.yml --- dockerfiles/Dockerfile.tipg | 2 +- dockerfiles/Dockerfile.titiler | 2 +- helm-chart/eoapi/test-unittest-values.yaml | 2 ++ helm-chart/eoapi/values.yaml | 4 ++-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dockerfiles/Dockerfile.tipg b/dockerfiles/Dockerfile.tipg index 335b1755..5fe673b3 100644 --- a/dockerfiles/Dockerfile.tipg +++ b/dockerfiles/Dockerfile.tipg @@ -1 +1 @@ -FROM ghcr.io/developmentseed/tipg:uvicorn-latest \ No newline at end of file +FROM ghcr.io/developmentseed/tipg:uvicorn-0.2.0 \ No newline at end of file diff --git a/dockerfiles/Dockerfile.titiler b/dockerfiles/Dockerfile.titiler index cc8d497f..2deefc0b 100644 --- a/dockerfiles/Dockerfile.titiler +++ b/dockerfiles/Dockerfile.titiler @@ -1 +1 @@ -FROM ghcr.io/stac-utils/titiler-pgstac:uvicorn-latest \ No newline at end of file +FROM ghcr.io/stac-utils/titiler-pgstac:uvicorn-0.4.1 \ No newline at end of file diff --git a/helm-chart/eoapi/test-unittest-values.yaml b/helm-chart/eoapi/test-unittest-values.yaml index cba5ad93..72890388 100644 --- a/helm-chart/eoapi/test-unittest-values.yaml +++ b/helm-chart/eoapi/test-unittest-values.yaml @@ -2,6 +2,8 @@ providerContext: "minikube" ingress: enabled: false + # used to set up NodePort in CI unittests on minikube + className: "neither" db: enabled: true raster: diff --git a/helm-chart/eoapi/values.yaml b/helm-chart/eoapi/values.yaml index e299285a..6235af4d 100644 --- a/helm-chart/eoapi/values.yaml +++ b/helm-chart/eoapi/values.yaml @@ -79,7 +79,7 @@ raster: enabled: true image: name: ghcr.io/stac-utils/titiler-pgstac - tag: uvicorn-latest + tag: uvicorn-0.4.1 settings: timeout: 10 # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ @@ -140,7 +140,7 @@ vector: enabled: true image: name: ghcr.io/developmentseed/tipg - tag: uvicorn-latest + tag: uvicorn-0.2.0 settings: timeout: 10 # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ From 1998b5c57f1a6c395808c6d4aced7b34f5cd5357 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 08:33:59 -0700 Subject: [PATCH 08/22] add new ingressClass for testing123 --- helm-chart/eoapi/test-unittest-values.yaml | 2 +- helm-chart/eoapi/values.schema.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-chart/eoapi/test-unittest-values.yaml b/helm-chart/eoapi/test-unittest-values.yaml index 72890388..dd186d64 100644 --- a/helm-chart/eoapi/test-unittest-values.yaml +++ b/helm-chart/eoapi/test-unittest-values.yaml @@ -3,7 +3,7 @@ providerContext: "minikube" ingress: enabled: false # used to set up NodePort in CI unittests on minikube - className: "neither" + className: "testing123" db: enabled: true raster: diff --git a/helm-chart/eoapi/values.schema.json b/helm-chart/eoapi/values.schema.json index 6cb432f0..dc7708bb 100644 --- a/helm-chart/eoapi/values.schema.json +++ b/helm-chart/eoapi/values.schema.json @@ -76,7 +76,7 @@ "properties": { "className": { "type": "string", - "pattern": "^(alb|nginx|minikube)$" + "pattern": "^(alb|nginx|testing123)$" } } }, From 9a16e6e189c8d722153bdcc9afab9251c5c242b2 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 08:41:02 -0700 Subject: [PATCH 09/22] fix tests --- .github/workflows/tests/test_raster.py | 2 +- .github/workflows/tests/test_stac.py | 2 +- .github/workflows/tests/test_vector.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests/test_raster.py b/.github/workflows/tests/test_raster.py index 65b32fff..39951a93 100644 --- a/.github/workflows/tests/test_raster.py +++ b/.github/workflows/tests/test_raster.py @@ -2,7 +2,7 @@ import httpx -raster_endpoint = "http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/raster" +raster_endpoint="http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/raster" def test_raster_api(): diff --git a/.github/workflows/tests/test_stac.py b/.github/workflows/tests/test_stac.py index 7c5204aa..7c125c96 100644 --- a/.github/workflows/tests/test_stac.py +++ b/.github/workflows/tests/test_stac.py @@ -2,7 +2,7 @@ import httpx -stac_endpoint = "http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/stac/" +stac_endpoint="http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/stac/" def test_stac_api(): diff --git a/.github/workflows/tests/test_vector.py b/.github/workflows/tests/test_vector.py index a57e5026..872d614b 100644 --- a/.github/workflows/tests/test_vector.py +++ b/.github/workflows/tests/test_vector.py @@ -2,7 +2,7 @@ import httpx -vector_endpoint = "http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/vector/" +vector_endpoint="http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/vector/" def test_vector_api(): From 156a9d1d02e688b1236c79b50793ee97e83affc0 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 10:52:35 -0700 Subject: [PATCH 10/22] try with helm version --- .github/workflows/helm-tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 621f0c97..68da9700 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -30,8 +30,9 @@ jobs: - uses: actions/checkout@v3 - uses: azure/setup-helm@v3 -# with: -# github-token: ${{ secrets.GITHUB_TOKEN }} + with: + helm-version: v3.8.2 + github-token: ${{ secrets.GITHUB_TOKEN }} - name: start minikube uses: medyagh/setup-minikube@master From 9354d785bdde26c2aed0bdd28fa269529b0df28c Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 11:04:33 -0700 Subject: [PATCH 11/22] try again --- .github/workflows/helm-tests.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 68da9700..74185427 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -83,8 +83,8 @@ jobs: cd helm-chart/eoapi kubectl apply -f eoapi-manfests.yaml - - name: sleep for 5s seconds while services boot - run: sleep 5s + - name: sleep for 10s seconds while services boot + run: sleep 10s shell: bash - name: restart the services @@ -97,20 +97,17 @@ jobs: kubectl get pod | grep '^stac-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} - - name: sleep for 10s seconds while services restart - run: sleep 10s + - name: sleep for 15s seconds while services restart + run: sleep 15s shell: bash - name: test service urls run: | minikube service list -# echo "------------------opening the service------------------" -# URL=$(minikube service vector --url) -# curl -XGET "$URL/collections" -# URL=$(minikube service stac --url) -# curl -XGET "$URL" -# URL=$(minikube service raster --url) -# curl -XGET "$URL" + echo "------------------opening the service------------------" + curl -XGET "$(minikube service vector --url)" + curl -XGET "$(minikube service stac --url)" + curl -XGET "$(minikube service raster --url)" # - name: setup upterm session # uses: lhotari/action-upterm@v1 From d66f1c8f75c1036a3d1c90f215e72fb3be2d11eb Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 11:09:38 -0700 Subject: [PATCH 12/22] do the ssh thing --- .github/workflows/helm-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 74185427..6dcc37db 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -101,6 +101,9 @@ jobs: run: sleep 15s shell: bash + - name: setup upterm session + uses: lhotari/action-upterm@v1 + - name: test service urls run: | minikube service list @@ -109,9 +112,6 @@ jobs: curl -XGET "$(minikube service stac --url)" curl -XGET "$(minikube service raster --url)" -# - name: setup upterm session -# uses: lhotari/action-upterm@v1 - - name: install python unit-test dependencies run: | python -m pip install pytest httpx From ea74b7f2b88d31a9f39580045810181d79733db1 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 11:29:49 -0700 Subject: [PATCH 13/22] change resources for unittests --- .github/workflows/helm-tests.yml | 12 +++++------ README.md | 9 ++++++++ helm-chart/eoapi/test-unittest-values.yaml | 24 ++++++++++++++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 6dcc37db..d991cd35 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -89,7 +89,7 @@ jobs: - name: restart the services run: | - # usually if a pod fails b/c the DB isn't up yet it will go into CrashLoopBack state + # usually if a pod fails b/c the DB isn't up yet it will go into CrashLoopBackOff state # and try to restart itself then on the second try be able to connect # minikube doesn't seem to do this by default so we force it to happen by deleting the `/pod` # and therefore the deployment will restart the pod @@ -97,13 +97,10 @@ jobs: kubectl get pod | grep '^stac-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} - - name: sleep for 15s seconds while services restart - run: sleep 15s + - name: sleep for 20s seconds while services restart + run: sleep 20s shell: bash - - name: setup upterm session - uses: lhotari/action-upterm@v1 - - name: test service urls run: | minikube service list @@ -112,6 +109,9 @@ jobs: curl -XGET "$(minikube service stac --url)" curl -XGET "$(minikube service raster --url)" +# - name: setup upterm session +# uses: lhotari/action-upterm@v1 + - name: install python unit-test dependencies run: | python -m pip install pytest httpx diff --git a/README.md b/README.md index 1df72fd9..187a2d9a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,14 @@ # k8s-eoapi +

+ + Test + + + Downloads + +

+ --- ## Getting Started diff --git a/helm-chart/eoapi/test-unittest-values.yaml b/helm-chart/eoapi/test-unittest-values.yaml index dd186d64..a7c869fb 100644 --- a/helm-chart/eoapi/test-unittest-values.yaml +++ b/helm-chart/eoapi/test-unittest-values.yaml @@ -11,13 +11,37 @@ raster: image: name: local/titiler tag: latest + settings: + resources: + limits: + cpu: "256m" + memory: "512Mi" + requests: + cpu: "256m" + memory: "512Mi" stac: enabled: true image: name: local/stac tag: latest + settings: + resources: + limits: + cpu: "256m" + memory: "512Mi" + requests: + cpu: "256m" + memory: "512Mi" vector: enabled: true image: name: local/tipg tag: latest + settings: + resources: + limits: + cpu: "256m" + memory: "512Mi" + requests: + cpu: "256m" + memory: "512Mi" From 0c95e09a40c37ac4d37b5efd54275b242535289b Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 11:41:57 -0700 Subject: [PATCH 14/22] service urls --- .github/workflows/helm-tests.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index d991cd35..be36b975 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -89,12 +89,11 @@ jobs: - name: restart the services run: | - # usually if a pod fails b/c the DB isn't up yet it will go into CrashLoopBackOff state - # and try to restart itself then on the second try be able to connect - # minikube doesn't seem to do this by default so we force it to happen by deleting the `/pod` - # and therefore the deployment will restart the pod - kubectl get pod | grep '^vector-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} - kubectl get pod | grep '^stac-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} + # usually if a pod fails b/c the DB isn't up yet it will go into CrashLoopBackOff state, so bounce them + #kubectl get pod | grep '^vector-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} + kubectl rollout restart deploy/vector + #kubectl get pod | grep '^stac-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} + kubectl rollout restart deploy/v kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} - name: sleep for 20s seconds while services restart @@ -104,10 +103,10 @@ jobs: - name: test service urls run: | minikube service list - echo "------------------opening the service------------------" - curl -XGET "$(minikube service vector --url)" - curl -XGET "$(minikube service stac --url)" - curl -XGET "$(minikube service raster --url)" +# echo "------------------opening the service------------------" +# curl -XGET "$(minikube service vector --url)" +# curl -XGET "$(minikube service stac --url)" +# curl -XGET "$(minikube service raster --url)" # - name: setup upterm session # uses: lhotari/action-upterm@v1 From 201d13405af99ffad3c5c19bb04559ab6c4df602 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 12:08:18 -0700 Subject: [PATCH 15/22] rollout restart --- .github/workflows/helm-tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index be36b975..f8cbd58a 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -93,8 +93,9 @@ jobs: #kubectl get pod | grep '^vector-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} kubectl rollout restart deploy/vector #kubectl get pod | grep '^stac-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} - kubectl rollout restart deploy/v - kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} + kubectl rollout restart deploy/stac + #kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} + kubectl rollout restart deploy/raster - name: sleep for 20s seconds while services restart run: sleep 20s From 55625aae0778477c3b45f9d6d8a3c87e8e8056d2 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 12:16:34 -0700 Subject: [PATCH 16/22] ssh again --- .github/workflows/helm-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index f8cbd58a..889b4f60 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -109,8 +109,8 @@ jobs: # curl -XGET "$(minikube service stac --url)" # curl -XGET "$(minikube service raster --url)" -# - name: setup upterm session -# uses: lhotari/action-upterm@v1 + - name: setup upterm session + uses: lhotari/action-upterm@v1 - name: install python unit-test dependencies run: | From 75c83ebd0c4db302fc21a1fa11c3a7973cfbb425 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 12:31:50 -0700 Subject: [PATCH 17/22] try sleeping until services ready --- .github/workflows/helm-tests.yml | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 889b4f60..874c314a 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -83,9 +83,9 @@ jobs: cd helm-chart/eoapi kubectl apply -f eoapi-manfests.yaml - - name: sleep for 10s seconds while services boot - run: sleep 10s + - name: sleep for 20s seconds while services boot shell: bash + run: sleep 20s - name: restart the services run: | @@ -97,9 +97,21 @@ jobs: #kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} kubectl rollout restart deploy/raster - - name: sleep for 20s seconds while services restart - run: sleep 20s - shell: bash + - name: sleep until services ready + run: | + sleep 10s + while [[ -z $(kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl log {} | grep "startup complete" | head -n 1) ]]; do + echo "still waiting for raster service to start... + sleep 1 + done + while [[ -z $(kubectl get pod | grep '^vector-.*$' | cut -d' ' -f1 | xargs -I{} kubectl log {} | grep "startup complete" | head -n 1) ]]; do + echo "still waiting for vector service to start... + sleep 1 + done + while [[ -z $(kubectl get pod | grep '^stac-.*$' | cut -d' ' -f1 | xargs -I{} kubectl log {} | grep "startup complete" | head -n 1) ]]; do + echo "still waiting for stac service to start... + sleep 1 + done - name: test service urls run: | @@ -109,9 +121,6 @@ jobs: # curl -XGET "$(minikube service stac --url)" # curl -XGET "$(minikube service raster --url)" - - name: setup upterm session - uses: lhotari/action-upterm@v1 - - name: install python unit-test dependencies run: | python -m pip install pytest httpx From 31dfba9d49bce3a22d9f00852ccd14507d19e969 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 12:36:45 -0700 Subject: [PATCH 18/22] logs not log --- .github/workflows/helm-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 874c314a..211efe07 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -100,15 +100,15 @@ jobs: - name: sleep until services ready run: | sleep 10s - while [[ -z $(kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl log {} | grep "startup complete" | head -n 1) ]]; do + while [[ -z $(kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "startup complete" | head -n 1) ]]; do echo "still waiting for raster service to start... sleep 1 done - while [[ -z $(kubectl get pod | grep '^vector-.*$' | cut -d' ' -f1 | xargs -I{} kubectl log {} | grep "startup complete" | head -n 1) ]]; do + while [[ -z $(kubectl get pod | grep '^vector-.*$' | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "startup complete" | head -n 1) ]]; do echo "still waiting for vector service to start... sleep 1 done - while [[ -z $(kubectl get pod | grep '^stac-.*$' | cut -d' ' -f1 | xargs -I{} kubectl log {} | grep "startup complete" | head -n 1) ]]; do + while [[ -z $(kubectl get pod | grep '^stac-.*$' | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "startup complete" | head -n 1) ]]; do echo "still waiting for stac service to start... sleep 1 done From 270b1dbf171a7e0a2029c51e7e3f565c1c335947 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 12:45:18 -0700 Subject: [PATCH 19/22] better services waiting --- .github/workflows/helm-tests.yml | 15 +++++++++------ README.md | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 211efe07..24f39ab4 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -100,18 +100,21 @@ jobs: - name: sleep until services ready run: | sleep 10s - while [[ -z $(kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "startup complete" | head -n 1) ]]; do - echo "still waiting for raster service to start... + while [[ -z "$(kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "startup complete" | head -n 1)" ]]; do + echo "still waiting for raster service to start..." sleep 1 done - while [[ -z $(kubectl get pod | grep '^vector-.*$' | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "startup complete" | head -n 1) ]]; do - echo "still waiting for vector service to start... + echo "raster service has started, moving on..." + while [[ -z "$(kubectl get pod | grep '^vector-.*$' | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "startup complete" | head -n 1)" ]]; do + echo "still waiting for vector service to start..." sleep 1 done - while [[ -z $(kubectl get pod | grep '^stac-.*$' | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "startup complete" | head -n 1) ]]; do - echo "still waiting for stac service to start... + echo "vector service has started, moving on..." + while [[ -z "$(kubectl get pod | grep '^stac-.*$' | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "startup complete" | head -n 1)" ]]; do + echo "still waiting for stac service to start..." sleep 1 done + echo "all services have started, moving on..." - name: test service urls run: | diff --git a/README.md b/README.md index 187a2d9a..0c429709 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@

- Test + Test Downloads From 76ae7c9360da92a9ec1d1e5b34fa2a97be1df60a Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 12:45:43 -0700 Subject: [PATCH 20/22] better wait for services --- .github/workflows/helm-tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 24f39ab4..2d52f9af 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -97,9 +97,8 @@ jobs: #kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} kubectl rollout restart deploy/raster - - name: sleep until services ready - run: | sleep 10s + while [[ -z "$(kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "startup complete" | head -n 1)" ]]; do echo "still waiting for raster service to start..." sleep 1 From 703842f4e72fd6d2c007e30697931beab66b78ef Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 12:51:42 -0700 Subject: [PATCH 21/22] try again --- .github/workflows/helm-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 2d52f9af..86195e29 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -115,6 +115,8 @@ jobs: done echo "all services have started, moving on..." + sleep 10s + - name: test service urls run: | minikube service list From 409b53dc122dd0e9cda6a0d94309e5bdfde58743 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 28 Jun 2023 12:58:03 -0700 Subject: [PATCH 22/22] this is it I can feel it --- .github/workflows/helm-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 86195e29..2660313c 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -134,17 +134,17 @@ jobs: # first substitute test enpoints in the test files # TODO: pytest should be able to have a pattern for injection here but htis is quicker URL=$(minikube service vector --url) - sed -i "s|vector_endpoint\=.*$|vector_endpoint\='$URL/vector'|g" .github/workflows/tests/test_vector.py + sed -i "s|vector_endpoint\=.*$|vector_endpoint\='$URL'|g" .github/workflows/tests/test_vector.py head -n 5 .github/workflows/tests/test_vector.py pytest .github/workflows/tests/test_vector.py URL=$(minikube service stac --url) - sed -i "s|stac_endpoint\=.*$|stac_endpoint\='$URL/stac'|g" .github/workflows/tests/test_stac.py + sed -i "s|stac_endpoint\=.*$|stac_endpoint\='$URL'|g" .github/workflows/tests/test_stac.py head -n 5 .github/workflows/tests/test_stac.py pytest .github/workflows/tests/test_stac.py URL=$(minikube service raster --url) - sed -i "s|raster_endpoint\=.*$|raster_endpoint\='$URL/raster'|g" .github/workflows/tests/test_raster.py + sed -i "s|raster_endpoint\=.*$|raster_endpoint\='$URL'|g" .github/workflows/tests/test_raster.py head -n 5 .github/workflows/tests/test_raster.py pytest .github/workflows/tests/test_raster.py