diff --git a/.github/workflows/ci.bkup b/.github/workflows/ci.bkup deleted file mode 100644 index 8fa11198..00000000 --- a/.github/workflows/ci.bkup +++ /dev/null @@ -1,126 +0,0 @@ -name: CI - -# On every pull request, but only on push to main -on: - push: - branches: - - main - tags: - - '*' - pull_request: - -jobs: - helm-tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: d3adb5/helm-unittest-action@v2 - with: - helm-version: v3.8.2 - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: | - cd infrastructure/k8s/helm-chart - helm unittest eoapi -f 'tests/*.yaml' -v eoapi/required-test-values.yaml - - - publish-docker: - needs: [helm-tests] - if: github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to Github - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set tag version - id: tag - run: | - echo "version=${GITHUB_REF#refs/*/}" - echo "version=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - - ############################################################################# - # RASTER - - name: RASTER - Build and push latest - if: github.ref == 'refs/heads/main' - uses: docker/build-push-action@v2 - with: - platforms: linux/amd64 - context: . - file: dockerfiles/Dockerfile.raster-uvicorn - push: true - tags: | - ghcr.io/developmentseed/eoapi-raster:latest - - - name: RASTER - Build and push tags - if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' - uses: docker/build-push-action@v2 - with: - platforms: linux/amd64 - context: . - file: dockerfiles/Dockerfile.raster-uvicorn - push: true - tags: | - ghcr.io/developmentseed/eoapi-raster:${{ steps.tag.outputs.tag }} - - ############################################################################# - # STAC - - name: STAC - Build and push latest - if: github.ref == 'refs/heads/main' - uses: docker/build-push-action@v2 - with: - context: . - file: dockerfiles/Dockerfile.stac-uvicorn - push: true - tags: | - ghcr.io/developmentseed/eoapi-stac:latest - - - name: STAC - Build and push tags - if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' - uses: docker/build-push-action@v2 - with: - context: . - file: dockerfiles/Dockerfile.stac-uvicorn - push: true - tags: | - ghcr.io/developmentseed/eoapi-stac:${{ steps.tag.outputs.tag }} - - ############################################################################# - # VECTOR - - name: VECTOR - Build and push latest - if: github.ref == 'refs/heads/main' - uses: docker/build-push-action@v2 - with: - context: . - file: dockerfiles/Dockerfile.vector-uvicorn - push: true - tags: | - ghcr.io/developmentseed/eoapi-vector:latest - - - name: VECTOR - Build and push tags - if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' - uses: docker/build-push-action@v2 - with: - context: . - file: dockerfiles/Dockerfile.vector-uvicorn - push: true - tags: | - ghcr.io/developmentseed/eoapi-vector:${{ steps.tag.outputs.tag }} - diff --git a/.github/workflows/deploy_mkdocs.bkup b/.github/workflows/deploy_mkdocs.bkup deleted file mode 100644 index 159aae18..00000000 --- a/.github/workflows/deploy_mkdocs.bkup +++ /dev/null @@ -1,34 +0,0 @@ -name: Publish docs via GitHub Pages - -on: - push: - branches: - - main - paths: - # Only rebuild website when docs have changed - - 'README.md' - - 'CHANGES.md' - - 'CONTRIBUTING.md' - - 'docs/**' - - '.github/workflows/deploy_mkdocs.yml' - -jobs: - build: - name: Deploy docs - runs-on: ubuntu-latest - steps: - - name: Checkout main - uses: actions/checkout@v3 - - - name: Set up Python 3.8 - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install mkdocs mkdocs-material pygments - - - name: Deploy docs - run: mkdocs gh-deploy --force -f docs/mkdocs.yml diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 1a6fb608..2660313c 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -30,6 +30,9 @@ jobs: - uses: actions/checkout@v3 - uses: azure/setup-helm@v3 + with: + helm-version: v3.8.2 + github-token: ${{ secrets.GITHUB_TOKEN }} - name: start minikube uses: medyagh/setup-minikube@master @@ -80,37 +83,47 @@ jobs: cd helm-chart/eoapi kubectl apply -f eoapi-manfests.yaml - - name: sleep for 5s seconds while services boot - run: sleep 5s + - name: sleep for 20s seconds while services boot shell: bash + run: sleep 20s - name: restart the services run: | - # usually if a pod fails b/c the DB isn't up yet it will go into CrashLoopBack state - # and try to restart itself then on the second try be able to connect - # minikube doesn't seem to do this by default so we force it to happen by deleting the `/pod` - # and therefore the deployment will restart the pod - kubectl get pod | grep '^vector-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} - kubectl get pod | grep '^stac-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} - kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} - - - name: sleep for 10s seconds while services restart - run: sleep 10s - shell: bash + # usually if a pod fails b/c the DB isn't up yet it will go into CrashLoopBackOff state, so bounce them + #kubectl get pod | grep '^vector-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} + kubectl rollout restart deploy/vector + #kubectl get pod | grep '^stac-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} + kubectl rollout restart deploy/stac + #kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl delete pod/{} + kubectl rollout restart deploy/raster + + sleep 10s + + while [[ -z "$(kubectl get pod | grep '^raster-.*$' | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "startup complete" | head -n 1)" ]]; do + echo "still waiting for raster service to start..." + sleep 1 + done + echo "raster service has started, moving on..." + while [[ -z "$(kubectl get pod | grep '^vector-.*$' | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "startup complete" | head -n 1)" ]]; do + echo "still waiting for vector service to start..." + sleep 1 + done + echo "vector service has started, moving on..." + while [[ -z "$(kubectl get pod | grep '^stac-.*$' | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "startup complete" | head -n 1)" ]]; do + echo "still waiting for stac service to start..." + sleep 1 + done + echo "all services have started, moving on..." + + sleep 10s - name: test service urls run: | minikube service list # echo "------------------opening the service------------------" -# URL=$(minikube service vector --url) -# curl -XGET "$URL/collections" -# URL=$(minikube service stac --url) -# curl -XGET "$URL" -# URL=$(minikube service raster --url) -# curl -XGET "$URL" - -# - name: setup upterm session -# uses: lhotari/action-upterm@v1 +# curl -XGET "$(minikube service vector --url)" +# curl -XGET "$(minikube service stac --url)" +# curl -XGET "$(minikube service raster --url)" - name: install python unit-test dependencies run: | @@ -121,17 +134,17 @@ jobs: # first substitute test enpoints in the test files # TODO: pytest should be able to have a pattern for injection here but htis is quicker URL=$(minikube service vector --url) - sed -i "s|vector_endpoint\=.*$|vector_endpoint\='$URL/vector'|g" .github/workflows/tests/test_vector.py + sed -i "s|vector_endpoint\=.*$|vector_endpoint\='$URL'|g" .github/workflows/tests/test_vector.py head -n 5 .github/workflows/tests/test_vector.py pytest .github/workflows/tests/test_vector.py URL=$(minikube service stac --url) - sed -i "s|stac_endpoint\=.*$|stac_endpoint\='$URL/stac'|g" .github/workflows/tests/test_stac.py + sed -i "s|stac_endpoint\=.*$|stac_endpoint\='$URL'|g" .github/workflows/tests/test_stac.py head -n 5 .github/workflows/tests/test_stac.py pytest .github/workflows/tests/test_stac.py URL=$(minikube service raster --url) - sed -i "s|raster_endpoint\=.*$|raster_endpoint\='$URL/raster'|g" .github/workflows/tests/test_raster.py + sed -i "s|raster_endpoint\=.*$|raster_endpoint\='$URL'|g" .github/workflows/tests/test_raster.py head -n 5 .github/workflows/tests/test_raster.py pytest .github/workflows/tests/test_raster.py diff --git a/.github/workflows/tests/test_raster.py b/.github/workflows/tests/test_raster.py index 65b32fff..39951a93 100644 --- a/.github/workflows/tests/test_raster.py +++ b/.github/workflows/tests/test_raster.py @@ -2,7 +2,7 @@ import httpx -raster_endpoint = "http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/raster" +raster_endpoint="http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/raster" def test_raster_api(): diff --git a/.github/workflows/tests/test_stac.py b/.github/workflows/tests/test_stac.py index 7c5204aa..7c125c96 100644 --- a/.github/workflows/tests/test_stac.py +++ b/.github/workflows/tests/test_stac.py @@ -2,7 +2,7 @@ import httpx -stac_endpoint = "http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/stac/" +stac_endpoint="http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/stac/" def test_stac_api(): diff --git a/.github/workflows/tests/test_vector.py b/.github/workflows/tests/test_vector.py index a57e5026..872d614b 100644 --- a/.github/workflows/tests/test_vector.py +++ b/.github/workflows/tests/test_vector.py @@ -2,7 +2,7 @@ import httpx -vector_endpoint = "http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/vector/" +vector_endpoint="http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/vector/" def test_vector_api(): diff --git a/README.md b/README.md index 55d8c372..0c429709 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,19 @@ # k8s-eoapi +
+ --- ## Getting Started -If you don't have a k8s cluster set up on AWS or GCP then follow an IAC guide below that is relevant to you. This helm chart hasn't been -tested on [minikube](https://github.com/kubernetes/minikube) yet: +If you don't have a k8s cluster set up on AWS or GCP then follow an IAC guide below that is relevant to you > ⓘ The helm chart in this repo assumes your cluster has a few third-party add-ons and controllers installed. So > it's in your best interest to read through the IAC guides to understand what those defaults are diff --git a/dockerfiles/Dockerfile.tipg b/dockerfiles/Dockerfile.tipg index 335b1755..5fe673b3 100644 --- a/dockerfiles/Dockerfile.tipg +++ b/dockerfiles/Dockerfile.tipg @@ -1 +1 @@ -FROM ghcr.io/developmentseed/tipg:uvicorn-latest \ No newline at end of file +FROM ghcr.io/developmentseed/tipg:uvicorn-0.2.0 \ No newline at end of file diff --git a/dockerfiles/Dockerfile.titiler b/dockerfiles/Dockerfile.titiler index cc8d497f..2deefc0b 100644 --- a/dockerfiles/Dockerfile.titiler +++ b/dockerfiles/Dockerfile.titiler @@ -1 +1 @@ -FROM ghcr.io/stac-utils/titiler-pgstac:uvicorn-latest \ No newline at end of file +FROM ghcr.io/stac-utils/titiler-pgstac:uvicorn-0.4.1 \ No newline at end of file diff --git a/helm-chart/eoapi/templates/db/deployment.yaml b/helm-chart/eoapi/templates/db/deployment.yaml index 76426a5a..bff11541 100644 --- a/helm-chart/eoapi/templates/db/deployment.yaml +++ b/helm-chart/eoapi/templates/db/deployment.yaml @@ -29,7 +29,7 @@ spec: name: pgstac-secrets ports: - containerPort: 5432 - {{- if not (eq $.Values.ingress.className "minikube") }} + {{- if not (eq .Values.providerContext "minikube") }} volumeMounts: - mountPath: /var/lib/postgresql/data name: pgstac-claim @@ -47,7 +47,7 @@ spec: ports: - containerPort: 6543 volumeMounts: - {{- if not (eq $.Values.ingress.className "minikube") }} + {{- if not (eq .Values.providerContext "minikube") }} - mountPath: /var/lib/postgresql/data name: pgstac-claim {{ end }} @@ -59,7 +59,7 @@ spec: name: initdb-sh-volume {{- end }} volumes: - {{- if not (eq $.Values.ingress.className "minikube") }} + {{- if not (eq .Values.providerContext "minikube") }} - name: pgstac-claim persistentVolumeClaim: claimName: pgstac-claim diff --git a/helm-chart/eoapi/templates/services/deployment.yaml b/helm-chart/eoapi/templates/services/deployment.yaml index 862930e2..3264617b 100644 --- a/helm-chart/eoapi/templates/services/deployment.yaml +++ b/helm-chart/eoapi/templates/services/deployment.yaml @@ -26,7 +26,7 @@ spec: spec: containers: - image: {{ index $v "image" "name" }}:{{ index $v "image" "tag" }} - {{- if (and ($.Values.ingress.className) (eq $.Values.ingress.className "minikube")) }} + {{- if (eq $.Values.providerContext "minikube") }} imagePullPolicy: Never {{- end }} name: {{ $serviceName }} diff --git a/helm-chart/eoapi/test-helm-values.yaml b/helm-chart/eoapi/test-helm-values.yaml index 285902aa..71ee356e 100644 --- a/helm-chart/eoapi/test-helm-values.yaml +++ b/helm-chart/eoapi/test-helm-values.yaml @@ -1,4 +1,6 @@ # this file is used with `helm-tests` job in CI +providerContext: "aws" + ingress: enabled: true diff --git a/helm-chart/eoapi/test-unittest-values.yaml b/helm-chart/eoapi/test-unittest-values.yaml index a2fff184..a7c869fb 100644 --- a/helm-chart/eoapi/test-unittest-values.yaml +++ b/helm-chart/eoapi/test-unittest-values.yaml @@ -1,8 +1,9 @@ # used in GH Actions `unit-tests.yml` - +providerContext: "minikube" ingress: enabled: false - className: "minikube" + # used to set up NodePort in CI unittests on minikube + className: "testing123" db: enabled: true raster: @@ -10,13 +11,37 @@ raster: image: name: local/titiler tag: latest + settings: + resources: + limits: + cpu: "256m" + memory: "512Mi" + requests: + cpu: "256m" + memory: "512Mi" stac: enabled: true image: name: local/stac tag: latest + settings: + resources: + limits: + cpu: "256m" + memory: "512Mi" + requests: + cpu: "256m" + memory: "512Mi" vector: enabled: true image: name: local/tipg tag: latest + settings: + resources: + limits: + cpu: "256m" + memory: "512Mi" + requests: + cpu: "256m" + memory: "512Mi" diff --git a/helm-chart/eoapi/values.schema.json b/helm-chart/eoapi/values.schema.json index 94b7bf5c..dc7708bb 100644 --- a/helm-chart/eoapi/values.schema.json +++ b/helm-chart/eoapi/values.schema.json @@ -2,6 +2,7 @@ "$schema": "http://json-schema.org/schema#", "type": "object", "required": [ + "providerContext", "db", "service", "gitSha" @@ -75,13 +76,17 @@ "properties": { "className": { "type": "string", - "pattern": "^(alb|nginx|minikube)$" + "pattern": "^(alb|nginx|testing123)$" } } }, "gitSha": { "type": "string", "pattern": "^.+$" + }, + "providerContext": { + "type": "string", + "pattern": "^(aws|minikube)$" } } } diff --git a/helm-chart/eoapi/values.yaml b/helm-chart/eoapi/values.yaml index 8474be79..6235af4d 100644 --- a/helm-chart/eoapi/values.yaml +++ b/helm-chart/eoapi/values.yaml @@ -59,6 +59,11 @@ db: PGPASSWORD: "" PGDATABASE: "postgis" +providerContext: "aws" +# 'aws' is only supported at the moment. +# our CI pipelines test against 'minikube' yet it hasn't been fully evaluated. +# 'gcp' will be coming shortly in the future + gitSha: "" service: @@ -74,7 +79,7 @@ raster: enabled: true image: name: ghcr.io/stac-utils/titiler-pgstac - tag: uvicorn-latest + tag: uvicorn-0.4.1 settings: timeout: 10 # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ @@ -135,7 +140,7 @@ vector: enabled: true image: name: ghcr.io/developmentseed/tipg - tag: uvicorn-latest + tag: uvicorn-0.2.0 settings: timeout: 10 # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/