Feat/k3s tests

.github/workflows/helm-tests.yml

@@ -1,4 +1,4 @@
-name: CI - GKE
+name: CI
 
 on:
   push:
@@ -30,14 +30,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: debianmaster/actions-k3s@master
-        id: k3s
-        with:
-          version: 'latest'
-      - uses: azure/setup-helm@v3
+
+      - name: Start a local k3s cluster
+        uses: jupyterhub/action-k3s-helm@v4
         with:
-          version: v3.8.2
-          token: ${{ secrets.GITHUB_TOKEN }}
+          # See available:
+          # - k3s release channels at https://github.com/k3s-io/k3s/blob/HEAD/channel.yaml
+          # - k3s versions at https://github.com/k3s-io/k3s/tags
+          # - helm versions at https://github.com/helm/helm/tags
+          k3s-channel: latest
+          helm-version: v3.8.2
+          metrics-enabled: false
+          docker-enabled: true
 
       - name: last commit sha if PR
         if: ${{ github.event_name == 'pull_request' }}
@@ -58,56 +62,46 @@ jobs:
           SALT=$(echo "${RANDOM}${RANDOM}${RANDOM}" | cut -c1-3)
           echo "RELEASE_NAME=eoapi$COMMITSHA$SALT" >> $GITHUB_ENV
 
-      - name: Install k3s
+      - name: helm install crunchydata postgres operator
         run: |
-          curl -sfL https://get.k3s.io | sh -
-          k3s kubectl wait --for=condition=Ready nodes --all --timeout=120s
+          helm upgrade --install --set disable_check_for_upgrades=true pgo oci://registry.developers.crunchydata.com/crunchydata/pgo
 
       - name: helm render/install eoapi templates
         run: |
-          export PGUSER=username
-          export POSTGRES_USER=username
-          export PGPASSWORD=password
-          export POSTGRES_PASSWORD=password
           export GITSHA='${{github.sha}}'
 
           cd helm-chart
 
           helm install $RELEASE_NAME \
-            --namespace eoapitest \
-            --create-namespace \
-            --set db.settings.secrets.POSTGRES_HOST=pgstac-$RELEASE_NAME \
-            --set db.settings.secrets.POSTGRES_HOST_READER=pgstac-$RELEASE_NAME \
-            --set db.settings.secrets.POSTGRES_HOST_WRITER=pgstac-$RELEASE_NAME \
-            --set db.settings.secrets.PGUSER=$PGUSER \
-            --set db.settings.secrets.POSTGRES_USER=$POSTGRES_USER \
-            --set db.settings.secrets.PGPASSWORD=$PGPASSWORD \
-            --set db.settings.secrets.POSTGRES_PASSWORD=$POSTGRES_PASSWORD \
+            --namespace default \
             -f ./eoapi/values.yaml \
-            -f ./eoapi/test-unittest-values.yaml \
+            -f ./eoapi/test-k3s-unittest-values.yaml \
             ./eoapi
 
       - name: sleep for 10s seconds while services boot
         shell: bash
         run: sleep 10s
 
+#      - name: Setup upterm session
+#        uses: lhotari/action-upterm@v1
+
       - id: watchservices
         name: watch services boot
         timeout-minutes: 10
         continue-on-error: true
         run: |
-          k3s kubectl config set-context --current --namespace=eoapitest
-          while [[ -z "$(k3s kubectl get pod  | grep "^raster-$RELEASE_NAME-.*$" | cut -d' ' -f1 | xargs -I{} k3s kubectl logs pod/{} | grep "GET /.*/healthz" | head -n 1)" ]]; do
+          kubectl config set-context --current --namespace=default
+          while [[ -z "$(kubectl get pod  | grep "^raster-$RELEASE_NAME-.*$" | cut -d' ' -f1 | xargs -I{}  kubectl logs pod/{} | grep "GET /.*/healthz" | head -n 1)" ]]; do
             echo "still waiting for raster service to start..."
             sleep 1
           done
           echo "raster service has started, moving on..."
-          while [[ -z "$(k3s kubectl get pod  | grep "^vector-$RELEASE_NAME-.*$" | cut -d' ' -f1 | xargs -I{} k3s kubectl logs pod/{} | grep "GET /healthz" | head -n 1)" ]]; do
+          while [[ -z "$(kubectl get pod  | grep "^vector-$RELEASE_NAME-.*$" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "GET /healthz" | head -n 1)" ]]; do
             echo "still waiting for vector service to start..."
             sleep 1
           done
           echo "vector service has started, moving on..."
-          while [[ -z "$(k3s kubectl get pod  | grep "^stac-$RELEASE_NAME-.*$" | cut -d' ' -f1 | xargs -I{} k3s kubectl logs pod/{} | grep "GET /_mgmt/ping" | head -n 1)" ]]; do
+          while [[ -z "$(kubectl get pod  | grep "^stac-$RELEASE_NAME-.*$" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "GET /_mgmt/ping" | head -n 1)" ]]; do
             echo "still waiting for stac service to start..."
             sleep 1
           done
@@ -124,15 +118,16 @@ jobs:
         run: |
           python -m pip install pytest httpx
 
-#      - name: Setup upterm session
-#        uses: lhotari/action-upterm@v1
-
       - name: run the tests
         id: testrunner
         continue-on-error: true
         run: |
-          k3s kubectl config set-context --current --namespace=eoapitest
-          PUBLICIP='http://'$(k3s kubectl -n ingress-nginx get svc/ingress-nginx-controller -o jsonpath='{.spec.loadBalancerIP}')
+          kubectl get svc --all-namespaces
+          kubectl get ingress --all-namespaces -o jsonpath='{range .items[0]}kubectl describe ingress {.metadata.name} -n {.metadata.namespace}{end}' | sh
+          kubectl get middleware.traefik.io --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers | while read -r namespace name; do kubectl describe middleware.traefik.io "$name" -n "$namespace"; done
+
+          kubectl config set-context --current --namespace=default
+          PUBLICIP='http://'$(kubectl -n kube-system get svc traefik -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
           echo '#################################'
           echo vector=$PUBLICIP/vector$RELEASE_NAME
           echo stac=$PUBLICIP/stac$RELEASE_NAME
@@ -141,15 +136,15 @@ jobs:
 
           sed -i "s|vector_endpoint\=.*$|vector_endpoint\='$PUBLICIP/vector$RELEASE_NAME'|g" .github/workflows/tests/test_vector.py
           head -n 5 .github/workflows/tests/test_vector.py
-          pytest .github/workflows/tests/test_vector.py
+          pytest .github/workflows/tests/test_vector.py || kubectl logs svc/vector-$RELEASE_NAME
 
           sed -i "s|stac_endpoint\=.*$|stac_endpoint\='$PUBLICIP/stac$RELEASE_NAME'|g" .github/workflows/tests/test_stac.py
           head -n 5 .github/workflows/tests/test_stac.py
           pytest .github/workflows/tests/test_stac.py
 
-          sed -i "s|raster_endpoint\=.*$|raster_endpoint\='$PUBLICIP/raster$RELEASE_NAME'|g" .github/workflows/tests/test_raster.py
-          head -n 5 .github/workflows/tests/test_raster.py
-          pytest .github/workflows/tests/test_raster.py
+#          sed -i "s|raster_endpoint\=.*$|raster_endpoint\='$PUBLICIP/raster$RELEASE_NAME'|g" .github/workflows/tests/test_raster.py
+#          head -n 5 .github/workflows/tests/test_raster.py
+#          pytest .github/workflows/tests/test_raster.py
 
       - name: error if tests failed
         if: steps.testrunner.outcome == 'failure'
@@ -164,11 +159,9 @@ jobs:
 
   integration-tests-gcp:
     # run on:
-    #  - all pushes to specified branch(es)
     #  - a PR was just labeled 'test-integration'
     #  - a PR with 'test-integration' label was opened, reopened, or synchronized
     if: |
-      github.event_name == 'push' ||
       github.event.label.name == 'test-integration-gcp' ||
       contains( github.event.pull_request.labels.*.name, 'test-integration-gcp')
     permissions:
@@ -220,6 +213,10 @@ jobs:
           gcloud components install gke-gcloud-auth-plugin
           gcloud container clusters get-credentials k8seed-labs-cluster --zone us-central1-f
 
+      - name: helm install crunchydata postgres operator
+        run: |
+          helm upgrade --install --set disable_check_for_upgrades=true pgo oci://registry.developers.crunchydata.com/crunchydata/pgo
+
       - name: helm render/install eoapi templates
         run: |
           export GITSHA='${{github.sha}}'
@@ -230,13 +227,16 @@ jobs:
             --namespace $RELEASE_NAME \
             --create-namespace \
             -f ./eoapi/values.yaml \
-            -f ./eoapi/test-unittest-values.yaml \
+            -f ./eoapi/test-gcp-unittest-values.yaml \
             ./eoapi
 
       - name: sleep for 10s seconds while services boot
         shell: bash
         run: sleep 10s
 
+#      - name: Setup upterm session
+#        uses: lhotari/action-upterm@v1
+
       - id: watchservices
         name: watch services boot
         timeout-minutes: 10
@@ -274,9 +274,6 @@ jobs:
         run: |
           python -m pip install pytest httpx
 
-#      - name: Setup upterm session
-#        uses: lhotari/action-upterm@v1
-
       - name: run the tests
         id: testrunner
         continue-on-error: true

helm-chart/eoapi/templates/_helpers.tpl

@@ -182,3 +182,13 @@ validate:
 
 {{- end -}}
 
+{{/*
+validate:
+that you can only use traefik as ingress when `testing=true`
+*/}}
+{{- define "eoapi.validateTraefik" -}}
+{{- if and (not .Values.testing) (eq .Values.ingress.className "traefik") $ -}}
+  {{- fail "you cannot use traefik yet outside of testing" -}}
+{{- end -}}
+
+{{- end -}}

helm-chart/eoapi/templates/services/deployment.yaml

@@ -29,9 +29,9 @@ spec:
         name: {{ $serviceName }}
         command:
           {{- toYaml (index $v "command") | nindent 10 }}
-          {{- if (and ($.Values.ingress.className) (eq $.Values.ingress.className "nginx") (not $.Values.testing)) }}
+          {{- if (and ($.Values.ingress.className) (or (eq $.Values.ingress.className "nginx") (eq $.Values.ingress.className "traefik")) (not $.Values.testing)) }}
           - "--root-path=/{{ $serviceName }}"
-          {{- else if (and ($.Values.ingress.className) (eq $.Values.ingress.className "nginx") ($.Values.testing)) }}
+          {{- else if (and ($.Values.ingress.className) (or (eq $.Values.ingress.className "nginx") (eq $.Values.ingress.className "traefik")) ($.Values.testing)) }}
           - "--root-path=/{{ $serviceName }}{{ $.Release.Name }}"
           {{- end }}{{/* needed for proxies and path rewrites on NLB */}}
         livenessProbe:

helm-chart/eoapi/templates/services/ingress-traefik.yaml

@@ -0,0 +1,123 @@
+{{- include "eoapi.validateTraefik" . -}}
+{{- if and .Values.ingress.enabled (eq .Values.ingress.className "traefik") }}
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: path-rewrite-middleware-{{ $.Release.Name }}
+  namespace: {{ $.Release.Namespace }}
+spec:
+  replacePathRegex:
+    regex: "^/(raster|vector|stac){{ $.Release.Name }}(/|$)(.*)"
+    replacement: "$1{{ $.Release.Name }}/$3"
+{{- end }}
+---
+{{- if (and (.Values.ingress.enabled) (eq .Values.ingress.className "traefik")) }}
+{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" $.Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: traefik-service-ingress-shared-{{ $.Release.Name }}
+  labels:
+    app: traefiksharedingress
+  annotations:
+    traefik.ingress.kubernetes.io/router.entrypoints: web
+    traefik.ingress.kubernetes.io/router.middlewares: "{{ $.Release.Namespace }}-path-rewrite-middleware-{{ $.Release.Name }}@kubernetescrd"
+    {{- if (and (.Values.ingress.tls.enabled) (.Values.ingress.tls.certManager)) }}
+    cert-manager.io/issuer: {{ .Values.ingress.tls.certManagerIssuer }}
+    {{- end }}
+{{- if (.Values.ingress.annotations) }}
+{{ toYaml .Values.ingress.annotations | indent 4 }}
+{{- end }}
+spec:
+  {{- if (and (.Values.ingress.className) (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion)) }}
+  ingressClassName: {{ .Values.ingress.className }}
+  {{- end }}
+  rules:
+    - http:
+        paths:
+          {{- range $serviceName, $v := .Values }}
+          {{- if (or (eq $serviceName "raster") (eq $serviceName "stac") (eq $serviceName "vector")) }}
+          {{- if (and (index $v "enabled") (not $.Values.testing)) }}
+          - pathType: Prefix
+            path: "/{{ $serviceName }}"
+            backend:
+              service:
+                name: {{ $serviceName }}
+                port:
+                  number: {{ $.Values.service.port }}
+          {{- else if (and (index $v "enabled") ($.Values.testing)) }}
+          - pathType: Prefix
+            path: "/{{ $serviceName }}{{ $.Release.Name }}"
+            backend:
+              service:
+                name: {{ $serviceName }}-{{ $.Release.Name }}
+                port:
+                  number: {{ $.Values.service.port }}
+          {{- end }}{{/* END: if index $v "enabled" */}}
+          {{- end }}{{/* END: if (or (eq $serviceName "raster") (eq $serviceName "stac") (eq $serviceName "vector")) */}}
+          {{- end }}{{/* END: range $serviceName, $v := .Values*/}}
+          {{- if (and (not $.Values.testing) (.Values.docServer.enabled)) }}
+          - pathType: Prefix
+            path: /(/|$)
+            backend:
+              service:
+                name: doc-server
+                port:
+                  number: 80
+          {{- end }}
+      {{- if (and (not .Values.testing) (.Values.ingress.host)) }}
+      host: {{ .Values.ingress.host }}
+      {{- end }}
+  {{- if (and (not .Values.testing) (.Values.ingress.host) (.Values.ingress.tls.enabled)) }}
+  tls:
+    - hosts:
+        - {{ .Values.ingress.host }}
+      secretName: {{ .Values.ingress.tls.secretName }}
+  {{- end }}
+---
+{{- if (and (.Values.ingress.tls.enabled) (.Values.ingress.tls.certManager)) }}
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: letsencrypt-staging
+spec:
+  acme:
+    # The ACME server URL
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
+    # Email address used for ACME registration
+    email: {{ .Values.ingress.tls.certManagerEmail }}
+    # Name of a secret used to store the ACME account private key
+    privateKeySecretRef:
+      name: letsencrypt-staging-key
+    # Enable the HTTP-01 challenge provider
+    solvers:
+      - http01:
+          ingress:
+            class: traefik
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: letsencrypt-prod
+spec:
+  acme:
+    # The ACME server URL
+    server: https://acme-v02.api.letsencrypt.org/directory
+    # Email address used for ACME registration
+    email: {{ .Values.ingress.tls.certManagerEmail }}
+    # Name of a secret used to store the ACME account private key
+    privateKeySecretRef:
+      name: letsencrypt-prod-key
+    # Enable the HTTP-01 challenge provider
+    solvers:
+      - http01:
+          ingress:
+            class: traefik
+{{- end }}
+{{/* END: if .Values.ingress.className "traefik" */}}
+{{- end }}