diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index 745118b5..44ad8543 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -7,6 +7,10 @@ on: branches: [ "main" ] types: [ opened, reopened, synchronize, labeled ] +env: + HELM_VERSION: v3.15.2 + PGO_VERSION: 5.5.2 + jobs: helm-tests: runs-on: ubuntu-latest @@ -15,7 +19,7 @@ jobs: - uses: d3adb5/helm-unittest-action@v2 with: - helm-version: v3.15.2 + helm-version: ${{ env.HELM_VERSION }} github-token: ${{ secrets.GITHUB_TOKEN }} - run: | @@ -39,7 +43,7 @@ jobs: # - k3s versions at https://github.com/k3s-io/k3s/tags # - helm versions at https://github.com/helm/helm/tags k3s-channel: latest - helm-version: v3.8.2 + helm-version: ${{ env.HELM_VERSION }} metrics-enabled: false docker-enabled: true @@ -64,7 +68,11 @@ jobs: - name: helm install crunchydata postgres operator run: | - helm upgrade --install --set disable_check_for_upgrades=true pgo oci://registry.developers.crunchydata.com/crunchydata/pgo + helm upgrade --install \ + --set disable_check_for_upgrades=true \ + pgo \ + oci://registry.developers.crunchydata.com/crunchydata/pgo \ + --version ${{ env.PGO_VERSION }} - name: helm render/install eoapi templates run: | @@ -128,23 +136,20 @@ jobs: kubectl config set-context --current --namespace=default PUBLICIP='http://'$(kubectl -n kube-system get svc traefik -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + export VECTOR_ENDPOINT=$PUBLICIP/vector$RELEASE_NAME + export STAC_ENDPOINT=$PUBLICIP/stac$RELEASE_NAME + export RASTER_ENDPOINT=$PUBLICIP/raster$RELEASE_NAME + echo '#################################' - echo vector=$PUBLICIP/vector$RELEASE_NAME - echo stac=$PUBLICIP/stac$RELEASE_NAME - echo raster=$PUBLICIP/raster$RELEASE_NAME + echo $VECTOR_ENDPOINT + echo $STAC_ENDPOINT + echo $RASTER_ENDPOINT echo '#################################' - sed -i "s|vector_endpoint\=.*$|vector_endpoint\='$PUBLICIP/vector$RELEASE_NAME'|g" .github/workflows/tests/test_vector.py - head -n 5 .github/workflows/tests/test_vector.py pytest .github/workflows/tests/test_vector.py || kubectl logs svc/vector-$RELEASE_NAME - - sed -i "s|stac_endpoint\=.*$|stac_endpoint\='$PUBLICIP/stac$RELEASE_NAME'|g" .github/workflows/tests/test_stac.py - head -n 5 .github/workflows/tests/test_stac.py pytest .github/workflows/tests/test_stac.py - -# sed -i "s|raster_endpoint\=.*$|raster_endpoint\='$PUBLICIP/raster$RELEASE_NAME'|g" .github/workflows/tests/test_raster.py -# head -n 5 .github/workflows/tests/test_raster.py -# pytest .github/workflows/tests/test_raster.py + # TODO: fix raster tests + #pytest .github/workflows/tests/test_raster.py - name: error if tests failed if: steps.testrunner.outcome == 'failure' @@ -157,8 +162,9 @@ jobs: run: | helm uninstall $RELEASE_NAME - integration-tests-gcp: + gcp-integration-tests: # run on: + # - a PR was just labeled 'test-integration-gcp' # - a PR was just labeled 'test-integration' # - a PR with 'test-integration' label was opened, reopened, or synchronized if: | @@ -174,7 +180,7 @@ jobs: - uses: azure/setup-helm@v4 with: - version: v3.15.2 + version: ${{ env.HELM_VERSION }} #token: ${{ secrets.GITHUB_TOKEN }} - name: last commit sha if PR @@ -215,17 +221,23 @@ jobs: - name: helm install crunchydata postgres operator run: | - helm upgrade --install --set disable_check_for_upgrades=true pgo oci://registry.developers.crunchydata.com/crunchydata/pgo + helm upgrade --install \ + --set disable_check_for_upgrades=true \ + pgo \ + oci://registry.developers.crunchydata.com/crunchydata/pgo \ + --version ${{ env.PGO_VERSION }} - name: helm render/install eoapi templates run: | - export GITSHA='${{github.sha}}' + HOST='${{ env.RELEASE_NAME }}.k8s.labs.ds.io' + echo $HOST cd helm-chart helm install $RELEASE_NAME \ --namespace $RELEASE_NAME \ --create-namespace \ + --set ingress.host=$HOST \ -f ./eoapi/values.yaml \ -f ./eoapi/test-gcp-unittest-values.yaml \ ./eoapi @@ -279,41 +291,32 @@ jobs: continue-on-error: true run: | kubectl config set-context --current --namespace=$RELEASE_NAME - PUBLICIP='http://'$(kubectl -n ingress-nginx get svc/ingress-nginx-controller -o jsonpath='{.spec.loadBalancerIP}') + PUBLICIP='https://${{ env.RELEASE_NAME }}.k8s.labs.ds.io' + export VECTOR_ENDPOINT=$PUBLICIP/vector$RELEASE_NAME + export STAC_ENDPOINT=$PUBLICIP/stac$RELEASE_NAME + export RASTER_ENDPOINT=$PUBLICIP/raster$RELEASE_NAME + echo '#################################' - echo vector=$PUBLICIP/vector$RELEASE_NAME - echo stac=$PUBLICIP/stac$RELEASE_NAME - echo raster=$PUBLICIP/raster$RELEASE_NAME + echo $VECTOR_ENDPOINT + echo $STAC_ENDPOINT + echo $RASTER_ENDPOINT echo '#################################' - # first substitute test endpoints in the test files - # TODO: pytest should be able to have a pattern for injection here but moving fast - sed -i "s|vector_endpoint\=.*$|vector_endpoint\='$PUBLICIP/vector$RELEASE_NAME'|g" .github/workflows/tests/test_vector.py - head -n 5 .github/workflows/tests/test_vector.py pytest .github/workflows/tests/test_vector.py - - sed -i "s|stac_endpoint\=.*$|stac_endpoint\='$PUBLICIP/stac$RELEASE_NAME'|g" .github/workflows/tests/test_stac.py - head -n 5 .github/workflows/tests/test_stac.py pytest .github/workflows/tests/test_stac.py - # TODO: fix raster tests - #sed -i "s|raster_endpoint\=.*$|raster_endpoint\='$PUBLICIP/raster$RELEASE_NAME'|g" .github/workflows/tests/test_raster.py - #head -n 5 .github/workflows/tests/test_raster.py #pytest .github/workflows/tests/test_raster.py - - name: cleanup if tests faile + - name: cleanup if tests fail if: steps.testrunner.outcome == 'failure' run: | echo "The previous step failed or timed out. Running cleanup logic..." helm uninstall $RELEASE_NAME - kubectl delete ns/$RELEASE_NAME - # force GH action to show failed result exit 128 - - name: helm uinstall eoapi templates + - name: helm uninstall eoapi templates run: | helm uninstall $RELEASE_NAME - kubectl delete ns/$RELEASE_NAME diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fb4fab6e..4919d567 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,6 +5,9 @@ on: branches: - gh-pages +env: + HELM_VERSION: v3.15.2 + jobs: release: if: github.actor == 'ranchodeluxe' || github.actor == 'gcorradini' || github.actor == 'sunu' @@ -21,7 +24,7 @@ jobs: - uses: azure/setup-helm@v4 with: - version: v3.15.2 + version: ${{ env.HELM_VERSION }} #token: ${{ secrets.GITHUB_TOKEN }} - name: update gh-pages with content from main diff --git a/.github/workflows/tests/conftest.py b/.github/workflows/tests/conftest.py new file mode 100644 index 00000000..8179e62f --- /dev/null +++ b/.github/workflows/tests/conftest.py @@ -0,0 +1,17 @@ +import pytest +import os + + +@pytest.fixture(scope='session') +def raster_endpoint(): + return os.getenv('RASTER_ENDPOINT', "http://127.0.0.1/raster") + + +@pytest.fixture(scope='session') +def vector_endpoint(): + return os.getenv('VECTOR_ENDPOINT', "http://127.0.0.1/vector") + + +@pytest.fixture(scope='session') +def stac_endpoint(): + return os.getenv('STAC_ENDPOINT', "http://127.0.0.1/stac") diff --git a/.github/workflows/tests/test_raster.py b/.github/workflows/tests/test_raster.py index 442dba64..b483a126 100644 --- a/.github/workflows/tests/test_raster.py +++ b/.github/workflows/tests/test_raster.py @@ -1,15 +1,16 @@ """test EOapi.""" - import httpx +import os # better timeouts timeout = httpx.Timeout(15.0, connect=60.0) -client = httpx.Client(timeout=timeout) - -raster_endpoint="http://k8s-eoapi-ingressn-cd16d0ed58-580a04972ec20c3c.elb.us-west-1.amazonaws.com/raster" +if bool(os.getenv("IGNORE_SSL_VERIFICATION", False)): + client = httpx.Client(timeout=timeout, verify=False) +else: + client = httpx.Client(timeout=timeout) -def test_raster_api(): +def test_raster_api(raster_endpoint): """test api.""" resp = client.get( f"{raster_endpoint}/healthz", headers={"Accept-Encoding": "br, gzip"} @@ -18,7 +19,7 @@ def test_raster_api(): assert resp.headers["content-type"] == "application/json" -def test_mosaic_api(): +def test_mosaic_api(raster_endpoint): """test mosaic.""" query = {"collections": ["noaa-emergency-response"], "filter-lang": "cql-json"} resp = client.post(f"{raster_endpoint}/searches/register", json=query) @@ -55,7 +56,7 @@ def test_mosaic_api(): assert "content-encoding" not in resp.headers -def test_mosaic_collection_api(): +def test_mosaic_collection_api(raster_endpoint): """test mosaic collection.""" resp = client.get( f"{raster_endpoint}/collections/noaa-emergency-response/-85.6358,36.1624/assets" @@ -85,7 +86,7 @@ def test_mosaic_collection_api(): assert "content-encoding" not in resp.headers -def test_mosaic_search(): +def test_mosaic_search(raster_endpoint): """test mosaic.""" # register some fake mosaic searches = [ @@ -203,7 +204,7 @@ def test_mosaic_search(): assert "owner" in resp.json()["searches"][0]["search"]["metadata"] -def test_item(): +def test_item(raster_endpoint): """test stac endpoints.""" resp = client.get( f"{raster_endpoint}/collections/noaa-emergency-response/items/20200307aC0853300w361200/assets", diff --git a/.github/workflows/tests/test_stac.py b/.github/workflows/tests/test_stac.py index d3287081..6b142fdc 100644 --- a/.github/workflows/tests/test_stac.py +++ b/.github/workflows/tests/test_stac.py @@ -1,14 +1,15 @@ """test EOapi.""" - import httpx +import os -stac_endpoint="http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/stac/" - -# better timeouts timeout = httpx.Timeout(15.0, connect=60.0) -client = httpx.Client(timeout=timeout) +if bool(os.getenv("IGNORE_SSL_VERIFICATION", False)): + client = httpx.Client(timeout=timeout, verify=False) +else: + client = httpx.Client(timeout=timeout) + -def test_stac_api(): +def test_stac_api(stac_endpoint): """test stac.""" # Ping assert client.get(f"{stac_endpoint}/_mgmt/ping").status_code == 200 @@ -40,7 +41,7 @@ def test_stac_api(): assert item["id"] == "20200307aC0853300w361200" -def test_stac_to_raster(): +def test_stac_to_raster(stac_endpoint): """test link to raster api.""" # tilejson resp = client.get( diff --git a/.github/workflows/tests/test_vector.py b/.github/workflows/tests/test_vector.py index 21a89ff7..e2c66c50 100644 --- a/.github/workflows/tests/test_vector.py +++ b/.github/workflows/tests/test_vector.py @@ -1,16 +1,15 @@ -"""test EOapi.vector""" - import httpx +import os -vector_endpoint="http://k8s-gcorradi-nginxing-553d3ea33b-3eef2e6e61e5d161.elb.us-west-1.amazonaws.com/vector/" +timeout = httpx.Timeout(15.0, connect=60.0) +if bool(os.getenv("IGNORE_SSL_VERIFICATION", False)): + client = httpx.Client(timeout=timeout, verify=False) +else: + client = httpx.Client(timeout=timeout) -def test_vector_api(): +def test_vector_api(vector_endpoint): """test vector.""" - # better timeouts - timeout = httpx.Timeout(15.0, connect=60.0) - client = httpx.Client(timeout=timeout) - # landing resp = client.get(f"{vector_endpoint}/") assert resp.status_code == 200 diff --git a/helm-chart/eoapi/CHANGELOG.md b/helm-chart/eoapi/CHANGELOG.md index 3c570074..e0c5bdbc 100644 --- a/helm-chart/eoapi/CHANGELOG.md +++ b/helm-chart/eoapi/CHANGELOG.md @@ -1,5 +1,11 @@ version numbers below correspond to helm chart `appVersion`: see `./helm-chart/eoapi/Chart.yaml` --- +### 0.3.3 (2024-06-26) + +* add k3 integration tests +* break out GCP integreation tests and force them to use TLS +* clean it up + ### 0.3.2 (2024-06-16) * backward breaking change: remove in-memory postgres database and secret management for crunchydata postgresql cluster diff --git a/helm-chart/eoapi/Chart.yaml b/helm-chart/eoapi/Chart.yaml index 5214acbf..86cead68 100644 --- a/helm-chart/eoapi/Chart.yaml +++ b/helm-chart/eoapi/Chart.yaml @@ -17,13 +17,13 @@ kubeVersion: ">=1.23.0-0" # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.3.0" +version: "0.3.1" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.3.2" +appVersion: "0.3.3" dependencies: - name: postgrescluster diff --git a/helm-chart/eoapi/templates/services/ingress-nginx.yaml b/helm-chart/eoapi/templates/services/ingress-nginx.yaml index 5d94eb5b..08763da4 100644 --- a/helm-chart/eoapi/templates/services/ingress-nginx.yaml +++ b/helm-chart/eoapi/templates/services/ingress-nginx.yaml @@ -54,14 +54,14 @@ spec: path: /(/|$) backend: service: - name: doc-server + name: doc-server-{{ $.Release.Name }} port: number: 80 {{- end }} - {{- if (and (not .Values.testing) (.Values.ingress.host)) }} + {{- if (.Values.ingress.host) }} host: {{ .Values.ingress.host }} {{- end }} - {{- if (and (not .Values.testing) (.Values.ingress.host) (.Values.ingress.tls.enabled)) }} + {{- if (and (.Values.ingress.host) (.Values.ingress.tls.enabled)) }} tls: - hosts: - {{ .Values.ingress.host }} diff --git a/helm-chart/eoapi/test-gcp-unittest-values.yaml b/helm-chart/eoapi/test-gcp-unittest-values.yaml index 05b1c6e6..dd2f4b88 100644 --- a/helm-chart/eoapi/test-gcp-unittest-values.yaml +++ b/helm-chart/eoapi/test-gcp-unittest-values.yaml @@ -1,8 +1,21 @@ # used in GH Actions `.github/workflows/helm-tests.yml.jobs.integration-tests` testing: true + ingress: enabled: true className: "nginx" + annotations: + kubernetes.io/ingress.class: nginx + ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + # NOTE: for these integration tests `ingress.host` + # is unique and passed during helm install using --set + # host: $RELEASE_NAME.k8s.labs.ds.io + tls: + enabled: true + certManager: true + secretName: eoapi-tls + pgstacBootstrap: enabled: true settings: @@ -46,3 +59,4 @@ vector: envVars: # needs to on so we can call /refresh for integration tests TIPG_DEBUG: "True" + TIPG_CATALOG_TTL: "30"