diff --git a/.github/workflows/on-pr-charts.yaml b/.github/workflows/on-pr-charts.yaml index ea5ae9794..011bbe725 100644 --- a/.github/workflows/on-pr-charts.yaml +++ b/.github/workflows/on-pr-charts.yaml @@ -1,5 +1,6 @@ #This is just extra testing, for lint check, and basic installation -#If those fail, no need to test the rest of the PR (github will cancel the rest of the builds) +#Those can fail earlier than functional tests (shorter tests) +# and give developer feedback soon if they didn't test themselves name: PR - charts on: pull_request: @@ -11,7 +12,7 @@ jobs: # tackling that for us. # Fail-fast ensures that if one of those matrix job fail, the other one gets cancelled. test-chart: - name: Test helm chart + name: Test helm chart changes runs-on: ubuntu-latest strategy: fail-fast: true @@ -42,7 +43,7 @@ jobs: # This doesn't re-use the ct actions, due to many limitations (auto tear down, no real testing) deploy-chart: - name: "Functional test of helm chart with pre-published images" + name: Functional test of helm chart in its current state (needs published image of the helm chart) runs-on: ubuntu-latest needs: test-chart steps: diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml index 27d43ccb9..1c9c8bb76 100644 --- a/.github/workflows/on-pr.yaml +++ b/.github/workflows/on-pr.yaml @@ -84,16 +84,21 @@ jobs: with: image-name: docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} - # If the PRs don't break the behaviour in the helm chart, we can simply publish the helm charts at the time of a branch commit. - e2e-helm: - name: "Functional test of helm chart, e2e testing" + # This ensures the latest code works with the manifests built from tree. + # It is useful for two things: + # - Test manifests changes (obviously), ensuring they don't break existing clusters + # - Ensure manifests work with the latest versions even with no manifest change + # (compared to helm charts, manifests cannot easily template changes based on versions) + # Helm charts are _trailing_ releases, while manifests are done during development. + e2e-manifests: + name: End-to-End test with kured with code and manifests from HEAD runs-on: ubuntu-latest - # only build with oldest and newest supported, it should be good enough. strategy: fail-fast: false matrix: kubernetes: - "1.18" + - "1.19" - "1.20" steps: - uses: actions/checkout@v2 @@ -109,9 +114,9 @@ jobs: - name: Build artifacts run: | make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image - make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" helm-chart + make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" manifest - - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions" + - name: Workaround "Failed to attach 1 to compat systemd cgroup /actions_job/..." on gh actions run: | sudo bash << EOF cp /etc/docker/daemon.json /etc/docker/daemon.json.old @@ -121,7 +126,7 @@ jobs: EOF # Default name for helm/kind-action kind clusters is "chart-testing" - - name: Create 5 node kind cluster + - name: Create kind cluster with 5 nodes uses: helm/kind-action@v1.1.0 with: config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml @@ -129,16 +134,13 @@ jobs: - name: Preload previously built images onto kind cluster run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing - - name: Deploy kured on default namespace with its helm chart + - name: Do not wait for an hour before detecting the rebootSentinel run: | - # Documented in official helm doc to live on the edge - curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - # Refresh bins - hash -r - helm install kured ./charts/kured/ --set configuration.period=1m - kubectl config set-context kind-chart-testing - kubectl get ds --all-namespaces - kubectl describe ds kured + sed -i 's/#\(.*\)--period=1h/\1--period=30s/g' kured-ds.yaml + + - name: Install kured with kubectl + run: | + kubectl apply -f kured-rbac.yaml && kubectl apply -f kured-ds.yaml - name: Ensure kured is ready uses: nick-invision/retry@v2.4.0 @@ -146,8 +148,8 @@ jobs: timeout_minutes: 10 max_attempts: 10 retry_wait_seconds: 60 - # DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = 5 - command: "kubectl get ds kured | grep -E 'kured.*5.*5.*5.*5.*5' " + # DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = to cluster_size + command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'" - name: Create reboot sentinel files run: | @@ -159,61 +161,8 @@ jobs: run: | ./tests/kind/follow-coordinated-reboot.sh - # This workflow is useful when introducing new versions, to ensure our manifests - # still work (even if there might be no manifest 'code' change). - # The version used here is what hasn't been tested with the helm chart - # This should therefore be "mid version" and (optionally) next new version - deploy-manifests: - name: Deploy kured with current manifests - runs-on: ubuntu-latest - strategy: - matrix: - kubernetes: - - "1.19" - steps: - - uses: actions/checkout@v2 - - name: Find go version - run: | - GO_VERSION=$(awk '/^go/ {print $2};' go.mod) - echo "::set-output name=version::${GO_VERSION}" - id: awk_gomod - - name: Ensure go version - uses: actions/setup-go@v2 - with: - go-version: "${{ steps.awk_gomod.outputs.version }}" - - name: Build artifacts - run: | - make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image - make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" manifest - - name: Workaround "Failed to attach 1 to compat systemd cgroup /actions_job/..." on gh actions - run: | - sudo bash << EOF - cp /etc/docker/daemon.json /etc/docker/daemon.json.old - echo '{}' > /etc/docker/daemon.json - systemctl restart docker || journalctl --no-pager -n 500 - systemctl status docker - EOF - # Default name for helm/kind-action kind clusters is "chart-testing" - - name: Create kind cluster - uses: helm/kind-action@v1.1.0 - with: - config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml - - name: Preload previously built images onto kind cluster - run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing - - name: Install kured with kubectl - run: | - kubectl apply -f kured-rbac.yaml && kubectl apply -f kured-ds.yaml - - name: Ensure kured is ready - uses: nick-invision/retry@v2.4.0 - with: - timeout_minutes: 10 - max_attempts: 10 - retry_wait_seconds: 60 - # DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = to cluster_size - command: "kubectl get ds -n kube-system kured | grep -E 'kured.*5.*5.*5.*5.*5'" - - test-prom: - name: "Test prometheus with latest code from HEAD" + scenario-prom-helm: + name: Test prometheus with latest code from HEAD (=overrides image of the helm chart) runs-on: ubuntu-latest # only build with oldest and newest supported, it should be good enough. strategy: @@ -237,7 +186,7 @@ jobs: make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" helm-chart - - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions" + - name: Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions run: | sudo bash << EOF cp /etc/docker/daemon.json /etc/docker/daemon.json.old @@ -288,7 +237,88 @@ jobs: - name: Get metrics (need reboot) uses: nick-invision/retry@v2.4.0 with: - timeout_minutes: 15 + timeout_minutes: 15 max_attempts: 10 retry_wait_seconds: 60 - command: "./tests/kind/test-metrics.sh 1" \ No newline at end of file + command: "./tests/kind/test-metrics.sh 1" + + + # TEMPLATE Scenario testing. + # Note: keep in mind that the helm chart's appVersion is overriden to test your HEAD of the branch, + # if you `make helm-chart`. + # This will allow you to test properly your scenario and not use an existing image which will not + # contain your feature. + + # scenario--helm: + # #example: Testing with helm chart and code from HEAD" + # name: "" + # runs-on: ubuntu-latest + # strategy: + # fail-fast: false + # # You can define your own kubernetes versions. For example if your helm chart change should behave differently with different kubernetes versions. + # matrix: + # kubernetes: + # - "1.20" + # steps: + # - uses: actions/checkout@v2 + # - name: Find go version + # run: | + # GO_VERSION=$(awk '/^go/ {print $2};' go.mod) + # echo "::set-output name=version::${GO_VERSION}" + # id: awk_gomod + # - name: Ensure go version + # uses: actions/setup-go@v2 + # with: + # go-version: "${{ steps.awk_gomod.outputs.version }}" + # - name: Build artifacts + # run: | + # make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" image + # make DH_ORG="${{ github.repository_owner }}" VERSION="${{ github.sha }}" helm-chart + # + # - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions" + # run: | + # sudo bash << EOF + # cp /etc/docker/daemon.json /etc/docker/daemon.json.old + # echo '{}' > /etc/docker/daemon.json + # systemctl restart docker || journalctl --no-pager -n 500 + # systemctl status docker + # EOF + # + # # Default name for helm/kind-action kind clusters is "chart-testing" + # - name: Create 5 node kind cluster + # uses: helm/kind-action@master + # with: + # config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml + # + # - name: Preload previously built images onto kind cluster + # run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} --name chart-testing + # + # - name: Deploy kured on default namespace with its helm chart + # run: | + # # Documented in official helm doc to live on the edge + # curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + # # Refresh bins + # hash -r + # helm install kured ./charts/kured/ --wait --values ./charts/kured/ci/-values.yaml + # kubectl config set-context kind-chart-testing + # kubectl get ds --all-namespaces + # kubectl describe ds kured + # + # - name: Ensure kured is ready + # uses: nick-invision/retry@v2.4.0 + # with: + # timeout_minutes: 10 + # max_attempts: 10 + # retry_wait_seconds: 60 + # # DESIRED CURRENT READY UP-TO-DATE AVAILABLE should all be = 5 + # command: "kubectl get ds kured | grep -E 'kured.*5.*5.*5.*5.*5' " + # + # - name: Create reboot sentinel files + # run: | + # ./tests/kind/create-reboot-sentinels.sh + # + # - name: Test + # env: + # DEBUG: true + # run: | + # diff --git a/.github/workflows/periodics-daily.yaml b/.github/workflows/periodics-daily.yaml index cc12cb395..de817b33d 100644 --- a/.github/workflows/periodics-daily.yaml +++ b/.github/workflows/periodics-daily.yaml @@ -68,7 +68,7 @@ jobs: image-name: docker.io/${{ github.repository_owner }}/kured:${{ github.sha }} deploy-helm: - name: Ensure a kubernetes change didn't break our code + name: Ensure our currently released helm chart works on all kubernetes versions runs-on: ubuntu-latest # only build with oldest and newest supported, it should be good enough. strategy: @@ -88,10 +88,6 @@ jobs: uses: actions/setup-go@v2 with: go-version: "${{ steps.awk_gomod.outputs.version }}" - - name: Build artifacts - run: | - make DH_ORG="${{ github.repository_owner }}" VERSION="main" image - make DH_ORG="${{ github.repository_owner }}" VERSION="main" helm-chart - name: "Workaround 'Failed to attach 1 to compat systemd cgroup /actions_job/...' on gh actions" run: | @@ -108,9 +104,6 @@ jobs: with: config: .github/kind-cluster-${{ matrix.kubernetes }}.yaml - - name: Preload previously built images onto kind cluster - run: kind load docker-image docker.io/${{ github.repository_owner }}/kured:main --name chart-testing - - name: Deploy kured on default namespace with its helm chart run: | # Documented in official helm doc to live on the edge @@ -139,4 +132,4 @@ jobs: env: DEBUG: true run: | - ./tests/kind/follow-coordinated-reboot.sh + ./tests/kind/follow-coordinated-reboot.sh \ No newline at end of file diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 9f9f743df..4da87d6cf 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -13,7 +13,9 @@ you are planning to contribute code. [issues]: https://github.com/weaveworks/kured/issues [readme]: README.md -## Updating k8s support +## Regular development activities + +### Updating k8s support Whenever we want to update e.g. the `kubectl` or `client-go` dependencies, some RBAC changes might be necessary too. @@ -24,15 +26,90 @@ This is what it took to support Kubernetes 1.14: That the process can be more involved that that can be seen in +Please update our .github/workflows with the new k8s images, starting by +the creation of a .github/kind-cluster-.yaml, then updating +our workflows with the new versions. + Once you updated everything, make sure you update the support matrix on the main [README][readme] as well. -## Release testing +### Updating other dependencies + +Dependabot proposes changes in our go.mod/go.sum. +Some of those changes are covered by CI testing, some are not. + +Please make sure to test those not covered by CI (mostly the integration +with other tools) manually before merging. + +### Review periodic jobs + +We run periodic jobs (see also Automated testing section of this documentation). +Those should be monitored for failures. + +If a failure happen in periodics, something terribly wrong must have happened +(or github is failing at the creation of a kind cluster). Please monitor those +failures carefully. + +### Introducing new features + +When you introduce a new feature, the kured team expects you to have tested +your change thoroughly. If possible, include all the necessary testing in your change. + +If your change involves a user facing change (change in flags of kured for example), +please include expose your new feature in our default manifest (`kured-ds.yaml`), +as a comment. + +Do not update the helm chart directly. +Helm charts and our release manifests (see below) are our stable interfaces. +Any user facing changes will therefore have to wait for a while before being +exposed to our users. + +This also means that when you expose a new feature, you should create another PR +for your changes in `charts/` to make your feature available for our next kured version. +In this change, you can directly bump the appVersion to the next minor version. +(for example, if current appVersion is 1.6.x, make sure you update your appVersion +to 1.7.0). It allows us to have an easy view of what we land each release. + +Do not hesitate to increase the test coverage for your feature, whether it's unit +testing to full functional testing (even using helm charts) + +### Increasing test coverage + +We are welcoming any change to increase our test coverage. +See also our github issues for the label `testing`. + +### Updating helm charts + +Helm charts are continuously published. Any change in `charts/` will be immediately +pushed in production. + +## Automated testing + +Our CI is covered by github actions. +You can see their contents in .github/workflows. + +We currently run: +- go tests and lint +- shellcheck +- a check for dead links in our docs +- a security check against our base image (alpine) +- a deep functional test using our manifests on all supported k8s versions +- basic deployment using our helm chart on any chart change + +Changes in helm charts are not functionally tested on PRs. We assume that +the PRs to implement the feature are properly tested by our users and +contributors before merge. + +To test your code manually, follow the section Manual testing. + +## Manual (release) testing Before `kured` is released, we want to make sure it still works fine on the previous, current and next minor version of Kubernetes (with respect to the `client-go` & `kubectl` dependencies in use). For local testing e.g. -`minikube` or `kind` can be sufficient. +`minikube` or `kind` can be sufficient. This will allow you to catch issues +that might not have been tested in our CI, like integration with other tools, +or your specific use case. Deploy kured in your test scenario, make sure you pass the right `image`, update the e.g. `period` and `reboot-days` options, so you get immediate @@ -42,7 +119,11 @@ results, if you login to a node and run: sudo touch /var/run/reboot-required ``` -### Testing with `minikube` +### Example of golang testing + +Please run `make test`. You should have golint installed. + +### Example of testing with `minikube` A test-run with `minikube` could look like this: @@ -82,7 +163,7 @@ If all the tests ran well, kured maintainers can reach out to the Weaveworks team to get an upcoming `kured` release tested in the Dev environment for real life testing. -### Testing with `kind` +### Example of testing with `kind` A test-run with `kind` could look like this: diff --git a/Makefile b/Makefile index ca80ff346..36b259c52 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .DEFAULT: all -.PHONY: all clean image publish-image minikube-publish manifest helm-chart +.PHONY: all clean image publish-image minikube-publish manifest helm-chart test tests DH_ORG=weaveworks VERSION=$(shell git symbolic-ref --short HEAD)-$(shell git rev-parse --short HEAD) @@ -43,3 +43,11 @@ helm-chart: sed -i "s#appVersion:.*#appVersion: \"$(VERSION)\"#g" charts/kured/Chart.yaml sed -i "s#\`[0-9]*\.[0-9]*\.[0-9]*\`#\`$(VERSION)\`#g" charts/kured/README.md echo "Please bump version in charts/kured/Chart.yaml" + +test: tests + echo "Running go tests" + go test ./... + echo "Running golint on pkg" + golint ./pkg/... + echo "Running golint on cmd" + golint ./cmd/...