Skip to content

feat: Consistent hashing with bounded loads algorithm for sharding #28644

feat: Consistent hashing with bounded loads algorithm for sharding

feat: Consistent hashing with bounded loads algorithm for sharding #28644

Workflow file for this run

name: Integration tests
on:
push:
branches:
- 'master'
- 'release-*'
- '!release-1.4'
- '!release-1.5'
pull_request:
branches:
- 'master'
- 'release-*'
env:
# Golang version to use across CI steps
GOLANG_VERSION: '1.22'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
changes:
runs-on: ubuntu-latest
outputs:
backend: ${{ steps.filter.outputs.backend_any_changed }}
frontend: ${{ steps.filter.outputs.frontend_any_changed }}
steps:
- uses: actions/checkout@8410ad0602e1e429cee44a835ae9f77f654a6694 # v4.0.0
- uses: tj-actions/changed-files@90a06d6ba9543371ab4df8eeca0be07ca6054959 # v42.0.2
id: filter
with:
# Any file which is not under docs/, ui/ or is not a markdown file is counted as a backend file
files_yaml: |
backend:
- '!ui/**'
- '!**.md'
- '!**/*.md'
- '!docs/**'
frontend:
- 'ui/**'
- Dockerfile
docs:
- 'docs/**'
check-go:
name: Ensure Go modules synchronicity
if: ${{ needs.changes.outputs.backend == 'true' }}
runs-on: ubuntu-22.04
needs:
- changes
steps:
- name: Checkout code
uses: actions/checkout@8410ad0602e1e429cee44a835ae9f77f654a6694 # v4.0.0
- name: Setup Golang
uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.0.0
with:
go-version: ${{ env.GOLANG_VERSION }}
- name: Download all Go modules
run: |
go mod download
- name: Check for tidiness of go.mod and go.sum
run: |
go mod tidy
git diff --exit-code -- .
build-go:
name: Build & cache Go code
if: ${{ needs.changes.outputs.backend == 'true' }}
runs-on: ubuntu-22.04
needs:
- changes
steps:
- name: Checkout code
uses: actions/checkout@8410ad0602e1e429cee44a835ae9f77f654a6694 # v4.0.0
- name: Setup Golang
uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.0.0
with:
go-version: ${{ env.GOLANG_VERSION }}
- name: Restore go build cache
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
with:
path: ~/.cache/go-build
key: ${{ runner.os }}-go-build-v1-${{ github.run_id }}
- name: Download all Go modules
run: |
go mod download
- name: Compile all packages
run: make build-local
lint-go:
permissions:
contents: read # for actions/checkout to fetch code
pull-requests: read # for golangci/golangci-lint-action to fetch pull requests
name: Lint Go code
if: ${{ needs.changes.outputs.backend == 'true' }}
runs-on: ubuntu-22.04
needs:
- changes
steps:
- name: Checkout code
uses: actions/checkout@8410ad0602e1e429cee44a835ae9f77f654a6694 # v4.0.0
- name: Setup Golang
uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.0.0
with:
go-version: ${{ env.GOLANG_VERSION }}
- name: Run golangci-lint
uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc # v3.7.0
with:
version: v1.54.0
args: --enable gofmt --timeout 10m --exclude SA5011 --verbose --max-issues-per-linter 0 --max-same-issues 0
test-go:
name: Run unit tests for Go packages
if: ${{ needs.changes.outputs.backend == 'true' }}
runs-on: ubuntu-22.04
needs:
- build-go
- changes
env:
GITHUB_TOKEN: ${{ secrets.E2E_TEST_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
GITLAB_TOKEN: ${{ secrets.E2E_TEST_GITLAB_TOKEN }}
steps:
- name: Create checkout directory
run: mkdir -p ~/go/src/github.com/argoproj
- name: Checkout code
uses: actions/checkout@8410ad0602e1e429cee44a835ae9f77f654a6694 # v4.0.0
- name: Create symlink in GOPATH
run: ln -s $(pwd) ~/go/src/github.com/argoproj/argo-cd
- name: Setup Golang
uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.0.0
with:
go-version: ${{ env.GOLANG_VERSION }}
- name: Install required packages
run: |
sudo apt-get install git -y
- name: Switch to temporal branch so we re-attach head
run: |
git switch -c temporal-pr-branch
git status
- name: Fetch complete history for blame information
run: |
git fetch --prune --no-tags --depth=1 origin +refs/heads/*:refs/remotes/origin/*
- name: Add ~/go/bin to PATH
run: |
echo "/home/runner/go/bin" >> $GITHUB_PATH
- name: Add /usr/local/bin to PATH
run: |
echo "/usr/local/bin" >> $GITHUB_PATH
- name: Restore go build cache
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
with:
path: ~/.cache/go-build
key: ${{ runner.os }}-go-build-v1-${{ github.run_id }}
- name: Install all tools required for building & testing
run: |
make install-test-tools-local
# We install kustomize in the dist directory
- name: Add dist to PATH
run: |
echo "/home/runner/work/argo-cd/argo-cd/dist" >> $GITHUB_PATH
- name: Setup git username and email
run: |
git config --global user.name "John Doe"
git config --global user.email "john.doe@example.com"
- name: Download and vendor all required packages
run: |
go mod download
- name: Run all unit tests
run: make test-local
- name: Generate code coverage artifacts
uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
with:
name: code-coverage
path: coverage.out
- name: Generate test results artifacts
uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
with:
name: test-results
path: test-results/
test-go-race:
name: Run unit tests with -race for Go packages
if: ${{ needs.changes.outputs.backend == 'true' }}
runs-on: ubuntu-22.04
needs:
- build-go
- changes
env:
GITHUB_TOKEN: ${{ secrets.E2E_TEST_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
GITLAB_TOKEN: ${{ secrets.E2E_TEST_GITLAB_TOKEN }}
steps:
- name: Create checkout directory
run: mkdir -p ~/go/src/github.com/argoproj
- name: Checkout code
uses: actions/checkout@8410ad0602e1e429cee44a835ae9f77f654a6694 # v4.0.0
- name: Create symlink in GOPATH
run: ln -s $(pwd) ~/go/src/github.com/argoproj/argo-cd
- name: Setup Golang
uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.0.0
with:
go-version: ${{ env.GOLANG_VERSION }}
- name: Install required packages
run: |
sudo apt-get install git -y
- name: Switch to temporal branch so we re-attach head
run: |
git switch -c temporal-pr-branch
git status
- name: Fetch complete history for blame information
run: |
git fetch --prune --no-tags --depth=1 origin +refs/heads/*:refs/remotes/origin/*
- name: Add ~/go/bin to PATH
run: |
echo "/home/runner/go/bin" >> $GITHUB_PATH
- name: Add /usr/local/bin to PATH
run: |
echo "/usr/local/bin" >> $GITHUB_PATH
- name: Restore go build cache
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
with:
path: ~/.cache/go-build
key: ${{ runner.os }}-go-build-v1-${{ github.run_id }}
- name: Install all tools required for building & testing
run: |
make install-test-tools-local
# We install kustomize in the dist directory
- name: Add dist to PATH
run: |
echo "/home/runner/work/argo-cd/argo-cd/dist" >> $GITHUB_PATH
- name: Setup git username and email
run: |
git config --global user.name "John Doe"
git config --global user.email "john.doe@example.com"
- name: Download and vendor all required packages
run: |
go mod download
- name: Run all unit tests
run: make test-race-local
- name: Generate test results artifacts
uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
with:
name: race-results
path: test-results/
codegen:
name: Check changes to generated code
if: ${{ needs.changes.outputs.backend == 'true' || needs.changes.outputs.docs == 'true'}}
runs-on: ubuntu-22.04
needs:
- changes
steps:
- name: Checkout code
uses: actions/checkout@8410ad0602e1e429cee44a835ae9f77f654a6694 # v4.0.0
- name: Setup Golang
uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.0.0
with:
go-version: ${{ env.GOLANG_VERSION }}
- name: Create symlink in GOPATH
run: |
mkdir -p ~/go/src/github.com/argoproj
cp -a ../argo-cd ~/go/src/github.com/argoproj
- name: Add ~/go/bin to PATH
run: |
echo "/home/runner/go/bin" >> $GITHUB_PATH
- name: Add /usr/local/bin to PATH
run: |
echo "/usr/local/bin" >> $GITHUB_PATH
- name: Download & vendor dependencies
run: |
# We need to vendor go modules for codegen yet
go mod download
go mod vendor -v
working-directory: /home/runner/go/src/github.com/argoproj/argo-cd
- name: Install toolchain for codegen
run: |
make install-codegen-tools-local
make install-go-tools-local
working-directory: /home/runner/go/src/github.com/argoproj/argo-cd
# We install kustomize in the dist directory
- name: Add dist to PATH
run: |
echo "/home/runner/work/argo-cd/argo-cd/dist" >> $GITHUB_PATH
- name: Run codegen
run: |
set -x
export GOPATH=$(go env GOPATH)
git checkout -- go.mod go.sum
make codegen-local
working-directory: /home/runner/go/src/github.com/argoproj/argo-cd
- name: Check nothing has changed
run: |
set -xo pipefail
git diff --exit-code -- . ':!go.sum' ':!go.mod' ':!assets/swagger.json' | tee codegen.patch
working-directory: /home/runner/go/src/github.com/argoproj/argo-cd
build-ui:
name: Build, test & lint UI code
if: ${{ needs.changes.outputs.frontend == 'true' }}
runs-on: ubuntu-22.04
needs:
- changes
steps:
- name: Checkout code
uses: actions/checkout@8410ad0602e1e429cee44a835ae9f77f654a6694 # v4.0.0
- name: Setup NodeJS
uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3.8.1
with:
node-version: '21.6.1'
- name: Restore node dependency cache
id: cache-dependencies
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
with:
path: ui/node_modules
key: ${{ runner.os }}-node-dep-v2-${{ hashFiles('**/yarn.lock') }}
- name: Install node dependencies
run: |
cd ui && yarn install --frozen-lockfile --ignore-optional --non-interactive
- name: Build UI code
run: |
yarn test
yarn build
env:
NODE_ENV: production
NODE_ONLINE_ENV: online
HOST_ARCH: amd64
working-directory: ui/
- name: Run ESLint
run: yarn lint
working-directory: ui/
analyze:
name: Process & analyze test artifacts
if: ${{ needs.changes.outputs.backend == 'true' || needs.changes.outputs.frontend == 'true' }}
runs-on: ubuntu-22.04
needs:
- test-go
- build-ui
- changes
env:
sonar_secret: ${{ secrets.SONAR_TOKEN }}
steps:
- name: Checkout code
uses: actions/checkout@8410ad0602e1e429cee44a835ae9f77f654a6694 # v4.0.0
with:
fetch-depth: 0
- name: Restore node dependency cache
id: cache-dependencies
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
with:
path: ui/node_modules
key: ${{ runner.os }}-node-dep-v2-${{ hashFiles('**/yarn.lock') }}
- name: Remove other node_modules directory
run: |
rm -rf ui/node_modules/argo-ui/node_modules
- name: Create test-results directory
run: |
mkdir -p test-results
- name: Get code coverage artifact
uses: actions/download-artifact@8caf195ad4b1dee92908e23f56eeb0696f1dd42d # v4.1.5
with:
name: code-coverage
- name: Get test result artifact
uses: actions/download-artifact@8caf195ad4b1dee92908e23f56eeb0696f1dd42d # v4.1.5
with:
name: test-results
path: test-results
- name: Upload code coverage information to codecov.io
uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # v3.1.4
with:
file: coverage.out
- name: Perform static code analysis using SonarCloud
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
SCANNER_VERSION: 4.2.0.1873
SCANNER_PATH: /tmp/cache/scanner
OS: linux
run: |
# We do not use the provided action, because it does contain an old
# version of the scanner, and also takes time to build.
set -e
mkdir -p ${SCANNER_PATH}
export SONAR_USER_HOME=${SCANNER_PATH}/.sonar
if [[ ! -x "${SCANNER_PATH}/sonar-scanner-${SCANNER_VERSION}-${OS}/bin/sonar-scanner" ]]; then
curl -Ol https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${SCANNER_VERSION}-${OS}.zip
unzip -qq -o sonar-scanner-cli-${SCANNER_VERSION}-${OS}.zip -d ${SCANNER_PATH}
fi
chmod +x ${SCANNER_PATH}/sonar-scanner-${SCANNER_VERSION}-${OS}/bin/sonar-scanner
chmod +x ${SCANNER_PATH}/sonar-scanner-${SCANNER_VERSION}-${OS}/jre/bin/java
# Explicitly set NODE_MODULES
export NODE_MODULES=${PWD}/ui/node_modules
export NODE_PATH=${PWD}/ui/node_modules
${SCANNER_PATH}/sonar-scanner-${SCANNER_VERSION}-${OS}/bin/sonar-scanner
if: env.sonar_secret != ''
test-e2e:
name: Run end-to-end tests
if: ${{ needs.changes.outputs.backend == 'true' }}
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
k3s-version: [v1.29.1, v1.28.6, v1.27.10, v1.26.13, v1.25.16]
needs:
- build-go
- changes
env:
GOPATH: /home/runner/go
ARGOCD_FAKE_IN_CLUSTER: "true"
ARGOCD_SSH_DATA_PATH: "/tmp/argo-e2e/app/config/ssh"
ARGOCD_TLS_DATA_PATH: "/tmp/argo-e2e/app/config/tls"
ARGOCD_E2E_SSH_KNOWN_HOSTS: "../fixture/certs/ssh_known_hosts"
ARGOCD_E2E_K3S: "true"
ARGOCD_IN_CI: "true"
ARGOCD_E2E_APISERVER_PORT: "8088"
ARGOCD_APPLICATION_NAMESPACES: "argocd-e2e-external,argocd-e2e-external-2"
ARGOCD_SERVER: "127.0.0.1:8088"
GITHUB_TOKEN: ${{ secrets.E2E_TEST_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
GITLAB_TOKEN: ${{ secrets.E2E_TEST_GITLAB_TOKEN }}
steps:
- name: Checkout code
uses: actions/checkout@8410ad0602e1e429cee44a835ae9f77f654a6694 # v4.0.0
- name: Setup Golang
uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.0.0
with:
go-version: ${{ env.GOLANG_VERSION }}
- name: GH actions workaround - Kill XSP4 process
run: |
sudo pkill mono || true
- name: Install K3S
env:
INSTALL_K3S_VERSION: ${{ matrix.k3s-version }}+k3s1
run: |
set -x
curl -sfL https://get.k3s.io | sh -
sudo chmod -R a+rw /etc/rancher/k3s
sudo mkdir -p $HOME/.kube && sudo chown -R runner $HOME/.kube
sudo k3s kubectl config view --raw > $HOME/.kube/config
sudo chown runner $HOME/.kube/config
sudo chmod go-r $HOME/.kube/config
kubectl version
- name: Restore go build cache
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
with:
path: ~/.cache/go-build
key: ${{ runner.os }}-go-build-v1-${{ github.run_id }}
- name: Add ~/go/bin to PATH
run: |
echo "/home/runner/go/bin" >> $GITHUB_PATH
- name: Add /usr/local/bin to PATH
run: |
echo "/usr/local/bin" >> $GITHUB_PATH
- name: Add ./dist to PATH
run: |
echo "$(pwd)/dist" >> $GITHUB_PATH
- name: Download Go dependencies
run: |
go mod download
go install github.com/mattn/goreman@latest
- name: Install all tools required for building & testing
run: |
make install-test-tools-local
- name: Setup git username and email
run: |
git config --global user.name "John Doe"
git config --global user.email "john.doe@example.com"
- name: Pull Docker image required for tests
run: |
docker pull ghcr.io/dexidp/dex:v2.38.0
docker pull argoproj/argo-cd-ci-builder:v1.0.0
docker pull redis:7.0.15-alpine
- name: Create target directory for binaries in the build-process
run: |
mkdir -p dist
chown runner dist
- name: Run E2E server and wait for it being available
timeout-minutes: 30
run: |
set -x
# Something is weird in GH runners -- there's a phantom listener for
# port 8080 which is not visible in netstat -tulpen, but still there
# with a HTTP listener. We have API server listening on port 8088
# instead.
make start-e2e-local 2>&1 | sed -r "s/[[:cntrl:]]\[[0-9]{1,3}m//g" > /tmp/e2e-server.log &
count=1
until curl -f http://127.0.0.1:8088/healthz; do
sleep 10;
if test $count -ge 180; then
echo "Timeout"
exit 1
fi
count=$((count+1))
done
- name: Run E2E testsuite
run: |
set -x
make test-e2e-local
- name: Upload e2e-server logs
uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
with:
name: e2e-server-k8s${{ matrix.k3s-version }}.log
path: /tmp/e2e-server.log
if: ${{ failure() }}
# workaround for status checks -- check this one job instead of each individual E2E job in the matrix
# this allows us to skip the entire matrix when it doesn't need to run while still having accurate status checks
# see:
# https://github.com/argoproj/argo-workflows/pull/12006
# https://github.com/orgs/community/discussions/9141#discussioncomment-2296809
# https://github.com/orgs/community/discussions/26822#discussioncomment-3305794
test-e2e-composite-result:
name: E2E Tests - Composite result
if: ${{ always() }}
needs:
- test-e2e
- changes
runs-on: ubuntu-22.04
steps:
- run: |
result="${{ needs.test-e2e.result }}"
# mark as successful even if skipped
if [[ $result == "success" || $result == "skipped" ]]; then
exit 0
else
exit 1
fi