From 514f241222edf8041390f017111204232ca230d2 Mon Sep 17 00:00:00 2001 From: Wantong Jiang Date: Tue, 10 Jun 2025 23:45:01 +0000 Subject: [PATCH 1/2] test: use clusterloader2 framework for large scale testing Signed-off-by: Wantong Jiang --- .github/workflows/markdown-lint.yml | 2 +- hack/cl2/README.md | 24 +++++ hack/cl2/busy_cluster.yaml | 107 ++++++++++++++++++++++ hack/cl2/busy_cluster_burst.yaml | 122 +++++++++++++++++++++++++ hack/cl2/cleanup.sh | 10 ++ hack/cl2/manifests/test-configmap.yaml | 7 ++ hack/cl2/manifests/test-crp.yaml | 19 ++++ hack/cl2/modules/configmaps.yaml | 14 +++ 8 files changed, 304 insertions(+), 1 deletion(-) create mode 100644 hack/cl2/README.md create mode 100644 hack/cl2/busy_cluster.yaml create mode 100644 hack/cl2/busy_cluster_burst.yaml create mode 100755 hack/cl2/cleanup.sh create mode 100644 hack/cl2/manifests/test-configmap.yaml create mode 100644 hack/cl2/manifests/test-crp.yaml create mode 100644 hack/cl2/modules/configmaps.yaml diff --git a/.github/workflows/markdown-lint.yml b/.github/workflows/markdown-lint.yml index 10653eeb0..e65a4999c 100644 --- a/.github/workflows/markdown-lint.yml +++ b/.github/workflows/markdown-lint.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: gaurav-nelson/github-action-markdown-link-check@v1 + - uses: tcort/github-action-markdown-link-check@v1 with: # this will only show errors in the output use-quiet-mode: 'yes' diff --git a/hack/cl2/README.md b/hack/cl2/README.md new file mode 100644 index 000000000..3585bb11c --- /dev/null +++ b/hack/cl2/README.md @@ -0,0 +1,24 @@ +This folder contains a list of [clusterloader2](https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/docs/GETTING_STARTED.md) configuration manifests for large scale testing. + +## Prerequisites + +Follow [clusterloader2 GETTING STARTED](https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/docs/GETTING_STARTED.md) to clone the `perf-tests` repository. + +## Execute Tests + +Run `clusterloader2` under `perf-tests/clusterloader2` directory: +```bash +go run cmd/clusterloader.go --testconfig= --provider=local --kubeconfig= --v=2 --enable-exec-service=false +``` +We need to set `--enable-exec-service=false` to prevent creating agnostic deployment on the hub cluster as hub +cluster does not allow pod creation. + +## Cleanup Resources After Tests + +By default, clusterloader2 automatically deletes all the generated namespaces after the tests. +In addition, we also want to delete all the CRPs created. To facilitate cleanup process, run the simple script +provided in this folder: +``` +export KUBECONFIG= +./cleanup.sh +``` \ No newline at end of file diff --git a/hack/cl2/busy_cluster.yaml b/hack/cl2/busy_cluster.yaml new file mode 100644 index 000000000..545a2d447 --- /dev/null +++ b/hack/cl2/busy_cluster.yaml @@ -0,0 +1,107 @@ +name: busy_cluster_test + +{{$duration := "30m"}} +{{$count := 1}} +{{$namespaceCount := 1000}} + +namespace: + number: {{$namespaceCount}} + prefix: busy-cluster-test-ns + +tuningSets: +- name: Uniform10qps + qpsLoad: + qps: 10 +- name: SteppedLoad + steppedLoad: + burstSize: 20 + stepDelay: 30s + +steps: +- name: Create CRPs + phases: + - replicasPerNamespace: {{$namespaceCount}} + tuningSet: Uniform10qps + objectBundle: + - basename: test-crp + objectTemplatePath: "manifests/test-crp.yaml" +- name: Wait for CRPs to be Ready + measurements: + - Identifier: WaitForGenericK8sObjects + Method: WaitForGenericK8sObjects + Params: + objectGroup: placement.kubernetes-fleet.io + objectVersion: v1beta1 + objectResource: clusterresourceplacements + timeout: {{$duration}} + successfulConditions: + - ClusterResourcePlacementAvailable=True + failedConditions: + - ClusterResourcePlacementAvailable=False + minDesiredObjectCount: {{$namespaceCount}} + maxFailedObjectCount: 0 +- module: + path: /modules/configmaps.yaml + params: + namespaceCount: {{$namespaceCount}} + configmapCount: {{$count}} +- module: + path: /modules/configmaps.yaml + params: + namespaceCount: {{$namespaceCount}} + configmapCount: {{MultiplyInt $count 2}} +- module: + path: /modules/configmaps.yaml + params: + namespaceCount: {{$namespaceCount}} + configmapCount: {{MultiplyInt $count 3}} +- module: + path: /modules/configmaps.yaml + params: + namespaceCount: {{$namespaceCount}} + configmapCount: {{MultiplyInt $count 4}} +- module: + path: /modules/configmaps.yaml + params: + namespaceCount: {{$namespaceCount}} + configmapCount: {{MultiplyInt $count 5}} +- module: + path: /modules/configmaps.yaml + params: + namespaceCount: {{$namespaceCount}} + configmapCount: {{MultiplyInt $count 6}} +- module: + path: /modules/configmaps.yaml + params: + namespaceCount: {{$namespaceCount}} + configmapCount: {{MultiplyInt $count 7}} +- module: + path: /modules/configmaps.yaml + params: + namespaceCount: {{$namespaceCount}} + configmapCount: {{MultiplyInt $count 8}} +- module: + path: /modules/configmaps.yaml + params: + namespaceCount: {{$namespaceCount}} + configmapCount: {{MultiplyInt $count 9}} +- module: + path: /modules/configmaps.yaml + params: + namespaceCount: {{$namespaceCount}} + configmapCount: {{MultiplyInt $count 10}} +- name: Wait for CRPs to be Ready + measurements: + - Identifier: WaitForGenericK8sObjects + Method: WaitForGenericK8sObjects + Params: + objectGroup: placement.kubernetes-fleet.io + objectVersion: v1beta1 + objectResource: clusterresourceplacements + timeout: {{$duration}} + successfulConditions: + - ClusterResourcePlacementAvailable=True + failedConditions: + - ClusterResourcePlacementAvailable=False + minDesiredObjectCount: {{$namespaceCount}} + maxFailedObjectCount: 0 diff --git a/hack/cl2/busy_cluster_burst.yaml b/hack/cl2/busy_cluster_burst.yaml new file mode 100644 index 000000000..cd4fa9c2b --- /dev/null +++ b/hack/cl2/busy_cluster_burst.yaml @@ -0,0 +1,122 @@ +name: busy_cluster_burst_test + +{{$duration := "10m"}} +{{$count := 1000}} + +namespace: + number: 2 + prefix: busy-cluster-burst-test-ns + +tuningSets: +- name: Uniform100qps + qpsLoad: + qps: 100 + +steps: +- name: Create CRPs + phases: + - replicasPerNamespace: 2 + tuningSet: Uniform100qps + objectBundle: + - basename: test-crp + objectTemplatePath: "manifests/test-crp.yaml" +- name: Create ConfigMaps + phases: + - namespaceRange: + min: 1 + max: 2 + replicasPerNamespace: {{$count}} + tuningSet: Uniform100qps + objectBundle: + - basename: test-configmap + objectTemplatePath: "manifests/test-configmap.yaml" +- name: Wait for CRPs to be Ready + measurements: + - Identifier: WaitForGenericK8sObjects + Method: WaitForGenericK8sObjects + Params: + objectGroup: placement.kubernetes-fleet.io + objectVersion: v1beta1 + objectResource: clusterresourceplacements + timeout: {{$duration}} + successfulConditions: + - ClusterResourcePlacementAvailable=True + failedConditions: + - ClusterResourcePlacementAvailable=False + minDesiredObjectCount: 2 + maxFailedObjectCount: 0 +- name: Delete ConfigMaps + phases: + - namespaceRange: + min: 1 + max: 2 + replicasPerNamespace: 0 + tuningSet: Uniform100qps + objectBundle: + - basename: test-configmap + objectTemplatePath: "manifests/test-configmap.yaml" +- name: Wait for CRPs to be Ready + measurements: + - Identifier: WaitForGenericK8sObjects + Method: WaitForGenericK8sObjects + Params: + objectGroup: placement.kubernetes-fleet.io + objectVersion: v1beta1 + objectResource: clusterresourceplacements + timeout: {{$duration}} + successfulConditions: + - ClusterResourcePlacementAvailable=True + failedConditions: + - ClusterResourcePlacementAvailable=False + minDesiredObjectCount: 2 + maxFailedObjectCount: 0 +- name: Create ConfigMaps + phases: + - namespaceRange: + min: 1 + max: 2 + replicasPerNamespace: {{$count}} + tuningSet: Uniform100qps + objectBundle: + - basename: test-configmap + objectTemplatePath: "manifests/test-configmap.yaml" +- name: Wait for CRPs to be Ready + measurements: + - Identifier: WaitForGenericK8sObjects + Method: WaitForGenericK8sObjects + Params: + objectGroup: placement.kubernetes-fleet.io + objectVersion: v1beta1 + objectResource: clusterresourceplacements + timeout: {{$duration}} + successfulConditions: + - ClusterResourcePlacementAvailable=True + failedConditions: + - ClusterResourcePlacementAvailable=False + minDesiredObjectCount: 2 + maxFailedObjectCount: 0 +- name: Delete ConfigMaps + phases: + - namespaceRange: + min: 1 + max: 2 + replicasPerNamespace: 0 + tuningSet: Uniform100qps + objectBundle: + - basename: test-configmap + objectTemplatePath: "manifests/test-configmap.yaml" +- name: Wait for CRPs to be Ready + measurements: + - Identifier: WaitForGenericK8sObjects + Method: WaitForGenericK8sObjects + Params: + objectGroup: placement.kubernetes-fleet.io + objectVersion: v1beta1 + objectResource: clusterresourceplacements + timeout: {{$duration}} + successfulConditions: + - ClusterResourcePlacementAvailable=True + failedConditions: + - ClusterResourcePlacementAvailable=False + minDesiredObjectCount: 2 + maxFailedObjectCount: 0 diff --git a/hack/cl2/cleanup.sh b/hack/cl2/cleanup.sh new file mode 100755 index 000000000..eea5120bf --- /dev/null +++ b/hack/cl2/cleanup.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +: "${KUBECONFIG:?Environment variable KUBECONFIG must be set}" + +echo "Deleting all CRPs generated during the load test from the hub cluster..." +kubectl delete crp -l test=cl2-test diff --git a/hack/cl2/manifests/test-configmap.yaml b/hack/cl2/manifests/test-configmap.yaml new file mode 100644 index 000000000..b7ba9459b --- /dev/null +++ b/hack/cl2/manifests/test-configmap.yaml @@ -0,0 +1,7 @@ +# Test configmap with 1k payload +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{.Name}} +data: + random: 1234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678 diff --git a/hack/cl2/manifests/test-crp.yaml b/hack/cl2/manifests/test-crp.yaml new file mode 100644 index 000000000..465d9c8fe --- /dev/null +++ b/hack/cl2/manifests/test-crp.yaml @@ -0,0 +1,19 @@ +apiVersion: placement.kubernetes-fleet.io/v1beta1 +kind: ClusterResourcePlacement +metadata: + name: {{.Name}} + labels: + test: cl2-test +spec: + resourceSelectors: + - group: "" + kind: Namespace + name: busy-cluster-test-ns-{{AddInt .Index 1}} + version: v1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + maxSurge: 25% + unavailablePeriodSeconds: 60 + revisionHistoryLimit: 15 diff --git a/hack/cl2/modules/configmaps.yaml b/hack/cl2/modules/configmaps.yaml new file mode 100644 index 000000000..808608089 --- /dev/null +++ b/hack/cl2/modules/configmaps.yaml @@ -0,0 +1,14 @@ +{{$namespaces := .namespaceCount}} +{{$count := .configmapCount}} + +steps: +- name: Create ConfigMaps + phases: + - namespaceRange: + min: 1 + max: {{$namespaces}} + replicasPerNamespace: {{$count}} + tuningSet: SteppedLoad + objectBundle: + - basename: test-configmap + objectTemplatePath: "manifests/test-configmap.yaml" From 445b6894d8d1dc7596b210ddbe9e1c3eff513e10 Mon Sep 17 00:00:00 2001 From: Wantong Jiang Date: Thu, 26 Jun 2025 02:34:49 +0000 Subject: [PATCH 2/2] fix comments Signed-off-by: Wantong Jiang --- hack/cl2/manifests/test-configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/cl2/manifests/test-configmap.yaml b/hack/cl2/manifests/test-configmap.yaml index b7ba9459b..a55202f81 100644 --- a/hack/cl2/manifests/test-configmap.yaml +++ b/hack/cl2/manifests/test-configmap.yaml @@ -4,4 +4,4 @@ kind: ConfigMap metadata: name: {{.Name}} data: - random: 1234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678123456781234567812345678 + random: O0RKEYYCLRvp03c8gUPGINapuUPUpJrL1WQQmP5UOngJJsbiPF876mELH1MyD1bw2FKrr4IuOHzjl0hiL7SvO46HRCKsEHAVFhelhTzwdR9CI5A9jjthpNR4t812R15MyumLrqMAhXv4ynykUa7bt1XtzsIhYjARSKthtPvN9Hbge26PX7lhQjsT1vm42bw7HHEfRVc2P0IwZluchc6aqJfhpJcNQGSjN0GjeuzkNP7V7exWkZqtAt7JnZneq3X72XMxrdQB3sdbK3WEYdsajUAyBO0Eznn8Ab8FSfwrvKUCbCQ8C2UophqThWwzAiMw1y5BQjT3EaIqeLPWDhrqkbMHzSbpcdXLHv8Ine5594QTZZYm8II0PZd3HdpoXjzeAAix6w1c4hXFwtQejti3ZDqZNDPrEUuepRradonBRcZcFccLTGiK6BUNNzZMC3yP5DNR2yWE6P30NdeBMb7mkAFhAVUp46KZUghF60Jpu0HSmEHs0gA0dBR2gyyRsWFjT03IGxixZ2sCuOTCEqdvatTVKi6FBjigRNnxZXyL9pzyyXhTRrdcts72lvKsbFp1n0rQmQUItfKVOXC1bdfGUPaoRA4jQXdUAVdJadYfTG12ETgxXGuKc44lkTFGMKuMA4RRZYZRZGJubW18rhfnleiiOFjPL83NJ8g81xFuqf4TWhXdgwhPwe2CvIeF2lfdmyer10Y8GC4Yl7LAFp5GshpmjeDZcuGj31NankEMZLY0znRW7M551yy2TCNHzsDaYtnPR4611wNu3nVQIN8iAXmPnnMev2bDfh8l5A14CUtONGno4DQVs4IyTmpwN1oyOxsXPaL8m2r9FtUzn7JCdepirNfm0ImVUDvlR96v6axjjGvoVWzQtNDQWdnJC6x1KKs9mDuERexSwGc4F56Iec5Q2a2QB9EFxdEOSDKFjGq1az7PHOf8uqOurzFvG6dylwQnKN5oy0PFW8wgxyJO6hsUDfrslO3zrh9gtcOIt9tHL3xo70bF8hPkTinn8xRs