test: use clusterloader2 framework for large scale testing

.github/workflows/markdown-lint.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
-    - uses: gaurav-nelson/github-action-markdown-link-check@v1
+    - uses: tcort/github-action-markdown-link-check@v1
       with:
         # this will only show errors in the output
         use-quiet-mode: 'yes'

hack/cl2/README.md

@@ -0,0 +1,24 @@
+This folder contains a list of [clusterloader2](https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/docs/GETTING_STARTED.md) configuration manifests for large scale testing.
+
+## Prerequisites
+
+Follow [clusterloader2 GETTING STARTED](https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/docs/GETTING_STARTED.md) to clone the `perf-tests` repository.
+
+## Execute Tests
+
+Run `clusterloader2` under `perf-tests/clusterloader2` directory:
+```bash
+go run cmd/clusterloader.go --testconfig=<path to the test yaml> --provider=local --kubeconfig=<path to the hub cluster kubeconfig> --v=2 --enable-exec-service=false
+```
+We need to set `--enable-exec-service=false` to prevent creating agnostic deployment on the hub cluster as hub
+cluster does not allow pod creation.
+
+## Cleanup Resources After Tests
+
+By default, clusterloader2 automatically deletes all the generated namespaces after the tests.
+In addition, we also want to delete all the CRPs created. To facilitate cleanup process, run the simple script
+provided in this folder:
+```
+export KUBECONFIG=<your path to the kubeconfig file>
+./cleanup.sh
+```

hack/cl2/busy_cluster.yaml

@@ -0,0 +1,107 @@
+name: busy_cluster_test
+
+{{$duration := "30m"}}
+{{$count := 1}}
+{{$namespaceCount := 1000}}
+
+namespace:
+  number: {{$namespaceCount}}
+  prefix: busy-cluster-test-ns
+
+tuningSets:
+- name: Uniform10qps
+  qpsLoad:
+    qps: 10
+- name: SteppedLoad
+  steppedLoad:
+    burstSize: 20
+    stepDelay: 30s
+
+steps:
+- name: Create CRPs
+  phases:
+  - replicasPerNamespace: {{$namespaceCount}}
+    tuningSet: Uniform10qps
+    objectBundle:
+    - basename: test-crp 
+      objectTemplatePath: "manifests/test-crp.yaml"
+- name: Wait for CRPs to be Ready
+  measurements:
+  - Identifier: WaitForGenericK8sObjects
+    Method: WaitForGenericK8sObjects
+    Params:
+      objectGroup: placement.kubernetes-fleet.io
+      objectVersion: v1beta1
+      objectResource: clusterresourceplacements
+      timeout: {{$duration}}
+      successfulConditions: 
+      - ClusterResourcePlacementAvailable=True
+      failedConditions:
+      - ClusterResourcePlacementAvailable=False
+      minDesiredObjectCount: {{$namespaceCount}}
+      maxFailedObjectCount: 0
+- module:
+    path: /modules/configmaps.yaml
+    params:
+      namespaceCount: {{$namespaceCount}}
+      configmapCount: {{$count}}
+- module:
+    path: /modules/configmaps.yaml
+    params:
+      namespaceCount: {{$namespaceCount}}
+      configmapCount: {{MultiplyInt $count 2}}
+- module:
+    path: /modules/configmaps.yaml
+    params:
+      namespaceCount: {{$namespaceCount}}
+      configmapCount: {{MultiplyInt $count 3}}
+- module:
+    path: /modules/configmaps.yaml
+    params:
+      namespaceCount: {{$namespaceCount}}
+      configmapCount: {{MultiplyInt $count 4}}
+- module:
+    path: /modules/configmaps.yaml
+    params:
+      namespaceCount: {{$namespaceCount}}
+      configmapCount: {{MultiplyInt $count 5}}
+- module:
+    path: /modules/configmaps.yaml
+    params:
+      namespaceCount: {{$namespaceCount}}
+      configmapCount: {{MultiplyInt $count 6}}
+- module:
+    path: /modules/configmaps.yaml
+    params:
+      namespaceCount: {{$namespaceCount}}
+      configmapCount: {{MultiplyInt $count 7}}
+- module:
+    path: /modules/configmaps.yaml
+    params:
+      namespaceCount: {{$namespaceCount}}
+      configmapCount: {{MultiplyInt $count 8}}
+- module:
+    path: /modules/configmaps.yaml
+    params:
+      namespaceCount: {{$namespaceCount}}
+      configmapCount: {{MultiplyInt $count 9}}
+- module:
+    path: /modules/configmaps.yaml
+    params:
+      namespaceCount: {{$namespaceCount}}
+      configmapCount: {{MultiplyInt $count 10}}
+- name: Wait for CRPs to be Ready
+  measurements:
+  - Identifier: WaitForGenericK8sObjects
+    Method: WaitForGenericK8sObjects
+    Params:
+      objectGroup: placement.kubernetes-fleet.io
+      objectVersion: v1beta1
+      objectResource: clusterresourceplacements
+      timeout: {{$duration}}
+      successfulConditions: 
+      - ClusterResourcePlacementAvailable=True
+      failedConditions:
+      - ClusterResourcePlacementAvailable=False
+      minDesiredObjectCount: {{$namespaceCount}}
+      maxFailedObjectCount: 0

hack/cl2/busy_cluster_burst.yaml

@@ -0,0 +1,122 @@
+name: busy_cluster_burst_test
+
+{{$duration := "10m"}}
+{{$count := 1000}}
+
+namespace:
+  number: 2
+  prefix: busy-cluster-burst-test-ns
+
+tuningSets:
+- name: Uniform100qps
+  qpsLoad:
+    qps: 100
+
+steps:
+- name: Create CRPs
+  phases:
+  - replicasPerNamespace: 2
+    tuningSet: Uniform100qps
+    objectBundle:
+    - basename: test-crp 
+      objectTemplatePath: "manifests/test-crp.yaml"
+- name: Create ConfigMaps
+  phases:
+  - namespaceRange:
+      min: 1
+      max: 2
+    replicasPerNamespace: {{$count}}
+    tuningSet: Uniform100qps
+    objectBundle:
+    - basename: test-configmap 
+      objectTemplatePath: "manifests/test-configmap.yaml"
+- name: Wait for CRPs to be Ready
+  measurements:
+  - Identifier: WaitForGenericK8sObjects
+    Method: WaitForGenericK8sObjects
+    Params:
+      objectGroup: placement.kubernetes-fleet.io
+      objectVersion: v1beta1
+      objectResource: clusterresourceplacements
+      timeout: {{$duration}}
+      successfulConditions: 
+      - ClusterResourcePlacementAvailable=True
+      failedConditions:
+      - ClusterResourcePlacementAvailable=False
+      minDesiredObjectCount: 2
+      maxFailedObjectCount: 0
+- name: Delete ConfigMaps
+  phases:
+  - namespaceRange:
+      min: 1
+      max: 2
+    replicasPerNamespace: 0
+    tuningSet: Uniform100qps
+    objectBundle:
+    - basename: test-configmap 
+      objectTemplatePath: "manifests/test-configmap.yaml"
+- name: Wait for CRPs to be Ready
+  measurements:
+  - Identifier: WaitForGenericK8sObjects
+    Method: WaitForGenericK8sObjects
+    Params:
+      objectGroup: placement.kubernetes-fleet.io
+      objectVersion: v1beta1
+      objectResource: clusterresourceplacements
+      timeout: {{$duration}}
+      successfulConditions: 
+      - ClusterResourcePlacementAvailable=True
+      failedConditions:
+      - ClusterResourcePlacementAvailable=False
+      minDesiredObjectCount: 2
+      maxFailedObjectCount: 0
+- name: Create ConfigMaps
+  phases:
+  - namespaceRange:
+      min: 1
+      max: 2
+    replicasPerNamespace: {{$count}}
+    tuningSet: Uniform100qps
+    objectBundle:
+    - basename: test-configmap 
+      objectTemplatePath: "manifests/test-configmap.yaml"
+- name: Wait for CRPs to be Ready
+  measurements:
+  - Identifier: WaitForGenericK8sObjects
+    Method: WaitForGenericK8sObjects
+    Params:
+      objectGroup: placement.kubernetes-fleet.io
+      objectVersion: v1beta1
+      objectResource: clusterresourceplacements
+      timeout: {{$duration}}
+      successfulConditions: 
+      - ClusterResourcePlacementAvailable=True
+      failedConditions:
+      - ClusterResourcePlacementAvailable=False
+      minDesiredObjectCount: 2
+      maxFailedObjectCount: 0
+- name: Delete ConfigMaps
+  phases:
+  - namespaceRange:
+      min: 1
+      max: 2
+    replicasPerNamespace: 0
+    tuningSet: Uniform100qps
+    objectBundle:
+    - basename: test-configmap 
+      objectTemplatePath: "manifests/test-configmap.yaml"
+- name: Wait for CRPs to be Ready
+  measurements:
+  - Identifier: WaitForGenericK8sObjects
+    Method: WaitForGenericK8sObjects
+    Params:
+      objectGroup: placement.kubernetes-fleet.io
+      objectVersion: v1beta1
+      objectResource: clusterresourceplacements
+      timeout: {{$duration}}
+      successfulConditions: 
+      - ClusterResourcePlacementAvailable=True
+      failedConditions:
+      - ClusterResourcePlacementAvailable=False
+      minDesiredObjectCount: 2
+      maxFailedObjectCount: 0

hack/cl2/cleanup.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+: "${KUBECONFIG:?Environment variable KUBECONFIG must be set}"
+
+echo "Deleting all CRPs generated during the load test from the hub cluster..."
+kubectl delete crp -l test=cl2-test

hack/cl2/manifests/test-configmap.yaml

@@ -0,0 +1,7 @@
+# Test configmap with 1k payload
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{.Name}}
+data:
+  random: O0RKEYYCLRvp03c8gUPGINapuUPUpJrL1WQQmP5UOngJJsbiPF876mELH1MyD1bw2FKrr4IuOHzjl0hiL7SvO46HRCKsEHAVFhelhTzwdR9CI5A9jjthpNR4t812R15MyumLrqMAhXv4ynykUa7bt1XtzsIhYjARSKthtPvN9Hbge26PX7lhQjsT1vm42bw7HHEfRVc2P0IwZluchc6aqJfhpJcNQGSjN0GjeuzkNP7V7exWkZqtAt7JnZneq3X72XMxrdQB3sdbK3WEYdsajUAyBO0Eznn8Ab8FSfwrvKUCbCQ8C2UophqThWwzAiMw1y5BQjT3EaIqeLPWDhrqkbMHzSbpcdXLHv8Ine5594QTZZYm8II0PZd3HdpoXjzeAAix6w1c4hXFwtQejti3ZDqZNDPrEUuepRradonBRcZcFccLTGiK6BUNNzZMC3yP5DNR2yWE6P30NdeBMb7mkAFhAVUp46KZUghF60Jpu0HSmEHs0gA0dBR2gyyRsWFjT03IGxixZ2sCuOTCEqdvatTVKi6FBjigRNnxZXyL9pzyyXhTRrdcts72lvKsbFp1n0rQmQUItfKVOXC1bdfGUPaoRA4jQXdUAVdJadYfTG12ETgxXGuKc44lkTFGMKuMA4RRZYZRZGJubW18rhfnleiiOFjPL83NJ8g81xFuqf4TWhXdgwhPwe2CvIeF2lfdmyer10Y8GC4Yl7LAFp5GshpmjeDZcuGj31NankEMZLY0znRW7M551yy2TCNHzsDaYtnPR4611wNu3nVQIN8iAXmPnnMev2bDfh8l5A14CUtONGno4DQVs4IyTmpwN1oyOxsXPaL8m2r9FtUzn7JCdepirNfm0ImVUDvlR96v6axjjGvoVWzQtNDQWdnJC6x1KKs9mDuERexSwGc4F56Iec5Q2a2QB9EFxdEOSDKFjGq1az7PHOf8uqOurzFvG6dylwQnKN5oy0PFW8wgxyJO6hsUDfrslO3zrh9gtcOIt9tHL3xo70bF8hPkTinn8xRs

hack/cl2/manifests/test-crp.yaml

@@ -0,0 +1,19 @@
+apiVersion: placement.kubernetes-fleet.io/v1beta1
+kind: ClusterResourcePlacement
+metadata:
+  name: {{.Name}}
+  labels:
+    test: cl2-test
+spec:
+  resourceSelectors:
+    - group: ""
+      kind: Namespace
+      name: busy-cluster-test-ns-{{AddInt .Index 1}}
+      version: v1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 25%
+      maxSurge: 25%
+      unavailablePeriodSeconds: 60
+  revisionHistoryLimit: 15

hack/cl2/modules/configmaps.yaml

@@ -0,0 +1,14 @@
+{{$namespaces := .namespaceCount}}
+{{$count := .configmapCount}}
+
+steps:
+- name: Create ConfigMaps
+  phases:
+  - namespaceRange:
+      min: 1
+      max: {{$namespaces}}
+    replicasPerNamespace: {{$count}}
+    tuningSet: SteppedLoad
+    objectBundle:
+    - basename: test-configmap 
+      objectTemplatePath: "manifests/test-configmap.yaml"