From 20da28fbb85eb15a2032968398180a066b906839 Mon Sep 17 00:00:00 2001 From: fengluodb Date: Sun, 7 Apr 2024 21:50:13 +0800 Subject: [PATCH] feat: add tpcds cmd --- addons | 2 +- docs/user_docs/cli/cli.md | 1 + docs/user_docs/cli/kbcli_bench.md | 1 + docs/user_docs/cli/kbcli_bench_pgbench.md | 2 +- .../cli/kbcli_bench_redis-benchmark.md | 2 +- docs/user_docs/cli/kbcli_bench_sysbench.md | 2 +- docs/user_docs/cli/kbcli_bench_tpcc.md | 2 +- docs/user_docs/cli/kbcli_bench_tpcds.md | 72 ++++++ docs/user_docs/cli/kbcli_bench_tpch.md | 3 +- docs/user_docs/cli/kbcli_bench_ycsb.md | 2 +- .../cli/kbcli_cluster_rebuild-instance.md | 66 +++++ go.mod | 2 +- go.sum | 2 + pkg/cmd/bench/bench.go | 1 + pkg/cmd/bench/pgbench.go | 2 +- pkg/cmd/bench/redis-benchmark.go | 2 +- pkg/cmd/bench/sysbench.go | 2 +- pkg/cmd/bench/tpcc.go | 2 +- pkg/cmd/bench/tpcds.go | 228 ++++++++++++++++++ pkg/cmd/bench/tpch.go | 5 +- pkg/cmd/bench/ycsb.go | 2 +- pkg/types/types.go | 5 + 22 files changed, 394 insertions(+), 14 deletions(-) create mode 100644 docs/user_docs/cli/kbcli_bench_tpcds.md create mode 100644 docs/user_docs/cli/kbcli_cluster_rebuild-instance.md create mode 100644 pkg/cmd/bench/tpcds.go diff --git a/addons b/addons index a33d64047..306c3a978 160000 --- a/addons +++ b/addons @@ -1 +1 @@ -Subproject commit a33d64047247388fa8671b83e67aa2fecf4e8978 +Subproject commit 306c3a9786dd3937e3fe5ff18491a5007e5dc1ec diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index c53b20771..87d271ec7 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -52,6 +52,7 @@ Run a benchmark. * [kbcli bench redis-benchmark](kbcli_bench_redis-benchmark.md) - Run redis-benchmark on a cluster * [kbcli bench sysbench](kbcli_bench_sysbench.md) - run a SysBench benchmark * [kbcli bench tpcc](kbcli_bench_tpcc.md) - Run tpcc benchmark +* [kbcli bench tpcds](kbcli_bench_tpcds.md) - Run TPC-DS benchmark * [kbcli bench tpch](kbcli_bench_tpch.md) - Run tpch benchmark * [kbcli bench ycsb](kbcli_bench_ycsb.md) - Run YCSB benchmark on a cluster diff --git a/docs/user_docs/cli/kbcli_bench.md b/docs/user_docs/cli/kbcli_bench.md index 21a0ebebe..0e6f8ac6b 100644 --- a/docs/user_docs/cli/kbcli_bench.md +++ b/docs/user_docs/cli/kbcli_bench.md @@ -44,6 +44,7 @@ Run a benchmark. * [kbcli bench redis-benchmark](kbcli_bench_redis-benchmark.md) - Run redis-benchmark on a cluster * [kbcli bench sysbench](kbcli_bench_sysbench.md) - run a SysBench benchmark * [kbcli bench tpcc](kbcli_bench_tpcc.md) - Run tpcc benchmark +* [kbcli bench tpcds](kbcli_bench_tpcds.md) - Run TPC-DS benchmark * [kbcli bench tpch](kbcli_bench_tpch.md) - Run tpch benchmark * [kbcli bench ycsb](kbcli_bench_ycsb.md) - Run YCSB benchmark on a cluster diff --git a/docs/user_docs/cli/kbcli_bench_pgbench.md b/docs/user_docs/cli/kbcli_bench_pgbench.md index 7fa7f69cf..be39436ef 100644 --- a/docs/user_docs/cli/kbcli_bench_pgbench.md +++ b/docs/user_docs/cli/kbcli_bench_pgbench.md @@ -15,7 +15,7 @@ kbcli bench pgbench [Step] [BenchmarkName] [flags] kbcli bench pgbench mytest --cluster pgcluster --database postgres --user xxx --password xxx # pgbench run on a cluster, but with cpu and memory limits set - kbcli bench pgbench mytest --cluster pgcluster --database postgres --user xxx --password xxx --cpu 1 --memory 1Gi + kbcli bench pgbench mytest --cluster pgcluster --database postgres --user xxx --password xxx --limit-cpu 1 --limit-memory 1Gi # pgbench run on a cluster with cleanup, only cleanup by deleting the testdata kbcli bench pgbench cleanup mytest --cluster pgcluster --database postgres --user xxx --password xxx diff --git a/docs/user_docs/cli/kbcli_bench_redis-benchmark.md b/docs/user_docs/cli/kbcli_bench_redis-benchmark.md index 6baae5950..9ae80505a 100644 --- a/docs/user_docs/cli/kbcli_bench_redis-benchmark.md +++ b/docs/user_docs/cli/kbcli_bench_redis-benchmark.md @@ -15,7 +15,7 @@ kbcli bench redis-benchmark [flags] kbcli bench redis-benchmark mytest --cluster rediscluster --clients 50 --requests 10000 --password xxx # redis-benchmark run on a cluster, but with cpu and memory limits set - kbcli bench redis-benchmark mytest --cluster rediscluster --clients 50 --requests 10000 --cpu 1 --memory 1Gi --password xxx + kbcli bench redis-benchmark mytest --cluster rediscluster --clients 50 --requests 10000 --limit-cpu 1 --limit-memory 1Gi --password xxx # redis-benchmark run on a cluster, just test set/get kbcli bench redis-benchmark mytest --cluster rediscluster --clients 50 --requests 10000 --tests set,get --password xxx diff --git a/docs/user_docs/cli/kbcli_bench_sysbench.md b/docs/user_docs/cli/kbcli_bench_sysbench.md index 3eb9ae71a..57c858bc1 100644 --- a/docs/user_docs/cli/kbcli_bench_sysbench.md +++ b/docs/user_docs/cli/kbcli_bench_sysbench.md @@ -15,7 +15,7 @@ kbcli bench sysbench [Step] [BenchmarkName] [flags] kbcli bench sysbench mytest --cluster mycluster --user xxx --password xxx --database mydb # sysbench on a cluster, but with cpu and memory limits set - kbcli bench sysbench mytest --cluster mycluster --user xxx --password xxx --database mydb --cpu 1 --memory 1Gi + kbcli bench sysbench mytest --cluster mycluster --user xxx --password xxx --database mydb --limit-cpu 1 --limit-memory 1Gi # sysbench run on a cluster with cleanup, only cleanup by deleting the testdata kbcli bench sysbench cleanup mytest --cluster mycluster --user xxx --password xxx --database mydb diff --git a/docs/user_docs/cli/kbcli_bench_tpcc.md b/docs/user_docs/cli/kbcli_bench_tpcc.md index dc6050d4c..b0c02cfcd 100644 --- a/docs/user_docs/cli/kbcli_bench_tpcc.md +++ b/docs/user_docs/cli/kbcli_bench_tpcc.md @@ -15,7 +15,7 @@ kbcli bench tpcc [Step] [BenchmarkName] [flags] kbcli bench tpcc mytest --cluster mycluster --user xxx --password xxx --database mydb # tpcc on a cluster, but with cpu and memory limits set - kbcli bench tpcc mytest --cluster mycluster --user xxx --password xxx --database mydb --cpu 1 --memory 1Gi + kbcli bench tpcc mytest --cluster mycluster --user xxx --password xxx --database mydb --limit-cpu 1 --limit-memory 1Gi # tpcc on a cluster with cleanup, only cleanup by deleting the testdata kbcli bench tpcc cleanup mytest --cluster mycluster --user xxx --password xxx --database mydb diff --git a/docs/user_docs/cli/kbcli_bench_tpcds.md b/docs/user_docs/cli/kbcli_bench_tpcds.md new file mode 100644 index 000000000..5d1efc78b --- /dev/null +++ b/docs/user_docs/cli/kbcli_bench_tpcds.md @@ -0,0 +1,72 @@ +--- +title: kbcli bench tpcds +--- + +Run TPC-DS benchmark + +``` +kbcli bench tpcds [Step] [Benchmark] [flags] +``` + +### Examples + +``` + # tpcds on a cluster, that will exec for all steps, cleanup, prepare and run + kbcli bench tpcds mytest --cluster mycluster --user xxx --password xxx --database mydb + + # tpcds on a cluster, but with cpu and memory limits set + kbcli bench tpcds mytest --cluster mycluster --user xxx --password xxx --database mydb --limit-cpu 1 --limit-memory 1Gi + + # tpcds on a cluster with 10GB data + kbcli bench tpcds mytest --cluster mycluster --user xxx --password xxx --database mydb --size 10 +``` + +### Options + +``` + --cluster string the cluster of database + --database string database name + --driver string the driver of database + --extra-args strings extra arguments for benchmark + -h, --help help for tpcds + --host string the host of database + --limit-cpu string the limit cpu of benchmark + --limit-memory string the limit memory of benchmark + --password string the password of database + --port int the port of database + --request-cpu string the request cpu of benchmark + --request-memory string the request memory of benchmark + --size int specify the scale factor of the benchmark, 1 means 1GB data (default 1) + --tolerations strings Tolerations for benchmark, such as '"dev=true:NoSchedule,large=true:NoSchedule"' + --use-key specify whether to create pk and fk, it will take extra time to create the keys + --user string the user of database +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server +``` + +### SEE ALSO + +* [kbcli bench](kbcli_bench.md) - Run a benchmark. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_bench_tpch.md b/docs/user_docs/cli/kbcli_bench_tpch.md index 7e3285cfc..e7fad2e67 100644 --- a/docs/user_docs/cli/kbcli_bench_tpch.md +++ b/docs/user_docs/cli/kbcli_bench_tpch.md @@ -15,7 +15,7 @@ kbcli bench tpch [Step] [BenchmarkName] [flags] kbcli bench tpch mytest --cluster mycluster --user xxx --password xxx --database mydb # tpch on a cluster, but with cpu and memory limits set - kbcli bench tpch mytest --cluster mycluster --user xxx --password xxx --database mydb --cpu 1 --memory 1Gi + kbcli bench tpch mytest --cluster mycluster --user xxx --password xxx --database mydb --limit-cpu 1 --limit-memory 1Gi # tpch on a cluster with run, just run by running the test kbcli bench tpch run mytest --cluster mycluster --user xxx --password xxx --database mydb @@ -36,6 +36,7 @@ kbcli bench tpch [Step] [BenchmarkName] [flags] --port int the port of database --request-cpu string the request cpu of benchmark --request-memory string the request memory of benchmark + --size int specify the overall database size scaling parameter, 1 means 1GB (default 1) --tolerations strings Tolerations for benchmark, such as '"dev=true:NoSchedule,large=true:NoSchedule"' --user string the user of database ``` diff --git a/docs/user_docs/cli/kbcli_bench_ycsb.md b/docs/user_docs/cli/kbcli_bench_ycsb.md index 598e24d4a..4e3322410 100644 --- a/docs/user_docs/cli/kbcli_bench_ycsb.md +++ b/docs/user_docs/cli/kbcli_bench_ycsb.md @@ -15,7 +15,7 @@ kbcli bench ycsb [Step] [BenchmarkName] [flags] kbcli bench ycsb mytest --cluster mycluster --user xxx --password xxx --database mydb # ycsb on a cluster, but with cpu and memory limits set - kbcli bench ycsb mytest --cluster mycluster --user xxx --password xxx --database mydb --cpu 1 --memory 1Gi + kbcli bench ycsb mytest --cluster mycluster --user xxx --password xxx --database mydb --limit-cpu 1 --limit-memory 1Gi # ycsb on a cluster with cleanup, only cleanup by deleting the testdata kbcli bench ycsb cleanup mytest --cluster mycluster --user xxx --password xxx --database mydb diff --git a/docs/user_docs/cli/kbcli_cluster_rebuild-instance.md b/docs/user_docs/cli/kbcli_cluster_rebuild-instance.md new file mode 100644 index 000000000..2c5d6f612 --- /dev/null +++ b/docs/user_docs/cli/kbcli_cluster_rebuild-instance.md @@ -0,0 +1,66 @@ +--- +title: kbcli cluster rebuild-instance +--- + +Rebuild the specified instances in the cluster. + +``` +kbcli cluster rebuild-instance NAME [flags] +``` + +### Examples + +``` + # rebuild instance without backup + kbcli cluster rebuild-instance mycluster --instances pod1,pod2 + + # rebuild instance from backup + kbcli cluster rebuild-instance mycluster --instances pod1,pod2 --backupName +``` + +### Options + +``` + --auto-approve Skip interactive approval before rebuilding the instances.gi + --backup string instances will be rebuild by the specified backup. + --dry-run string[="unchanged"] Must be "client", or "server". If with client strategy, only print the object that would be sent, and no data is actually sent. If with server strategy, submit the server-side request, but no data is persistent. (default "none") + --env stringArray provide the necessary env for the 'Restore' operation from the backup. format: key1=value, key2=value + --force skip the pre-checks of the opsRequest to run the opsRequest forcibly + -h, --help help for rebuild-instance + --instance strings instance which need to rebuild. + --name string OpsRequest name. if not specified, it will be randomly generated + --node strings specified the target node which rebuilds the instance on the node otherwise will rebuild on a randon node. format: insName1=nodeName,insName2=nodeName + -o, --output format Prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli cluster](kbcli_cluster.md) - Cluster command. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/go.mod b/go.mod index d28b4ecea..0ec7712ff 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( cuelang.org/go v0.8.0 github.com/99designs/keyring v1.2.2 github.com/Masterminds/semver/v3 v3.2.1 - github.com/apecloud/kubebench v0.0.0-20240313105909-ba8654f654fc + github.com/apecloud/kubebench v0.0.0-20240327101848-6a031d3f4ebe github.com/apecloud/kubeblocks v0.9.0-beta.1 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 github.com/benbjohnson/clock v1.3.5 diff --git a/go.sum b/go.sum index 2a0730645..01f174dad 100644 --- a/go.sum +++ b/go.sum @@ -274,6 +274,8 @@ github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/apecloud/kubebench v0.0.0-20240313105909-ba8654f654fc h1:BkFyEBEmDMdDhkWNAs9yg6zAj7xZjWON2u8b265T2a8= github.com/apecloud/kubebench v0.0.0-20240313105909-ba8654f654fc/go.mod h1:5IZiDkFdgiZRGLsL+FOlvPsiF9LbyU55DVj4/5vT7+4= +github.com/apecloud/kubebench v0.0.0-20240327101848-6a031d3f4ebe h1:OFkCAToRcNt0VqgCndw0mw1Ke3P2XgRJLrXsNOyc3sQ= +github.com/apecloud/kubebench v0.0.0-20240327101848-6a031d3f4ebe/go.mod h1:5IZiDkFdgiZRGLsL+FOlvPsiF9LbyU55DVj4/5vT7+4= github.com/apecloud/kubeblocks v0.9.0-beta.1 h1:spCqfGH53vXVQhrvemZeSGJpx+D60TJ36PGVJvCXENQ= github.com/apecloud/kubeblocks v0.9.0-beta.1/go.mod h1:l2pwRoBU560naATQrKe6/jaWU4JGg3TWGwQ2GcrpC5c= github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= diff --git a/pkg/cmd/bench/bench.go b/pkg/cmd/bench/bench.go index 61de5cab8..f3eee2133 100644 --- a/pkg/cmd/bench/bench.go +++ b/pkg/cmd/bench/bench.go @@ -197,6 +197,7 @@ func NewBenchCmd(f cmdutil.Factory, streams genericiooptions.IOStreams) *cobra.C NewPgBenchCmd(f, streams), NewYcsbCmd(f, streams), NewTpccCmd(f, streams), + NewTpcdsCmd(f, streams), NewTpchCmd(f, streams), NewRedisBenchmarkCmd(f, streams), newListCmd(f, streams), diff --git a/pkg/cmd/bench/pgbench.go b/pkg/cmd/bench/pgbench.go index 5a74594fd..47aa139d1 100644 --- a/pkg/cmd/bench/pgbench.go +++ b/pkg/cmd/bench/pgbench.go @@ -46,7 +46,7 @@ var pgbenchExample = templates.Examples(` kbcli bench pgbench mytest --cluster pgcluster --database postgres --user xxx --password xxx # pgbench run on a cluster, but with cpu and memory limits set - kbcli bench pgbench mytest --cluster pgcluster --database postgres --user xxx --password xxx --cpu 1 --memory 1Gi + kbcli bench pgbench mytest --cluster pgcluster --database postgres --user xxx --password xxx --limit-cpu 1 --limit-memory 1Gi # pgbench run on a cluster with cleanup, only cleanup by deleting the testdata kbcli bench pgbench cleanup mytest --cluster pgcluster --database postgres --user xxx --password xxx diff --git a/pkg/cmd/bench/redis-benchmark.go b/pkg/cmd/bench/redis-benchmark.go index 53ce13f22..6d9b5b5b2 100644 --- a/pkg/cmd/bench/redis-benchmark.go +++ b/pkg/cmd/bench/redis-benchmark.go @@ -46,7 +46,7 @@ var redisBenchExample = templates.Examples(` kbcli bench redis-benchmark mytest --cluster rediscluster --clients 50 --requests 10000 --password xxx # redis-benchmark run on a cluster, but with cpu and memory limits set - kbcli bench redis-benchmark mytest --cluster rediscluster --clients 50 --requests 10000 --cpu 1 --memory 1Gi --password xxx + kbcli bench redis-benchmark mytest --cluster rediscluster --clients 50 --requests 10000 --limit-cpu 1 --limit-memory 1Gi --password xxx # redis-benchmark run on a cluster, just test set/get kbcli bench redis-benchmark mytest --cluster rediscluster --clients 50 --requests 10000 --tests set,get --password xxx diff --git a/pkg/cmd/bench/sysbench.go b/pkg/cmd/bench/sysbench.go index 2c0cb353f..5d14188b3 100644 --- a/pkg/cmd/bench/sysbench.go +++ b/pkg/cmd/bench/sysbench.go @@ -51,7 +51,7 @@ var sysbenchExample = templates.Examples(` kbcli bench sysbench mytest --cluster mycluster --user xxx --password xxx --database mydb # sysbench on a cluster, but with cpu and memory limits set - kbcli bench sysbench mytest --cluster mycluster --user xxx --password xxx --database mydb --cpu 1 --memory 1Gi + kbcli bench sysbench mytest --cluster mycluster --user xxx --password xxx --database mydb --limit-cpu 1 --limit-memory 1Gi # sysbench run on a cluster with cleanup, only cleanup by deleting the testdata kbcli bench sysbench cleanup mytest --cluster mycluster --user xxx --password xxx --database mydb diff --git a/pkg/cmd/bench/tpcc.go b/pkg/cmd/bench/tpcc.go index 2d9d4e187..30d5832e3 100644 --- a/pkg/cmd/bench/tpcc.go +++ b/pkg/cmd/bench/tpcc.go @@ -51,7 +51,7 @@ var tpccExample = templates.Examples(` kbcli bench tpcc mytest --cluster mycluster --user xxx --password xxx --database mydb # tpcc on a cluster, but with cpu and memory limits set - kbcli bench tpcc mytest --cluster mycluster --user xxx --password xxx --database mydb --cpu 1 --memory 1Gi + kbcli bench tpcc mytest --cluster mycluster --user xxx --password xxx --database mydb --limit-cpu 1 --limit-memory 1Gi # tpcc on a cluster with cleanup, only cleanup by deleting the testdata kbcli bench tpcc cleanup mytest --cluster mycluster --user xxx --password xxx --database mydb diff --git a/pkg/cmd/bench/tpcds.go b/pkg/cmd/bench/tpcds.go new file mode 100644 index 000000000..3b0604787 --- /dev/null +++ b/pkg/cmd/bench/tpcds.go @@ -0,0 +1,228 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package bench + +import ( + "context" + "fmt" + "strings" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubebench/api/v1alpha1" + + "github.com/apecloud/kbcli/pkg/cluster" + "github.com/apecloud/kbcli/pkg/types" +) + +var ( + tpcdsDriverMap = map[string]string{ + "mysql": "mysql", + "postgresql": "postgresql", + } + tpcdsSupportedDrivers = []string{"mysql", "postgresql"} +) + +type TpcdsOptions struct { + BenchBaseOptions + + Size int + UseKey bool +} + +var tpcdsExample = templates.Examples(` + # tpcds on a cluster, that will exec for all steps, cleanup, prepare and run + kbcli bench tpcds mytest --cluster mycluster --user xxx --password xxx --database mydb + + # tpcds on a cluster, but with cpu and memory limits set + kbcli bench tpcds mytest --cluster mycluster --user xxx --password xxx --database mydb --limit-cpu 1 --limit-memory 1Gi + + # tpcds on a cluster with 10GB data + kbcli bench tpcds mytest --cluster mycluster --user xxx --password xxx --database mydb --size 10 +`) + +func NewTpcdsCmd(f cmdutil.Factory, streams genericiooptions.IOStreams) *cobra.Command { + o := &TpcdsOptions{ + BenchBaseOptions: BenchBaseOptions{ + IOStreams: streams, + factory: f, + }, + } + + cmd := &cobra.Command{ + Use: "tpcds [Step] [Benchmark]", + Short: "Run TPC-DS benchmark", + Example: tpcdsExample, + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(o.Complete(args)) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Run()) + }, + } + + o.AddFlags(cmd) + cmd.Flags().IntVar(&o.Size, "size", 1, "specify the scale factor of the benchmark, 1 means 1GB data") + cmd.Flags().BoolVar(&o.UseKey, "use-key", false, "specify whether to create pk and fk, it will take extra time to create the keys") + + return cmd +} + +func (o *TpcdsOptions) Complete(args []string) error { + var err error + var driver string + var host string + var port int + + if err := o.BenchBaseOptions.BaseComplete(); err != nil { + return err + } + + o.Step, o.name = parseStepAndName(args, "tpcds") + + o.namespace, _, err = o.factory.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + if o.dynamic, err = o.factory.DynamicClient(); err != nil { + return err + } + + if o.client, err = o.factory.KubernetesClientSet(); err != nil { + return err + } + + if o.ClusterName != "" { + clusterGetter := cluster.ObjectsGetter{ + Client: o.client, + Dynamic: o.dynamic, + Name: o.ClusterName, + Namespace: o.namespace, + GetOptions: cluster.GetOptions{ + WithClusterDef: cluster.Maybe, + WithService: cluster.Need, + WithPod: cluster.Need, + WithEvent: cluster.Need, + WithPVC: cluster.Need, + WithDataProtection: cluster.Need, + }, + } + if o.ClusterObjects, err = clusterGetter.Get(); err != nil { + return err + } + driver, host, port, err = getDriverAndHostAndPort(o.Cluster, o.Services) + if err != nil { + return err + } + } + + // don't overwrite the driver if it's already set + if v, ok := tpcdsDriverMap[driver]; ok && o.Driver == "" { + o.Driver = v + } + + // don't overwrite the host and port if they are already set + if o.Host == "" && o.Port == 0 { + o.Host = host + o.Port = port + } + + return nil +} + +func (o *TpcdsOptions) Validate() error { + if err := o.BenchBaseOptions.BaseValidate(); err != nil { + return err + } + + var supported bool + for _, v := range tpcdsDriverMap { + if o.Driver == v { + supported = true + break + } + } + if !supported { + return fmt.Errorf("tpcds now only supports drivers in [%s], current cluster driver is %s", + strings.Join(tpcdsSupportedDrivers, ","), o.Driver) + } + + if o.User == "" { + return fmt.Errorf("user is required") + } + + return nil +} + +func (o *TpcdsOptions) Run() error { + tpcds := v1alpha1.Tpcds{ + TypeMeta: metav1.TypeMeta{ + Kind: "Tpcds", + APIVersion: types.TpcdsGVR().GroupVersion().String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: o.name, + Namespace: o.namespace, + }, + Spec: v1alpha1.TpcdsSpec{ + BenchCommon: v1alpha1.BenchCommon{ + ExtraArgs: o.ExtraArgs, + Step: o.Step, + Tolerations: o.Tolerations, + Target: v1alpha1.Target{ + Driver: o.Driver, + Host: o.Host, + Port: o.Port, + User: o.User, + Password: o.Password, + Database: o.Database, + }, + }, + Size: o.Size, + UseKey: o.UseKey, + }, + } + + // set cpu and memory if specified + setCPUAndMemory(&tpcds.Spec.BenchCommon, o.RequestCPU, o.RequestMemory, o.LimitCPU, o.LimitMemory) + + obj := &unstructured.Unstructured{ + Object: map[string]interface{}{}, + } + data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&tpcds) + if err != nil { + return err + } + obj.SetUnstructuredContent(data) + + obj, err = o.dynamic.Resource(types.TpcdsGVR()).Namespace(o.namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + if err != nil { + return err + } + + fmt.Fprintf(o.Out, "%s %s created\n", obj.GetKind(), obj.GetName()) + return nil +} diff --git a/pkg/cmd/bench/tpch.go b/pkg/cmd/bench/tpch.go index 0816aa311..eb8dd48fa 100644 --- a/pkg/cmd/bench/tpch.go +++ b/pkg/cmd/bench/tpch.go @@ -50,7 +50,7 @@ var tpchExample = templates.Examples(` kbcli bench tpch mytest --cluster mycluster --user xxx --password xxx --database mydb # tpch on a cluster, but with cpu and memory limits set - kbcli bench tpch mytest --cluster mycluster --user xxx --password xxx --database mydb --cpu 1 --memory 1Gi + kbcli bench tpch mytest --cluster mycluster --user xxx --password xxx --database mydb --limit-cpu 1 --limit-memory 1Gi # tpch on a cluster with run, just run by running the test kbcli bench tpch run mytest --cluster mycluster --user xxx --password xxx --database mydb @@ -58,6 +58,7 @@ var tpchExample = templates.Examples(` type TpchOptions struct { BenchBaseOptions + Size int } func NewTpchCmd(f cmdutil.Factory, streams genericiooptions.IOStreams) *cobra.Command { @@ -79,6 +80,7 @@ func NewTpchCmd(f cmdutil.Factory, streams genericiooptions.IOStreams) *cobra.Co } o.AddFlags(cmd) + cmd.Flags().IntVar(&o.Size, "size", 1, "specify the overall database size scaling parameter, 1 means 1GB") return cmd } @@ -198,6 +200,7 @@ func (o *TpchOptions) Run() error { Database: o.Database, }, }, + Size: o.Size, }, } diff --git a/pkg/cmd/bench/ycsb.go b/pkg/cmd/bench/ycsb.go index 4f6f44e70..39e984c3a 100644 --- a/pkg/cmd/bench/ycsb.go +++ b/pkg/cmd/bench/ycsb.go @@ -53,7 +53,7 @@ var ycsbExample = templates.Examples(` kbcli bench ycsb mytest --cluster mycluster --user xxx --password xxx --database mydb # ycsb on a cluster, but with cpu and memory limits set - kbcli bench ycsb mytest --cluster mycluster --user xxx --password xxx --database mydb --cpu 1 --memory 1Gi + kbcli bench ycsb mytest --cluster mycluster --user xxx --password xxx --database mydb --limit-cpu 1 --limit-memory 1Gi # ycsb on a cluster with cleanup, only cleanup by deleting the testdata kbcli bench ycsb cleanup mytest --cluster mycluster --user xxx --password xxx --database mydb diff --git a/pkg/types/types.go b/pkg/types/types.go index 4ae558986..f98ac1942 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -216,6 +216,7 @@ const ( ResourceYcsb = "ycsbs" ResourceTpcc = "tpccs" ResourceTpch = "tpches" + ResourceTpcds = "tpcds" ResourceRedisBench = "redisbenches" ) @@ -507,6 +508,10 @@ func TpchGVR() schema.GroupVersionResource { return schema.GroupVersionResource{Group: KubebenchAPIGroup, Version: KubebenchAPIVersion, Resource: ResourceTpch} } +func TpcdsGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: KubebenchAPIGroup, Version: KubebenchAPIVersion, Resource: ResourceTpcds} +} + func RedisBenchGVR() schema.GroupVersionResource { return schema.GroupVersionResource{Group: KubebenchAPIGroup, Version: KubebenchAPIVersion, Resource: ResourceRedisBench} }