diff --git a/Makefile-test.mk b/Makefile-test.mk index 0b58b6791b..4bda73c6fb 100644 --- a/Makefile-test.mk +++ b/Makefile-test.mk @@ -64,8 +64,9 @@ test: gotestsum ## Run tests. $(GOTESTSUM) --junitfile $(ARTIFACTS)/junit.xml -- $(GO_TEST_FLAGS) $(shell $(GO_CMD) list ./... | grep -v '/test/') -coverpkg=./... -coverprofile $(ARTIFACTS)/cover.out .PHONY: test-integration -test-integration: gomod-download envtest ginkgo mpi-operator-crd ray-operator-crd jobset-operator-crd kf-training-operator-crd cluster-autoscaler-crd ## Run tests. +test-integration: gomod-download envtest ginkgo mpi-operator-crd ray-operator-crd jobset-operator-crd kf-training-operator-crd cluster-autoscaler-crd kueuectl ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" \ + KUEUE_BIN=$(PROJECT_DIR)/bin \ $(GINKGO) $(GINKGO_ARGS) -procs=$(INTEGRATION_NPROCS) --junit-report=junit.xml --output-dir=$(ARTIFACTS) -v $(INTEGRATION_TARGET) CREATE_KIND_CLUSTER ?= true diff --git a/cmd/kueuectl/app/cmd.go b/cmd/kueuectl/app/cmd.go index 32936da43b..316c9ac25f 100644 --- a/cmd/kueuectl/app/cmd.go +++ b/cmd/kueuectl/app/cmd.go @@ -17,6 +17,7 @@ limitations under the License. package app import ( + "fmt" "os" "github.com/spf13/cobra" @@ -25,6 +26,7 @@ import ( "sigs.k8s.io/kueue/cmd/kueuectl/app/create" "sigs.k8s.io/kueue/cmd/kueuectl/app/list" + "sigs.k8s.io/kueue/cmd/kueuectl/app/passthrough" "sigs.k8s.io/kueue/cmd/kueuectl/app/resume" "sigs.k8s.io/kueue/cmd/kueuectl/app/stop" "sigs.k8s.io/kueue/cmd/kueuectl/app/util" @@ -68,6 +70,13 @@ func NewKueuectlCmd(o KueuectlOptions) *cobra.Command { cmd.AddCommand(resume.NewResumeCmd(clientGetter, o.IOStreams)) cmd.AddCommand(stop.NewStopCmd(clientGetter, o.IOStreams)) cmd.AddCommand(list.NewListCmd(clientGetter, o.IOStreams)) + pCommands, err := passthrough.NewCommands() + if err != nil { + // we can still use the other commands, jut push an warning + fmt.Println(err) + } else { + cmd.AddCommand(pCommands...) + } return cmd } diff --git a/cmd/kueuectl/app/passthrough/passthrough.go b/cmd/kueuectl/app/passthrough/passthrough.go new file mode 100644 index 0000000000..0661c045d6 --- /dev/null +++ b/cmd/kueuectl/app/passthrough/passthrough.go @@ -0,0 +1,74 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package passthrough + +import ( + "fmt" + "os" + "os/exec" + "syscall" + + "github.com/spf13/cobra" +) + +var ( + passthroughTypes = []string{"workload", "wl", "clusterqueue", "cq", "localqueue", "lq"} + passthroughCmds = []string{"get", "delete", "edit", "describe", "patch"} +) + +func NewCommands() ([]*cobra.Command, error) { + kubectlPath, err := exec.LookPath("kubectl") + if err != nil { + return nil, fmt.Errorf("pass-through commands are not available: %w, PATH=%q", err, os.Getenv("PATH")) + } + + commands := make([]*cobra.Command, len(passthroughCmds)) + for i, pCmd := range passthroughCmds { + commands[i] = newCmd(kubectlPath, pCmd, passthroughTypes) + } + return commands, nil +} + +func newCmd(kubectlPath string, command string, subcommands []string) *cobra.Command { + cmd := &cobra.Command{ + Use: command, + Short: fmt.Sprintf("Pass-through %q to kubectl", command), + } + + for _, subcommand := range subcommands { + cmd.AddCommand(newSubcommand(kubectlPath, command, subcommand)) + } + + return cmd +} + +func newSubcommand(kubectlPath string, command, subcommand string) *cobra.Command { + cmd := &cobra.Command{ + Use: subcommand, + Short: fmt.Sprintf("Pass-through \"%s %s\" to kubectl", command, subcommand), + FParseErrWhitelist: cobra.FParseErrWhitelist{UnknownFlags: true}, + RunE: func(cmd *cobra.Command, _ []string) error { + // prepare the args + args := os.Args + args[0] = kubectlPath + + // go in kubectl + return syscall.Exec(kubectlPath, args, os.Environ()) + }, + } + return cmd +} diff --git a/keps/2076-kueuectl/README.md b/keps/2076-kueuectl/README.md index 075aaeedd3..11a9349e1c 100644 --- a/keps/2076-kueuectl/README.md +++ b/keps/2076-kueuectl/README.md @@ -302,13 +302,14 @@ None. ### Pass-through -For completeness there will be 4 additional commands that will simply execute regular kubectl +For completeness there will be 5 additional commands that will simply execute regular kubectl so that the users won't have to remember to switch the command to kubectl. -* `delete workload|clusterqueue|cq|localqueue|lq` * `get workload|clusterqueue|cq|localqueue|lq` -* `edit workload|clusterqueue|cq|localqueue|lq` * `describe workload|clusterqueue|cq|localqueue|lq` +* `edit workload|clusterqueue|cq|localqueue|lq` +* `patch workload|clusterqueue|cq|localqueue|lq` +* `delete workload|clusterqueue|cq|localqueue|lq` ### Test Plan @@ -344,4 +345,4 @@ KEP: 2023-04-27. * Use existing kubectl functionality and perform management operations via API manipulations. -* Don't use kubectl plugins but write CLI from scratch. \ No newline at end of file +* Don't use kubectl plugins but write CLI from scratch. diff --git a/site/content/en/docs/reference/kubectl-kueue/commands/passthrough.md b/site/content/en/docs/reference/kubectl-kueue/commands/passthrough.md new file mode 100644 index 0000000000..ee24f921ea --- /dev/null +++ b/site/content/en/docs/reference/kubectl-kueue/commands/passthrough.md @@ -0,0 +1,46 @@ +--- +title: "Pass-through commands" +linkTitle: "Pass-through" +date: 2024-05-14 +weight: 100 +description: > + Commands delegated to `kubectl` +--- + +### Usage: + +``` +kubectl kueue [FLAGS] +``` + +### Examples: + +```bash +# Edit a local queue +kubectl kueue edit localqueue my-local-queue + +# Delete a cluster queue +kubectl kueue delete cq my-cluster-queue +``` + +## Supported commands + +The following table includes a list of commands that are passed through to `kubectl`. + +| Name | Short | +|------------|--------------------------------------------------------------------------------------------------| +| get | Display one or many resources +| describe | Show details of a specific resource or group of resources +| edit | Edit a resource on the server +| patch | Update fields of a resource +| delete | Delete resources by file names, stdin, resources and names, or by resources and label selector + +## Resource types + +The following table includes a list of all the supported resource types and their abbreviated aliases: + +| Name | Short | API version | Namespaced | Kind | +|--------------|-------|------------------------|------------|--------------| +| workload | wl | kueue.x-k8s.io/v1beta1 | true | Workload | +| clusterqueue | cq | kueue.x-k8s.io/v1beta1 | false | ClusterQueue | +| localqueue | lq | kueue.x-k8s.io/v1beta1 | true | LocalQueue | diff --git a/test/integration/kueuectl/passthrough_test.go b/test/integration/kueuectl/passthrough_test.go new file mode 100644 index 0000000000..caf1d236c1 --- /dev/null +++ b/test/integration/kueuectl/passthrough_test.go @@ -0,0 +1,133 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kueuectl + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/set" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/kueue/pkg/util/testing" + "sigs.k8s.io/kueue/test/util" +) + +func makePassThroughWorkload(ns string) client.Object { + return testing.MakeWorkload("pass-through-wl", ns).Obj() +} + +func makePassThroughCQ(_ string) client.Object { + return testing.MakeClusterQueue("pass-through-cq").Obj() +} + +func makePassThroughLQ(ns string) client.Object { + return testing.MakeLocalQueue("pass-through-lq", ns).ClusterQueue("queue").Obj() +} + +func setupEnv(c *exec.Cmd, kassetsPath string, kubeconfigPath string) { + c.Env = os.Environ() + cmdPath := os.Getenv("PATH") + if cmdPath == "" { + cmdPath = kassetsPath + } else if !set.New(filepath.SplitList(cmdPath)...).Has(kassetsPath) { + cmdPath = fmt.Sprintf("%s%c%s", kassetsPath, os.PathListSeparator, cmdPath) + } + + c.Env = append(c.Env, "PATH="+cmdPath) + c.Env = append(c.Env, "KUBECONFIG="+kubeconfigPath) +} + +var _ = ginkgo.Describe("Kueuectl Pass-through", ginkgo.Ordered, ginkgo.ContinueOnFailure, func() { + var ( + ns *corev1.Namespace + ) + + ginkgo.BeforeEach(func() { + ns = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-"}} + gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed()) + }) + + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + }) + + ginkgo.DescribeTable("Pass-through commands", + func(oType string, namespaced bool, makeObject func(ns string) client.Object, getPath string, expectGet string, patch string, expectGetAfterPatch string) { + obj := makeObject(ns.Name) + gomega.Expect(k8sClient.Create(ctx, obj)).To(gomega.Succeed()) + key := client.ObjectKeyFromObject(obj) + + identityArgs := []string{oType, key.Name} + if namespaced { + identityArgs = append(identityArgs, "-n", key.Namespace) + } + + // get it + args := append([]string{"get"}, identityArgs...) + args = append(args, fmt.Sprintf("-o=jsonpath='%s'", getPath)) + cmd := exec.Command(kueuectlPath, args...) + setupEnv(cmd, kassetsPath, kubeconfigPath) + out, err := cmd.CombinedOutput() + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%q", string(out)) + gomega.Expect(string(out)).To(gomega.BeComparableTo(expectGet)) + + // patch it + if patch != "" { + args = append([]string{"patch"}, identityArgs...) + args = append(args, "--type=merge", "-p", patch) + cmd = exec.Command(kueuectlPath, args...) + setupEnv(cmd, kassetsPath, kubeconfigPath) + _, err = cmd.CombinedOutput() + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%q", string(out)) + + args = append([]string{"get"}, identityArgs...) + args = append(args, fmt.Sprintf("-o=jsonpath='%s'", getPath)) + cmd = exec.Command(kueuectlPath, args...) + setupEnv(cmd, kassetsPath, kubeconfigPath) + out, err = cmd.CombinedOutput() + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%q", string(out)) + gomega.Expect(string(out)).To(gomega.BeComparableTo(expectGetAfterPatch)) + } + // delete it + args = []string{"delete", oType} + if namespaced { + args = append(args, "-n", key.Namespace) + } + args = append(args, key.Name) + cmd = exec.Command(kueuectlPath, args...) + setupEnv(cmd, kassetsPath, kubeconfigPath) + _, err = cmd.CombinedOutput() + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%q", string(out)) + + gomega.Eventually(func() error { + return k8sClient.Get(ctx, key, obj) + }, util.Timeout, util.Interval).Should(testing.BeNotFoundError()) + }, + ginkgo.Entry("Workload", "workload", true, makePassThroughWorkload, "{.spec.active}", "'true'", `{"spec":{"active":false}}`, "'false'"), + ginkgo.Entry("Workload(short)", "wl", true, makePassThroughWorkload, "{.spec.active}", "'true'", `{"spec":{"active":false}}`, "'false'"), + ginkgo.Entry("Cluster Queue", "clusterqueue", false, makePassThroughCQ, "{.spec.stopPolicy}", "'None'", `{"spec":{"stopPolicy":"Hold"}}`, "'Hold'"), + ginkgo.Entry("Cluster Queue(short)", "cq", false, makePassThroughCQ, "{.spec.stopPolicy}", "'None'", `{"spec":{"stopPolicy":"Hold"}}`, "'Hold'"), + ginkgo.Entry("Local Queue", "localqueue", true, makePassThroughLQ, "{.spec.clusterQueue}", "'queue'", ``, ""), + ginkgo.Entry("Local Queue (short)", "lq", true, makePassThroughLQ, "{.spec.clusterQueue}", "'queue'", ``, ""), + ) +}) diff --git a/test/integration/kueuectl/suite_test.go b/test/integration/kueuectl/suite_test.go index 9b005a1ce6..2decc1abb9 100644 --- a/test/integration/kueuectl/suite_test.go +++ b/test/integration/kueuectl/suite_test.go @@ -18,6 +18,8 @@ package kueuectl import ( "context" + "os" + "path" "path/filepath" "testing" @@ -32,6 +34,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/core" "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/queue" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" "sigs.k8s.io/kueue/pkg/webhooks" "sigs.k8s.io/kueue/test/integration/framework" // +kubebuilder:scaffold:imports @@ -44,6 +47,10 @@ var ( fwk *framework.Framework crdPath = filepath.Join("..", "..", "..", "config", "components", "crd", "bases") webhookPath = filepath.Join("..", "..", "..", "config", "components", "webhook") + + kueuectlPath string + kassetsPath string + kubeconfigPath string ) func TestKueuectl(t *testing.T) { @@ -55,6 +62,13 @@ var _ = ginkgo.BeforeSuite(func() { fwk = &framework.Framework{CRDPath: crdPath, WebhookPath: webhookPath} cfg = fwk.Init() ctx, k8sClient = fwk.RunManager(cfg, managerSetup) + + kassetsPath = os.Getenv("KUBEBUILDER_ASSETS") + kueuectlPath = path.Join(os.Getenv("KUEUE_BIN"), "kubectl-kueue") + kubeconfigPath = path.Join(ginkgo.GinkgoT().TempDir(), "testing.kubeconfig") + configBites, err := utiltesting.RestConfigToKubeConfig(cfg) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(os.WriteFile(kubeconfigPath, configBites, 0666)).To(gomega.Succeed()) }) var _ = ginkgo.AfterSuite(func() {