diff --git a/goldens/Basic_cluster_create.txt b/goldens/Basic_cluster_create.txt index e4dad6601..a3a62c57c 100644 --- a/goldens/Basic_cluster_create.txt +++ b/goldens/Basic_cluster_create.txt @@ -14,9 +14,14 @@ gcloud beta container clusters create golden-cluster --project=golden-project -- gcloud container clusters describe golden-cluster --project=golden-project --region=us-central1 --format="value(privateClusterConfig.enablePrivateNodes)" [XPK] Private Nodes is not enabled on the cluster. [XPK] Cluster is public and no need to authorize networks. -[XPK] Try 1: get-credentials to cluster golden-cluster -[XPK] Task: `get-credentials to cluster golden-cluster` is implemented by the following command not running since it is a dry run. -gcloud container clusters get-credentials golden-cluster --region=us-central1 --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Try 1: get-credentials-dns-endpoint to cluster golden-cluster +[XPK] Task: `get-credentials-dns-endpoint to cluster golden-cluster` is implemented by the following command not running since it is a dry run. +gcloud container clusters get-credentials golden-cluster --region=us-central1 --dns-endpoint --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Testing credentials with kubectl... +[XPK] Task: `kubectl get pods` is implemented by the following command not running since it is a dry run. +kubectl get pods +[XPK] Credentials test succeeded. +[XPK] Finished get-credentials and kubectl setup. [XPK] Task: 'Checking CoreDNS deployment existence' in progress for namespace: kube-system [XPK] Task: `Check CoreDNS deployment in kube-system` is implemented by the following command not running since it is a dry run. kubectl get deployment coredns -n kube-system @@ -70,7 +75,7 @@ kubectl kueue version kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.12.2/manifests.yaml [XPK] Wait for Kueue to be fully available [XPK] Task: `Wait for Kueue to be available` is implemented by the following command not running since it is a dry run. -kubectl wait deploy/kueue-controller-manager -nkueue-system --for=condition=available --timeout=10m +kubectl wait deploy/kueue-controller-manager -n kueue-system --for=condition=available --timeout=10m [XPK] Install Kueue Custom Resources [XPK] Try 1: Applying Kueue Custom Resources [XPK] Task: `Applying Kueue Custom Resources` is implemented by the following command not running since it is a dry run. diff --git a/goldens/Batch.txt b/goldens/Batch.txt index c87e66ef6..a7f1b5d15 100644 --- a/goldens/Batch.txt +++ b/goldens/Batch.txt @@ -1,9 +1,14 @@ $ python3 xpk.py batch --project=golden-project --zone=us-central1-a --cluster=golden-cluster --dry-run batch-read.sh [XPK] Starting xpk [XPK] Working on golden-project and us-central1-a -[XPK] Try 1: get-credentials to cluster golden-cluster -[XPK] Task: `get-credentials to cluster golden-cluster` is implemented by the following command not running since it is a dry run. -gcloud container clusters get-credentials golden-cluster --region=us-central1 --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Try 1: get-credentials-dns-endpoint to cluster golden-cluster +[XPK] Task: `get-credentials-dns-endpoint to cluster golden-cluster` is implemented by the following command not running since it is a dry run. +gcloud container clusters get-credentials golden-cluster --region=us-central1 --dns-endpoint --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Testing credentials with kubectl... +[XPK] Task: `kubectl get pods` is implemented by the following command not running since it is a dry run. +kubectl get pods +[XPK] Credentials test succeeded. +[XPK] Finished get-credentials and kubectl setup. [XPK] Task: `GKE Cluster Get ConfigMap` is implemented by the following command not running since it is a dry run. kubectl get configmap golden-cluster-resources-configmap -o=custom-columns="ConfigData:data" --no-headers=true [XPK] Task: `GKE Cluster Get ConfigMap` is implemented by the following command not running since it is a dry run. diff --git a/goldens/Cluster_create_private.txt b/goldens/Cluster_create_private.txt index 8c584fc07..3a74d4047 100644 --- a/goldens/Cluster_create_private.txt +++ b/goldens/Cluster_create_private.txt @@ -16,9 +16,14 @@ gcloud container clusters describe golden-cluster-private --project=golden-proje [XPK] Task: `Fetching the list of authorized network from cluster describe.` is implemented by the following command not running since it is a dry run. gcloud container clusters describe golden-cluster-private --project=golden-project --region=us-central1 --format="value(masterAuthorizedNetworksConfig.cidrBlocks[].cidrBlock)" [XPK] Current machine's IP adrress is already authorized. -[XPK] Try 1: get-credentials to cluster golden-cluster-private -[XPK] Task: `get-credentials to cluster golden-cluster-private` is implemented by the following command not running since it is a dry run. -gcloud container clusters get-credentials golden-cluster-private --region=us-central1 --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Try 1: get-credentials-dns-endpoint to cluster golden-cluster-private +[XPK] Task: `get-credentials-dns-endpoint to cluster golden-cluster-private` is implemented by the following command not running since it is a dry run. +gcloud container clusters get-credentials golden-cluster-private --region=us-central1 --dns-endpoint --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Testing credentials with kubectl... +[XPK] Task: `kubectl get pods` is implemented by the following command not running since it is a dry run. +kubectl get pods +[XPK] Credentials test succeeded. +[XPK] Finished get-credentials and kubectl setup. [XPK] Task: 'Checking CoreDNS deployment existence' in progress for namespace: kube-system [XPK] Task: `Check CoreDNS deployment in kube-system` is implemented by the following command not running since it is a dry run. kubectl get deployment coredns -n kube-system @@ -77,7 +82,7 @@ kubectl kueue version kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.12.2/manifests.yaml [XPK] Wait for Kueue to be fully available [XPK] Task: `Wait for Kueue to be available` is implemented by the following command not running since it is a dry run. -kubectl wait deploy/kueue-controller-manager -nkueue-system --for=condition=available --timeout=10m +kubectl wait deploy/kueue-controller-manager -n kueue-system --for=condition=available --timeout=10m [XPK] Install Kueue Custom Resources [XPK] Try 1: Applying Kueue Custom Resources [XPK] Task: `Applying Kueue Custom Resources` is implemented by the following command not running since it is a dry run. diff --git a/goldens/Cluster_create_with_gb200-4.txt b/goldens/Cluster_create_with_gb200-4.txt index caaa33758..e1fc53681 100644 --- a/goldens/Cluster_create_with_gb200-4.txt +++ b/goldens/Cluster_create_with_gb200-4.txt @@ -14,9 +14,14 @@ gcloud beta container clusters create golden-cluster --project=golden-project -- gcloud container clusters describe golden-cluster --project=golden-project --region=us-central1 --format="value(privateClusterConfig.enablePrivateNodes)" [XPK] Private Nodes is not enabled on the cluster. [XPK] Cluster is public and no need to authorize networks. -[XPK] Try 1: get-credentials to cluster golden-cluster -[XPK] Task: `get-credentials to cluster golden-cluster` is implemented by the following command not running since it is a dry run. -gcloud container clusters get-credentials golden-cluster --region=us-central1 --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Try 1: get-credentials-dns-endpoint to cluster golden-cluster +[XPK] Task: `get-credentials-dns-endpoint to cluster golden-cluster` is implemented by the following command not running since it is a dry run. +gcloud container clusters get-credentials golden-cluster --region=us-central1 --dns-endpoint --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Testing credentials with kubectl... +[XPK] Task: `kubectl get pods` is implemented by the following command not running since it is a dry run. +kubectl get pods +[XPK] Credentials test succeeded. +[XPK] Finished get-credentials and kubectl setup. [XPK] Task: 'Checking CoreDNS deployment existence' in progress for namespace: kube-system [XPK] Task: `Check CoreDNS deployment in kube-system` is implemented by the following command not running since it is a dry run. kubectl get deployment coredns -n kube-system @@ -76,7 +81,7 @@ kubectl kueue version kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.12.2/manifests.yaml [XPK] Wait for Kueue to be fully available [XPK] Task: `Wait for Kueue to be available` is implemented by the following command not running since it is a dry run. -kubectl wait deploy/kueue-controller-manager -nkueue-system --for=condition=available --timeout=10m +kubectl wait deploy/kueue-controller-manager -n kueue-system --for=condition=available --timeout=10m [XPK] Install Kueue Custom Resources [XPK] Try 1: Applying Kueue Custom Resources [XPK] Task: `Applying Kueue Custom Resources` is implemented by the following command not running since it is a dry run. diff --git a/goldens/Job_cancel.txt b/goldens/Job_cancel.txt index 7e71ece4a..60118f0a8 100644 --- a/goldens/Job_cancel.txt +++ b/goldens/Job_cancel.txt @@ -2,9 +2,14 @@ $ python3 xpk.py job cancel golden-job --project=golden-project --zone=us-centra [XPK] Starting xpk [XPK] Starting job cancel for job: ['golden-job'] [XPK] Working on golden-project and us-central1-a -[XPK] Try 1: get-credentials to cluster golden-cluster -[XPK] Task: `get-credentials to cluster golden-cluster` is implemented by the following command not running since it is a dry run. -gcloud container clusters get-credentials golden-cluster --region=us-central1 --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Try 1: get-credentials-dns-endpoint to cluster golden-cluster +[XPK] Task: `get-credentials-dns-endpoint to cluster golden-cluster` is implemented by the following command not running since it is a dry run. +gcloud container clusters get-credentials golden-cluster --region=us-central1 --dns-endpoint --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Testing credentials with kubectl... +[XPK] Task: `kubectl get pods` is implemented by the following command not running since it is a dry run. +kubectl get pods +[XPK] Credentials test succeeded. +[XPK] Finished get-credentials and kubectl setup. [XPK] Task: `delete job` is implemented by the following command not running since it is a dry run. kubectl-kjob delete slurm golden-job [XPK] Exiting XPK cleanly diff --git a/goldens/Job_list.txt b/goldens/Job_list.txt index e659f2a6a..8b1c00c49 100644 --- a/goldens/Job_list.txt +++ b/goldens/Job_list.txt @@ -1,9 +1,14 @@ $ python3 xpk.py job ls --project=golden-project --zone=us-central1-a --cluster=golden-cluster --dry-run [XPK] Starting xpk [XPK] Working on golden-project and us-central1-a -[XPK] Try 1: get-credentials to cluster golden-cluster -[XPK] Task: `get-credentials to cluster golden-cluster` is implemented by the following command not running since it is a dry run. -gcloud container clusters get-credentials golden-cluster --region=us-central1 --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Try 1: get-credentials-dns-endpoint to cluster golden-cluster +[XPK] Task: `get-credentials-dns-endpoint to cluster golden-cluster` is implemented by the following command not running since it is a dry run. +gcloud container clusters get-credentials golden-cluster --region=us-central1 --dns-endpoint --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Testing credentials with kubectl... +[XPK] Task: `kubectl get pods` is implemented by the following command not running since it is a dry run. +kubectl get pods +[XPK] Credentials test succeeded. +[XPK] Finished get-credentials and kubectl setup. [XPK] Listing jobs for project golden-project and zone us-central1-a: [XPK] Task: `list jobs` is implemented by the following command not running since it is a dry run. kubectl-kjob list slurm --profile xpk-def-app-profile diff --git a/goldens/NAP_cluster-create.txt b/goldens/NAP_cluster-create.txt index f765cd9bc..3f29115f5 100644 --- a/goldens/NAP_cluster-create.txt +++ b/goldens/NAP_cluster-create.txt @@ -14,9 +14,14 @@ gcloud beta container clusters create golden-cluster --project=golden-project -- gcloud container clusters describe golden-cluster --project=golden-project --region=us-central1 --format="value(privateClusterConfig.enablePrivateNodes)" [XPK] Private Nodes is not enabled on the cluster. [XPK] Cluster is public and no need to authorize networks. -[XPK] Try 1: get-credentials to cluster golden-cluster -[XPK] Task: `get-credentials to cluster golden-cluster` is implemented by the following command not running since it is a dry run. -gcloud container clusters get-credentials golden-cluster --region=us-central1 --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Try 1: get-credentials-dns-endpoint to cluster golden-cluster +[XPK] Task: `get-credentials-dns-endpoint to cluster golden-cluster` is implemented by the following command not running since it is a dry run. +gcloud container clusters get-credentials golden-cluster --region=us-central1 --dns-endpoint --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Testing credentials with kubectl... +[XPK] Task: `kubectl get pods` is implemented by the following command not running since it is a dry run. +kubectl get pods +[XPK] Credentials test succeeded. +[XPK] Finished get-credentials and kubectl setup. [XPK] Task: 'Checking CoreDNS deployment existence' in progress for namespace: kube-system [XPK] Task: `Check CoreDNS deployment in kube-system` is implemented by the following command not running since it is a dry run. kubectl get deployment coredns -n kube-system @@ -81,7 +86,7 @@ kubectl kueue version kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.12.2/manifests.yaml [XPK] Wait for Kueue to be fully available [XPK] Task: `Wait for Kueue to be available` is implemented by the following command not running since it is a dry run. -kubectl wait deploy/kueue-controller-manager -nkueue-system --for=condition=available --timeout=10m +kubectl wait deploy/kueue-controller-manager -n kueue-system --for=condition=available --timeout=10m [XPK] Install Kueue Custom Resources [XPK] Try 1: Applying Kueue Custom Resources [XPK] Task: `Applying Kueue Custom Resources` is implemented by the following command not running since it is a dry run. diff --git a/goldens/NAP_cluster-create_with_pathways.txt b/goldens/NAP_cluster-create_with_pathways.txt index 51982dbfb..9b83bba68 100644 --- a/goldens/NAP_cluster-create_with_pathways.txt +++ b/goldens/NAP_cluster-create_with_pathways.txt @@ -14,9 +14,14 @@ gcloud beta container clusters create golden-cluster --project=golden-project -- gcloud container clusters describe golden-cluster --project=golden-project --region=us-central1 --format="value(privateClusterConfig.enablePrivateNodes)" [XPK] Private Nodes is not enabled on the cluster. [XPK] Cluster is public and no need to authorize networks. -[XPK] Try 1: get-credentials to cluster golden-cluster -[XPK] Task: `get-credentials to cluster golden-cluster` is implemented by the following command not running since it is a dry run. -gcloud container clusters get-credentials golden-cluster --region=us-central1 --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Try 1: get-credentials-dns-endpoint to cluster golden-cluster +[XPK] Task: `get-credentials-dns-endpoint to cluster golden-cluster` is implemented by the following command not running since it is a dry run. +gcloud container clusters get-credentials golden-cluster --region=us-central1 --dns-endpoint --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Testing credentials with kubectl... +[XPK] Task: `kubectl get pods` is implemented by the following command not running since it is a dry run. +kubectl get pods +[XPK] Credentials test succeeded. +[XPK] Finished get-credentials and kubectl setup. [XPK] Task: 'Checking CoreDNS deployment existence' in progress for namespace: kube-system [XPK] Task: `Check CoreDNS deployment in kube-system` is implemented by the following command not running since it is a dry run. kubectl get deployment coredns -n kube-system @@ -82,7 +87,7 @@ kubectl kueue version kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.12.2/manifests.yaml [XPK] Wait for Kueue to be fully available [XPK] Task: `Wait for Kueue to be available` is implemented by the following command not running since it is a dry run. -kubectl wait deploy/kueue-controller-manager -nkueue-system --for=condition=available --timeout=10m +kubectl wait deploy/kueue-controller-manager -n kueue-system --for=condition=available --timeout=10m [XPK] Install Kueue Custom Resources [XPK] Try 1: Applying Kueue Custom Resources [XPK] Task: `Applying Kueue Custom Resources` is implemented by the following command not running since it is a dry run. diff --git a/goldens/Workload_delete.txt b/goldens/Workload_delete.txt index 683c8cca6..632beab4e 100644 --- a/goldens/Workload_delete.txt +++ b/goldens/Workload_delete.txt @@ -2,9 +2,14 @@ $ python3 xpk.py workload delete --project=golden-project --zone=us-central1-a - [XPK] Starting xpk [XPK] Starting Workload delete [XPK] Working on golden-project and us-central1-a -[XPK] Try 1: get-credentials to cluster golden-cluster -[XPK] Task: `get-credentials to cluster golden-cluster` is implemented by the following command not running since it is a dry run. -gcloud container clusters get-credentials golden-cluster --region=us-central1 --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Try 1: get-credentials-dns-endpoint to cluster golden-cluster +[XPK] Task: `get-credentials-dns-endpoint to cluster golden-cluster` is implemented by the following command not running since it is a dry run. +gcloud container clusters get-credentials golden-cluster --region=us-central1 --dns-endpoint --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Testing credentials with kubectl... +[XPK] Task: `kubectl get pods` is implemented by the following command not running since it is a dry run. +kubectl get pods +[XPK] Credentials test succeeded. +[XPK] Finished get-credentials and kubectl setup. [XPK] Task: `Check if PathwaysJob is installed on golden-cluster` is implemented by the following command not running since it is a dry run. kubectl get pods -n pathways-job-system --no-headers -o custom-columns=NAME:.metadata.name [XPK] check_if_pathways_job_is_installed 0 0 diff --git a/goldens/Workload_list.txt b/goldens/Workload_list.txt index e23d631fb..10abaebd4 100644 --- a/goldens/Workload_list.txt +++ b/goldens/Workload_list.txt @@ -2,9 +2,14 @@ $ python3 xpk.py workload list --project=golden-project --zone=us-central1-a --c [XPK] Starting xpk [XPK] Starting workload list [XPK] Working on golden-project and us-central1-a -[XPK] Try 1: get-credentials to cluster golden-cluster -[XPK] Task: `get-credentials to cluster golden-cluster` is implemented by the following command not running since it is a dry run. -gcloud container clusters get-credentials golden-cluster --region=us-central1 --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Try 1: get-credentials-dns-endpoint to cluster golden-cluster +[XPK] Task: `get-credentials-dns-endpoint to cluster golden-cluster` is implemented by the following command not running since it is a dry run. +gcloud container clusters get-credentials golden-cluster --region=us-central1 --dns-endpoint --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default +[XPK] Testing credentials with kubectl... +[XPK] Task: `kubectl get pods` is implemented by the following command not running since it is a dry run. +kubectl get pods +[XPK] Credentials test succeeded. +[XPK] Finished get-credentials and kubectl setup. [XPK] Task: `List Jobs with filter-by-status=EVERYTHING with filter-by-job=None` is implemented by the following command not running since it is a dry run. kubectl get workloads --ignore-not-found -o=custom-columns="Jobset Name:.metadata.ownerReferences[0].name,Created Time:.metadata.creationTimestamp,Priority:.spec.priorityClassName,TPU VMs Needed:.spec.podSets[0].count,TPU VMs Running/Ran:.status.admission.podSetAssignments[-1].count,TPU VMs Done:.status.reclaimablePods[0].count,Status:.status.conditions[-1].type,Status Message:.status.conditions[-1].message,Status Time:.status.conditions[-1].lastTransitionTime" [XPK] Workload List Output: diff --git a/src/xpk/commands/cluster.py b/src/xpk/commands/cluster.py index eb585d148..bf0416ec9 100644 --- a/src/xpk/commands/cluster.py +++ b/src/xpk/commands/cluster.py @@ -779,7 +779,7 @@ def scale_down_deployment(deployment_name: str, namespace: str = 'kube-system'): if return_code != 0: xpk_print(f'Scale down {deployment_name} error {return_code}') xpk_exit(return_code) - xpk_print(f'\n{deployment_name} has been scaled down.') + xpk_print(f'{deployment_name} has been scaled down.') def scale_up_coredns(replicas: int = 15, namespace: str = 'kube-system'): diff --git a/src/xpk/core/cluster.py b/src/xpk/core/cluster.py index b77a4556f..524100fbd 100644 --- a/src/xpk/core/cluster.py +++ b/src/xpk/core/cluster.py @@ -867,7 +867,53 @@ def update_cluster_with_gcsfuse_driver_if_necessary(args) -> int: return 0 -def get_cluster_credentials(args) -> None: +def test_and_retry_credentials_with_dns_logic(args) -> int: + """Tests kubectl credentials and retries with default settings if a DNS error is found. + + Args: + args: user provided arguments for running the command. + + Returns: + 0 if credentials are valid after retrying, 1 otherwise. + """ + + xpk_print('Testing credentials with kubectl...') + kubectl_command = 'kubectl get pods' + kubectl_return_code, kubectl_output = run_command_for_value( + kubectl_command, 'kubectl get pods' + ) + if kubectl_return_code == 0: + xpk_print('Credentials test succeeded.') + return 0 + + dns_endpoint_error = ( + 'control_plane_endpoints_config.dns_endpoint_config.allow_external_traffic' + ' is disabled' + ) + if dns_endpoint_error not in kubectl_output: + xpk_print(f'kubectl failed with an unhandled error: {kubectl_output}') + xpk_exit(kubectl_return_code) + xpk_print( + 'Detected DNS endpoint-related error. Retrying without --dns-endpoint' + ' flag...' + ) + without_dns_command = ( + 'gcloud container clusters get-credentials' + f' {args.cluster} --region={zone_to_region(args.zone)}' + f' --project={args.project} &&' + ' kubectl config view && kubectl config set-context --current' + ' --namespace=default' + ) + return_code = run_command_with_updates_retry( + without_dns_command, 'get-credentials to cluster', verbose=False + ) + if return_code != 0: + xpk_print('Failed to get credentials even without --dns-endpoint. Exiting.') + xpk_exit(return_code) + return 0 + + +def get_cluster_credentials(args) -> int: """Run cluster configuration command to set the kubectl config. Args: @@ -879,12 +925,19 @@ def get_cluster_credentials(args) -> None: command = ( 'gcloud container clusters get-credentials' f' {args.cluster} --region={zone_to_region(args.zone)}' + ' --dns-endpoint' f' --project={args.project} &&' ' kubectl config view && kubectl config set-context --current' ' --namespace=default' ) - task = f'get-credentials to cluster {args.cluster}' + task = f'get-credentials-dns-endpoint to cluster {args.cluster}' return_code = run_command_with_updates_retry(command, task, verbose=False) + if return_code != 0: xpk_print(f'{task} returned ERROR {return_code}') xpk_exit(return_code) + + return_code = test_and_retry_credentials_with_dns_logic(args) + xpk_print('Finished get-credentials and kubectl setup.') + + return return_code diff --git a/src/xpk/core/kueue.py b/src/xpk/core/kueue.py index 6678f858f..b47dfb888 100644 --- a/src/xpk/core/kueue.py +++ b/src/xpk/core/kueue.py @@ -372,7 +372,7 @@ def wait_for_kueue_available() -> int: 0 if successful and 1 otherwise. """ command = ( - 'kubectl wait deploy/kueue-controller-manager -nkueue-system' + 'kubectl wait deploy/kueue-controller-manager -n kueue-system' f' --for=condition=available --timeout={WAIT_FOR_KUEUE_TIMEOUT}' ) task = 'Wait for Kueue to be available'