Release v0.13.0

.github/CODEOWNERS

@@ -1,2 +1 @@
-* @Obliviour @44past4 @sharabiani @pawloch00 @BluValor @gcie @RoshaniN @scaliby @jamOne- @SikaGrr @FIoannides @fatoshoti
-slice/ @mwysokin @mimowo @gabesaba @PBundyra @mwielgus @pajakd
+* @scaliby @jamOne- @SikaGrr @FIoannides

.github/workflows/build_tests.yaml

@@ -131,6 +131,11 @@ jobs:
     uses: ./.github/workflows/reusable_lint_and_format.yml
     with:
       run-id: '${{needs.set-variables.outputs.run-id}}'
+  verify-goldens:
+    needs: [install-dependencies, set-variables]
+    uses: ./.github/workflows/reusable_goldens.yaml
+    with:
+      run-id: '${{needs.set-variables.outputs.run-id}}'
   run-unit-tests:
     needs: [install-dependencies, set-variables]
     uses: ./.github/workflows/reusable_unit_tests.yaml

.github/workflows/reusable_goldens.yaml

@@ -0,0 +1,48 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+on:
+  workflow_call:
+    inputs:
+      run-id:
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  verify-goldens:
+    runs-on: [ubuntu-22.04]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+    - name: Prepare directories
+      run: mkdir -p ~/.cache/pip
+    - name: Restore cached dependencies
+      uses: actions/cache@v4
+      with:
+        path: |
+          /usr/local/bin/kubectl-kueue
+          /usr/local/bin/kubectl-kjob
+          ~/.cache/pip
+          ${{env.pythonLocation}}
+        key: xpk-deps-3.10-${{github.run_id}}-${{github.run_attempt}}
+        restore-keys: xpk-deps-3.10-
+    - name: Verify goldens
+      run: ./golden_buddy.sh verify goldens.yaml goldens
+      env:
+        UPDATE_GOLDEN_COMMAND: make goldens

.gitignore

@@ -140,4 +140,10 @@ dmypy.json
 **/.DS_Store
 
 # XPK/Cluster Toolkit working directory
-xpkclusters/*
+xpkclusters/*
+
+# gemini-cli settings
+.gemini/
+
+# GitHub App credentials
+gha-creds-*.json

golden_buddy.sh

@@ -90,30 +90,46 @@ fi
 
 mkdir -p "$GOLDENS_DIR"
 
-cat "$GOLDENS_FILE" | yq -r '.goldens | to_entries[] | [.key, .value.command] | @tsv' | \
-  while IFS=$'\t' read -r key command; do
-    if [[ "$MODE" = "update" ]]; then
-      printf "${YELLOW}Updating: %s${NC}\n" "$key"
-    fi
-    if [[ "$MODE" = "verify" ]]; then
-      printf "${YELLOW}Evaluating: %s${NC}\n" "$key"
-    fi
-    eval "$command" > "$GOLDENS_DIR/${key// /_}.txt" 2>&1
-done
+has_diffs=false
+while read -r key; do
+  command=$(yq -r '.goldens["'"$key"'"].command' "$GOLDENS_FILE")
+  if [[ "$MODE" = "update" ]]; then
+    printf "${YELLOW}Updating: %s...${NC} " "$key"
+  fi
+  if [[ "$MODE" = "verify" ]]; then
+    printf "${YELLOW}Evaluating: %s...${NC} " "$key"
+  fi
 
-if [[ "$MODE" = "verify" ]]; then
-  git add "$GOLDENS_DIR"
-  DIFF_OUTPUT=$(git diff HEAD -- "$GOLDENS_DIR" | cat)
+  REFERENCE_FILE="$GOLDENS_DIR/${key// /_}.txt"
+  echo "\$ $command" > $REFERENCE_FILE
+  eval "$command" >> $REFERENCE_FILE 2>&1
+  if [[ "$MODE" = "update" ]]; then
+    printf "${GREEN}DONE${NC}\n"
+  fi
+
+  if [[ "$MODE" = "verify" ]]; then
+    git add $REFERENCE_FILE
+
+    DIFF_OUTPUT=$(git diff --color=always HEAD -- $REFERENCE_FILE | cat)
+
+    git reset HEAD -- $REFERENCE_FILE &> /dev/null
+    git restore $REFERENCE_FILE &> /dev/null
+    git clean -fd -- $REFERENCE_FILE &> /dev/null
 
-  git reset HEAD -- "$GOLDENS_DIR" &> /dev/null
-  git restore "$GOLDENS_DIR" &> /dev/null
-  git clean -fd -- "$GOLDENS_DIR" &> /dev/null
+    if [[ -n "$DIFF_OUTPUT" ]]; then
+      printf "${RED}FAIL${NC}\n"
 
-  if [[ -n "$DIFF_OUTPUT" ]]; then
-    printf "%s\n" "$DIFF_OUTPUT" >&2
+      has_diffs=true
+      echo "\$ $command" $REFERENCE_FILE
+      printf "%s\n" "$DIFF_OUTPUT" >&2
+    else
+      printf "${GREEN}OK${NC}\n"
+    fi
+  fi
+done < <(yq -r '.goldens | keys[]' "$GOLDENS_FILE")
 
-    echo "" >&2
-    
+if [[ "$MODE" = "verify" ]]; then
+  if [[ "$has_diffs" == true ]]; then
     printf "${RED}Golden diffs found! Please use the following command to regenerate goldens:${NC}\n" >&2
     printf "${YELLOW}\n\t%s${NC}\n\n" "${UPDATE_GOLDEN_COMMAND:-"golden_buddy.sh update $GOLDENS_FILE $GOLDENS_DIR"}" >&2
     exit 1
@@ -126,4 +142,4 @@ fi
 if [[ "$MODE" = "update" ]]; then
   printf "${GREEN}Goldens updated!${NC}\n"
   exit 0
-fi
+fi

goldens.yaml

@@ -3,3 +3,29 @@ goldens:
     command: python3 xpk.py cluster create-pathways --project=golden-project --zone=us-central1-a --enable-autoprovisioning --cluster=golden-cluster --tpu-type=tpu7x-8 --on-demand --dry-run
   "NAP cluster-create":
     command: python3 xpk.py cluster create --project=golden-project --zone=us-central1-a --enable-autoprovisioning --cluster=golden-cluster --tpu-type=tpu7x-8 --on-demand --dry-run
+  "Basic cluster create":
+    command: python3 xpk.py cluster create --project=golden-project --zone=us-central1-a --cluster=golden-cluster --tpu-type=tpu7x-8 --spot --dry-run
+  "Cluster create private":
+    command: python3 xpk.py cluster create-pathways --project=golden-project --zone=us-central1-a --cluster=golden-cluster-private --private --tpu-type=v5p-8  --num-slices=1 --default-pool-cpu-machine-type=n1-standard-16 --default-pool-cpu-num-nodes=4 --reservation=golden-reservation --dry-run
+  "Cluster delete":
+    command: python3 xpk.py cluster delete --project=golden-project --zone=us-central1-a --cluster=golden-cluster --dry-run
+  "Cluster delete force":
+    command: python3 xpk.py cluster delete --project=golden-project --zone=us-central1-a --cluster=golden-cluster --force --dry-run
+  "Workload create":
+    command: python3 xpk.py workload create --project=golden-project --zone=us-central1-a --cluster=golden-cluster --workload=golden-workload --command "bash hello" --tpu-type=v5p-8 --num-slices=1 --script-dir=/tmp --dry-run
+  "Workload create pathways":
+    command: python3 xpk.py workload create-pathways --project=golden-project --zone=us-central1-a --cluster=golden-cluster --workload=golden-workload --command "bash hello" --tpu-type=v5p-8 --num-slices=1 --script-dir=/tmp --dry-run
+  "Workload delete":
+    command: python3 xpk.py workload delete --project=golden-project --zone=us-central1-a --cluster=golden-cluster --workload=golden-workload --dry-run
+  "Workload list":
+    command: python3 xpk.py workload list --project=golden-project --zone=us-central1-a --cluster=golden-cluster --dry-run
+  "Storage list":
+    command: python3 xpk.py storage list --project=golden-project --zone=us-central1-a --cluster=golden-cluster --dry-run
+  "Job cancel":
+    command: python3 xpk.py job cancel golden-job --project=golden-project --zone=us-central1-a --cluster=golden-cluster --dry-run
+  "Batch":
+    command: python3 xpk.py batch --project=golden-project --zone=us-central1-a --cluster=golden-cluster --dry-run batch-read.sh
+  "Job list":
+    command: python3 xpk.py job ls --project=golden-project --zone=us-central1-a --cluster=golden-cluster --dry-run
+  "Job info":
+    command: python3 xpk.py job info golden-job --project=golden-project --zone=us-central1-a --cluster=golden-cluster --dry-run

goldens/Basic_cluster_create.txt

@@ -0,0 +1,102 @@
+$ python3 xpk.py cluster create --project=golden-project --zone=us-central1-a --cluster=golden-cluster --tpu-type=tpu7x-8 --spot --dry-run
+[XPK] Starting xpk
+[XPK] Starting cluster create for cluster golden-cluster:
+[XPK] Working on golden-project and us-central1-a
+[XPK] Task: `Determine server supported GKE versions for default rapid gke version` is implemented by the following command not running since it is a dry run. 
+gcloud container get-server-config --project=golden-project --region=us-central1 --flatten="channels" --filter="channels.channel=RAPID" --format="value(channels.defaultVersion)"
+[XPK] Task: `Determine server supported GKE versions for valid versions` is implemented by the following command not running since it is a dry run. 
+gcloud container get-server-config --project=golden-project --region=us-central1 --flatten="channels" --filter="channels.channel=RAPID" --format="value(channels.validVersions)"
+[XPK] Task: `Find if Cluster Exists` is implemented by the following command not running since it is a dry run. 
+gcloud container clusters list --project=golden-project --region=us-central1 --format="csv[no-heading](name)"
+[XPK] Task: `GKE Cluster Create` is implemented by the following command not running since it is a dry run. 
+gcloud beta container clusters create golden-cluster --project=golden-project --region=us-central1 --node-locations=us-central1-a --cluster-version=0 --machine-type=e2-standard-16 --enable-autoscaling --total-min-nodes 1 --total-max-nodes 1000 --num-nodes 6   --enable-dns-access --autoscaling-profile=optimize-utilization --location-policy=BALANCED --scopes=storage-full,gke-default
+[XPK] Task: `Check if Private Nodes is enabled in cluster.` is implemented by the following command not running since it is a dry run. 
+gcloud container clusters describe golden-cluster --project=golden-project --region=us-central1 --format="value(privateClusterConfig.enablePrivateNodes)"
+[XPK] Private Nodes is not enabled on the cluster.
+[XPK] Cluster is public and no need to authorize networks.
+[XPK] Try 1: get-credentials to cluster golden-cluster
+[XPK] Task: `get-credentials to cluster golden-cluster` is implemented by the following command not running since it is a dry run. 
+gcloud container clusters get-credentials golden-cluster --region=us-central1 --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default
+[XPK] Task: 'Checking CoreDNS deployment existence' in progress for namespace: kube-system
+[XPK] Task: `Check CoreDNS deployment in kube-system` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment coredns -n kube-system
+[XPK] Now verifying CoreDNS readiness...
+[XPK] Task: `Waiting for kubeDNS to be checked.` is implemented by the following command not running since it is a dry run. 
+kubectl get deployment kube-dns -n kube-system --ignore-not-found
+[XPK] kube-dns deployment not found.
+[XPK] Verifying if CoreDNS is available...
+[XPK] Task: `Wait for coredns available` is implemented by the following command not running since it is a dry run. 
+kubectl wait deployment/coredns --for=condition=Available=true --namespace=kube-system --timeout=240s
+[XPK] CoreDNS has successfully started and passed verification.
+[XPK] CoreDNS deployment 'coredns' found in namespace 'kube-system'.
+[XPK] Skipping CoreDNS deployment since it already exists.
+[XPK] Task: `Determine current gke master version` is implemented by the following command not running since it is a dry run. 
+gcloud beta container clusters describe golden-cluster --region us-central1 --project golden-project --format="value(currentMasterVersion)"
+[XPK] Creating 1 node pool or pools of tpu7x-8
+We assume that the underlying system is: SystemCharacteristics(topology='2x2x1', vms_per_slice=1, gke_accelerator='tpu7x', gce_machine_type='tpu7x-standard-4t', chips_per_vm=4, accelerator_type=1, device_type='tpu7x-8')
+[XPK] Task: `Get All Node Pools` is implemented by the following command not running since it is a dry run. 
+gcloud beta container node-pools list --cluster golden-cluster --project=golden-project --region=us-central1 --format="csv[no-heading](name)"
+[XPK] Creating 1 node pool or pools of tpu7x-8
+Underlyingly, we assume that means: SystemCharacteristics(topology='2x2x1', vms_per_slice=1, gke_accelerator='tpu7x', gce_machine_type='tpu7x-standard-4t', chips_per_vm=4, accelerator_type=1, device_type='tpu7x-8')
+[XPK] Task: `Get Node Pool Zone` is implemented by the following command not running since it is a dry run. 
+gcloud beta container node-pools describe 0 --cluster golden-cluster --project=golden-project --region=us-central1 --format="value(locations)"
+[XPK] Task: `GKE Cluster Get ConfigMap` is implemented by the following command not running since it is a dry run. 
+kubectl get configmap golden-cluster-resources-configmap -o=custom-columns="ConfigData:data" --no-headers=true
+[XPK] Existing node pool names  ['0']
+[XPK] To complete NodepoolCreate-golden-cluster-np-0 we are executing gcloud beta container node-pools create golden-cluster-np-0 --region=us-central1 --cluster=golden-cluster --project=golden-project --node-locations=us-central1-a --machine-type=tpu7x-standard-4t --host-maintenance-interval=AS_NEEDED --spot --enable-gvnic --node-version=0 --num-nodes=1 --scopes=storage-full,gke-default,"https://www.googleapis.com/auth/cloud-platform" --placement-type=COMPACT  --max-pods-per-node 15 --tpu-topology=2x2x1  
+[XPK] Breaking up a total of 1 commands into 1 batches
+[XPK] Pretending all the jobs succeeded
+[XPK] Create or delete node pool request complete.
+[XPK] Creating ConfigMap for cluster
+[XPK] Breaking up a total of 2 commands into 1 batches
+[XPK] Pretending all the jobs succeeded
+[XPK] Enabling the jobset API on our cluster, to be deprecated when Jobset is globally available
+[XPK] Try 1: Install Jobset on golden-cluster
+[XPK] Task: `Install Jobset on golden-cluster` is implemented by the following command not running since it is a dry run. 
+kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/jobset/releases/download/v0.8.0/manifests.yaml
+[XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
+kubectl get node --no-headers | wc -l
+[XPK] Try 1: Updating jobset Controller Manager resources
+[XPK] Task: `Updating jobset Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+kubectl apply -f 1b31e624e490f9c8c4ef4e369f08d3fa467990af5a261e4405bd045265d70e95
+[XPK] Try 1: Install PathwaysJob on golden-cluster
+[XPK] Task: `Install PathwaysJob on golden-cluster` is implemented by the following command not running since it is a dry run. 
+kubectl apply --server-side -f https://github.com/google/pathways-job/releases/download/v0.1.2/install.yaml
+[XPK] Enabling Kueue on the cluster
+[XPK] Task: `Get kueue version on server` is implemented by the following command not running since it is a dry run. 
+kubectl kueue version
+[XPK] Try 1: Set Kueue On Cluster
+[XPK] Task: `Set Kueue On Cluster` is implemented by the following command not running since it is a dry run. 
+kubectl apply --server-side --force-conflicts -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.12.2/manifests.yaml
+[XPK] Wait for Kueue to be fully available
+[XPK] Task: `Wait for Kueue to be available` is implemented by the following command not running since it is a dry run. 
+kubectl wait deploy/kueue-controller-manager -nkueue-system --for=condition=available --timeout=10m
+[XPK] Install Kueue Custom Resources
+[XPK] Try 1: Applying Kueue Custom Resources
+[XPK] Task: `Applying Kueue Custom Resources` is implemented by the following command not running since it is a dry run. 
+kubectl apply -f c49da377b542c14a80a64a13236f8d3a1c8e022dc7c82cc6f6f0560d980ee9e7
+[XPK] Update Kueue Controller Manager resources
+[XPK] Task: `Count total nodes` is implemented by the following command not running since it is a dry run. 
+kubectl get node --no-headers | wc -l
+[XPK] Try 1: Updating Kueue Controller Manager resources
+[XPK] Task: `Updating Kueue Controller Manager resources` is implemented by the following command not running since it is a dry run. 
+kubectl apply -f 012e1b15b6941e9d47cb2cdb35488d57c2f3ce0ef0b18093d2759f2e02ed81dc
+[XPK] Verifying kjob installation
+[XPK] Task: `Verify kjob installation ` is implemented by the following command not running since it is a dry run. 
+kubectl-kjob help
+[XPK] kjob found
+[XPK] Applying kjob CDRs
+[XPK] Task: `Create kjob CRDs on cluster` is implemented by the following command not running since it is a dry run. 
+kubectl kjob printcrds | kubectl apply --server-side -f -
+[XPK] Creating kjob CRDs succeeded
+[XPK] Task: `GKE Cluster Get ConfigMap` is implemented by the following command not running since it is a dry run. 
+kubectl get configmap golden-cluster-resources-configmap -o=custom-columns="ConfigData:data" --no-headers=true
+[XPK] Task: `Creating JobTemplate` is implemented by the following command not running since it is a dry run. 
+kubectl apply -f 4abb796ed6e7c9d7256a51f13124efd989fc12ee83839bed432fcf7d64f68e61
+[XPK] Task: `Creating PodTemplate` is implemented by the following command not running since it is a dry run. 
+kubectl apply -f a63aa3c4593c38ad90671fd8b067d1886f6313ad558379b364b51791aa50f4e8
+[XPK] Task: `Creating AppProfile` is implemented by the following command not running since it is a dry run. 
+kubectl apply -f 1d13ddebae3c90a05ba26b312df088982dd0df0edc4f4013b88384e476c20486
+[XPK] GKE commands done! Resources are created.
+[XPK] See your GKE Cluster here: https://console.cloud.google.com/kubernetes/clusters/details/us-central1/golden-cluster/details?project=golden-project
+[XPK] Exiting XPK cleanly

goldens/Batch.txt

@@ -0,0 +1,15 @@
+$ python3 xpk.py batch --project=golden-project --zone=us-central1-a --cluster=golden-cluster --dry-run batch-read.sh
+[XPK] Starting xpk
+[XPK] Working on golden-project and us-central1-a
+[XPK] Try 1: get-credentials to cluster golden-cluster
+[XPK] Task: `get-credentials to cluster golden-cluster` is implemented by the following command not running since it is a dry run. 
+gcloud container clusters get-credentials golden-cluster --region=us-central1 --project=golden-project && kubectl config view && kubectl config set-context --current --namespace=default
+[XPK] Task: `GKE Cluster Get ConfigMap` is implemented by the following command not running since it is a dry run. 
+kubectl get configmap golden-cluster-resources-configmap -o=custom-columns="ConfigData:data" --no-headers=true
+[XPK] Task: `GKE Cluster Get ConfigMap` is implemented by the following command not running since it is a dry run. 
+kubectl get configmap golden-cluster-resources-configmap -o=custom-columns="ConfigData:data" --no-headers=true
+[XPK] Task: `GKE Cluster Get ConfigMap` is implemented by the following command not running since it is a dry run. 
+kubectl get configmap golden-cluster-metadata-configmap -o=custom-columns="ConfigData:data" --no-headers=true
+[XPK] Task: `submit job` is implemented by the following command not running since it is a dry run. 
+kubectl kjob create slurm --profile xpk-def-app-profile --localqueue multislice-queue --worker-container xpk-batch-container --first-node-ip --pod-template-annotation kueue.x-k8s.io/podset-preferred-topology=cloud.google.com/gce-topology-host -- batch-read.sh --partition multislice-queue
+[XPK] XPK Done.