Enable fail fast CI feature to reduce the End-to-End testing consumption

.prow.yaml

@@ -135,7 +135,7 @@ presubmits:
             - "-c"
             - |
               set -eE; cd "$(git rev-parse --show-toplevel)/e2e/terraform"; trap 'terraform destroy -target module.gcp-ubuntu-focal -auto-approve' EXIT;
-              terraform init && timeout 75m terraform apply -target module.gcp-ubuntu-focal -var="pkg_version=v1.0.1" -auto-approve
+              terraform init && timeout 120m terraform apply -target module.gcp-ubuntu-focal -var="fail_fast=true" -auto-approve
           volumeMounts:
             - name: satoken
               mountPath: "/etc/satoken"
@@ -186,7 +186,7 @@ presubmits:
             - "-c"
             - |
               set -eE; cd "$(git rev-parse --show-toplevel)/e2e/terraform"; trap 'terraform destroy -target module.gcp-fedora-34 -auto-approve' EXIT;
-              terraform init && timeout 75m terraform apply -target module.gcp-fedora-34 -var="pkg_version=v1.0.1" -auto-approve
+              terraform init && timeout 120m terraform apply -target module.gcp-fedora-34 -var="fail_fast=true" -auto-approve
           volumeMounts:
             - name: satoken
               mountPath: "/etc/satoken"
@@ -236,7 +236,7 @@ presubmits:
             - "-c"
             - |
               set -eE; cd "$(git rev-parse --show-toplevel)/e2e/terraform"; trap 'terraform destroy -target module.gcp-ubuntu-jammy -auto-approve' EXIT;
-              terraform init && timeout 75m terraform apply -target module.gcp-ubuntu-jammy -var="pkg_version=v1.0.1" -var="e2e_type=oai" -auto-approve
+              terraform init && timeout 120m terraform apply -target module.gcp-ubuntu-jammy -var="e2e_type=oai" -var="fail_fast=true" -auto-approve
           volumeMounts:
             - name: satoken
               mountPath: "/etc/satoken"

e2e/lib/kpt.sh

@@ -41,9 +41,8 @@ function _wait_for_user_repo {
     if [[ $found != "True" ]]; then
         curl_gitea_api "users/$user/repos" 'import json; import sys; print("\n".join(repo["full_name"] for repo in json.loads(sys.stdin.read())))'
         error "Timed out waiting for $repo repository"
-    else
-        info "Found $user/$repo repository"
     fi
+    info "Found $user/$repo repository"
 }
 
 # kpt_wait_pkg() - Wait for a given kpt package to exist in a given repository
@@ -69,8 +68,6 @@ function kpt_wait_pkg {
     if [[ $found != "True" ]]; then
         curl_gitea_api "repos/$user/$repo/contents" 'import json; import sys; print("\n".join(dir["path"] for dir in json.loads(sys.stdin.read()) if dir["type"] == "dir" ))'
         error "Timed out waiting for $pkg kpt package"
-    else
-        info "Found $user/$repo repository"
     fi
-
+    info "Found $pkg kpt package in $user/$repo repository"
 }

e2e/lib/porch.sh

@@ -31,7 +31,37 @@ function porch_wait_log_entry {
     if [[ -z $found ]]; then
         kubectl logs -n porch-system "$(kubectl get pods -n porch-system -l app=porch-server -o jsonpath='{.items[*].metadata.name}')"
         error "Timed out waiting for $pattern"
-    else
-        info "Found $pattern log entry in porch server"
     fi
+    info "Found $pattern log entry in porch server"
+}
+
+# porch_wait_published_packagerev() - Waits for a kpt package revision gets published
+function porch_wait_published_packagerev {
+    local pkg_name="$1"
+    local repository="$2"
+    local revision="${3:-main}"
+    local timeout=${4:-900}
+
+    info "looking for package published revision on $pkg_name"
+    local found=""
+    while [[ $timeout -gt 0 ]]; do
+        for pkg_rev in $(kubectl get packagerevisions -o jsonpath="{range .items[?(@.spec.packageName==\"$pkg_name\")]}{.metadata.name}{\"\\n\"}{end}"); do
+            if [ "$(kubectl get packagerevision "$pkg_rev" -o jsonpath='{.spec.repository}/{.spec.revision}')" == "$repository/$revision" ]; then
+                found=$pkg_rev
+                break
+            fi
+        done
+        if [[ $found ]]; then
+            debug "timeout: $timeout"
+            break
+        fi
+        timeout=$((timeout - 5))
+        sleep 5
+    done
+
+    if [[ -z $found ]]; then
+        kubectl get packagerevisions -o jsonpath="{range .items[?(@.spec.packageName==\"$pkg_name\")]}{.metadata.name}{\"\\n\\\"}{end}"
+        error "Timed out waiting for revisions on $pkg_name package"
+    fi
+    kubectl wait --for jsonpath='{.spec.lifecycle}'=Published packagerevisions $found --timeout=10m
 }

e2e/provision/Vagrantfile

@@ -61,6 +61,7 @@ Vagrant.configure('2') do |config|
       NEPHIO_REPO_DIR: '/opt/test-infra',
       ANSIBLE_CMD_EXTRA_VAR_LIST: "k8s.context='kind-kind' nephio_catalog_repo_uri='#{ENV['NEPHIO_CATALOG_REPO_URI'] || 'https://github.com/nephio-project/nephio-catalog-packages.git'}'",
       E2EDIR: '/opt/test-infra/e2e',
+      FAIL_FAST: true,
       E2ETYPE: ENV.fetch('E2ETYPE', 'free5gc')
     }
     sh.inline = <<-SHELL

e2e/provision/init.sh

@@ -64,6 +64,7 @@ HOME=${NEPHIO_HOME:-/home/$NEPHIO_USER}
 REPO_DIR=${NEPHIO_REPO_DIR:-$HOME/test-infra}
 DOCKERHUB_USERNAME=${DOCKERHUB_USERNAME:-""}
 DOCKERHUB_TOKEN=${DOCKERHUB_TOKEN:-""}
+FAIL_FAST=${FAIL_FAST:-$(get_metadata fail_fast "false")}
 
 echo "$DEBUG, $DEPLOYMENT_TYPE, $RUN_E2E, $REPO, $BRANCH, $NEPHIO_USER, $HOME, $REPO_DIR, $DOCKERHUB_USERNAME, $DOCKERHUB_TOKEN"
 trap get_status ERR
@@ -72,6 +73,8 @@ if ! command -v git >/dev/null; then
     source /etc/os-release || source /usr/lib/os-release
     case ${ID,,} in
     ubuntu | debian)
+        # Removed the damaged list
+        rm -rvf /var/lib/apt/lists/*
         apt-get update
         apt-get install -y git
         ;;
@@ -109,7 +112,7 @@ sed -e "s/vagrant/$NEPHIO_USER/" <"$REPO_DIR/e2e/provision/nephio.yaml" >"$HOME/
 # Sandbox Creation
 int_start=$(date +%s)
 cd "$REPO_DIR/e2e/provision"
-export DEBUG DEPLOYMENT_TYPE DOCKERHUB_USERNAME DOCKERHUB_TOKEN
+export DEBUG DEPLOYMENT_TYPE DOCKERHUB_USERNAME DOCKERHUB_TOKEN FAIL_FAST
 runuser -u "$NEPHIO_USER" ./install_sandbox.sh
 printf "%s secs\n" "$(($(date +%s) - int_start))"
 

e2e/provision/install_sandbox.sh

@@ -19,6 +19,8 @@ export HOME=${HOME:-/home/ubuntu/}
 source /etc/os-release || source /usr/lib/os-release
 case ${ID,,} in
 ubuntu | debian)
+    # Removed the damaged list
+    sudo rm -vrf /var/lib/apt/lists/*
     sudo apt-get update
     sudo -E DEBIAN_FRONTEND=noninteractive apt-get remove -q -y python3-openssl
     sudo -E NEEDRESTART_SUSPEND=1 DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confnew" --allow-downgrades --allow-remove-essential --allow-change-held-packages -fuy install -q -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" python3-pip

e2e/terraform/main.tf

@@ -1,32 +1,32 @@
 module "gcp-ubuntu-focal" {
-  source             = ".//modules/gcp"
-  nephio_pkg_version = var.pkg_version
-  nephio_e2e_type    = var.e2e_type
+  source               = ".//modules/gcp"
+  nephio_e2e_type      = var.e2e_type
+  nephio_e2e_fail_fast = var.fail_fast
 }
 
 module "gcp-ubuntu-jammy" {
-  source             = ".//modules/gcp"
-  vmimage            = "ubuntu-os-cloud/ubuntu-2204-lts"
-  nephio_pkg_version = var.pkg_version
-  nephio_e2e_type    = var.e2e_type
+  source               = ".//modules/gcp"
+  vmimage              = "ubuntu-os-cloud/ubuntu-2204-lts"
+  nephio_e2e_type      = var.e2e_type
+  nephio_e2e_fail_fast = var.fail_fast
 }
 
 module "gcp-fedora-34" {
-  source             = ".//modules/gcp"
-  vmimage            = "fedora-cloud/fedora-cloud-34"
-  ansible_user       = "fedora"
-  nephio_pkg_version = var.pkg_version
-  nephio_e2e_type    = var.e2e_type
-}
-
-variable "pkg_version" {
-  description = "The version used for all the nephio packages"
-  default     = "main"
-  type        = string
+  source               = ".//modules/gcp"
+  vmimage              = "fedora-cloud/fedora-cloud-34"
+  ansible_user         = "fedora"
+  nephio_e2e_type      = var.e2e_type
+  nephio_e2e_fail_fast = var.fail_fast
 }
 
 variable "e2e_type" {
   description = "The End-to-End testing type"
   default     = "free5gc"
   type        = string
 }
+
+variable "fail_fast" {
+  description = "Defines the behavior after failing a testing"
+  default     = "false"
+  type        = string
+}

e2e/terraform/modules/gcp/main.tf

@@ -112,7 +112,7 @@ resource "google_compute_instance" "e2e_instances" {
     inline = [
       "cd /home/${var.ansible_user}/test-infra/e2e/provision/",
       "chmod +x init.sh",
-      "sudo -E E2ETYPE=${var.nephio_e2e_type} NEPHIO_PKG_VERSION=${var.nephio_pkg_version} NEPHIO_REPO_DIR=/home/${var.ansible_user}/test-infra NEPHIO_DEBUG=true NEPHIO_RUN_E2E=true NEPHIO_USER=${var.ansible_user} ./init.sh"
+      "sudo -E FAIL_FAST=${var.nephio_e2e_fail_fast} E2ETYPE=${var.nephio_e2e_type} NEPHIO_REPO_DIR=/home/${var.ansible_user}/test-infra NEPHIO_DEBUG=true NEPHIO_RUN_E2E=true NEPHIO_USER=${var.ansible_user} ./init.sh"
     ]
   }
 }

e2e/terraform/modules/gcp/variables.tf

@@ -64,14 +64,14 @@ variable "nephio_e2e_nodes" {
   type        = number
 }
 
-variable "nephio_pkg_version" {
-  description = "The version used for all the nephio packages"
-  default     = "main"
-  type        = string
-}
-
 variable "nephio_e2e_type" {
   description = "The Nephio End-to-End testing type"
   default     = "free5gc"
   type        = string
 }
+
+variable "nephio_e2e_fail_fast" {
+  description = "The Nephio End-to-End testing failing behavior"
+  default     = "false"
+  type        = string
+}

e2e/tests/free5gc/004.sh

@@ -24,8 +24,18 @@ source "$E2EDIR/defaults.env"
 # shellcheck source=e2e/lib/k8s.sh
 source "${LIBDIR}/k8s.sh"
 
+# shellcheck source=e2e/lib/kpt.sh
+source "${LIBDIR}/kpt.sh"
+
+# shellcheck source=e2e/lib/porch.sh
+source "${LIBDIR}/porch.sh"
+
 k8s_apply "$TESTDIR/004-free5gc-operator.yaml"
 
-for cluster in "regional" "edge01" "edge02"; do
+kubeconfig="$HOME/.kube/config"
+for cluster in $(kubectl get cl -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' --kubeconfig "$kubeconfig"); do
+    k8s_wait_exists "packagevariant" "free5gc-operator-$cluster-free5gc-operator"
+    porch_wait_log_entry "refs/heads/proposed/free5gc-operator/packagevariant-1 :refs/heads/drafts/free5gc-operator/packagevariant-1"
+    kpt_wait_pkg "$cluster" "free5gc-operator"
     k8s_wait_ready_replicas "deployment" "free5gc-operator" "$(k8s_get_capi_kubeconfig "$cluster")" "free5gc"
 done

e2e/tests/free5gc/005.sh

@@ -24,12 +24,18 @@ source "$E2EDIR/defaults.env"
 # shellcheck source=e2e/lib/k8s.sh
 source "${LIBDIR}/k8s.sh"
 
-k8s_apply "$TESTDIR/005-edge-free5gc-upf.yaml"
+# shellcheck source=e2e/lib/kpt.sh
+source "${LIBDIR}/kpt.sh"
 
-for cluster in "edge01" "edge02"; do
-    k8s_wait_ready "packagevariant" "edge-free5gc-upf-${cluster}-free5gc-upf"
-done
+# shellcheck source=e2e/lib/porch.sh
+source "${LIBDIR}/porch.sh"
+
+k8s_apply "$TESTDIR/005-edge-free5gc-upf.yaml"
 
 for cluster in "edge01" "edge02"; do
+    k8s_wait_exists "packagevariant" "edge-free5gc-upf-${cluster}-free5gc-upf"
+    porch_wait_log_entry 'updated to "free5gc-upf"'
+    porch_wait_published_packagerev "free5gc-upf" "$cluster"
+    kpt_wait_pkg "$cluster" "free5gc-upf"
     k8s_wait_ready_replicas "deployment" "upf-${cluster}" "$(k8s_get_capi_kubeconfig "$cluster")" "free5gc-upf"
 done

e2e/tests/free5gc/006.sh

@@ -24,16 +24,23 @@ source "$E2EDIR/defaults.env"
 # shellcheck source=e2e/lib/k8s.sh
 source "${LIBDIR}/k8s.sh"
 
+# shellcheck source=e2e/lib/kpt.sh
+source "${LIBDIR}/kpt.sh"
+
+# shellcheck source=e2e/lib/porch.sh
+source "${LIBDIR}/porch.sh"
+
 # apply both AMF and SMF so they both start processing
 k8s_apply "$TESTDIR/006-regional-free5gc-amf.yaml"
 k8s_apply "$TESTDIR/006-regional-free5gc-smf.yaml"
 
 cluster_kubeconfig=$(k8s_get_capi_kubeconfig "regional")
 
-# check the AMF
-k8s_wait_ready "packagevariant" "regional-free5gc-amf-regional-free5gc-amf"
-k8s_wait_ready_replicas "deployment" "amf-regional" "$cluster_kubeconfig" "free5gc-cp"
-
-# check the SMF
-k8s_wait_ready "packagevariant" "regional-free5gc-smf-regional-free5gc-smf"
-k8s_wait_ready_replicas "deployment" "smf-regional" "$cluster_kubeconfig" "free5gc-cp"
+# check the NFs
+for nf in amf smf; do
+    k8s_wait_exists "packagevariant" "regional-free5gc-$nf-regional-free5gc-$nf"
+    porch_wait_log_entry "\"$nf-example\" updated to \"free5gc-cp\""
+    porch_wait_published_packagerev "free5gc-$nf" "regional"
+    kpt_wait_pkg "regional" "free5gc-$nf"
+    k8s_wait_ready_replicas "deployment" "$nf-regional" "$cluster_kubeconfig" "free5gc-cp"
+done

e2e/tests/free5gc/007.sh

@@ -27,27 +27,29 @@ source "${LIBDIR}/_utils.sh"
 # shellcheck source=e2e/lib/k8s.sh
 source "${LIBDIR}/k8s.sh"
 
+# shellcheck source=e2e/lib/porch.sh
+source "${LIBDIR}/porch.sh"
+
 function _wait_for_uesimtun0 {
     kubeconfig="$1"
     pod_name="$2"
 
     info "waiting for tunnel to be established"
     timeout=600
     found=""
-    while [[ -z $found && $timeout -gt 0 ]]; do
+    while [[ $timeout -gt 0 ]]; do
         ip_a=$(k8s_exec "$kubeconfig" "ueransim" "$pod_name" "ip address show")
         if [[ $ip_a == *"uesimtun0"* ]]; then
+            debug "timeout: $timeout"
             found="yes"
+            break
         fi
         timeout=$((timeout - 5))
-        if [[ -z $found && $timeout -gt 0 ]]; then
-            sleep 5
-        fi
+        sleep 5
     done
-    debug "timeout: $timeout"
+    debug "$(kubectl logs -n ueransim --kubeconfig "$kubeconfig" "$pod_name")"
 
     if [[ -z $found ]]; then
-        kubectl logs -n ueransim --kubeconfig "$kubeconfig" "$pod_name"
         k8s_exec "$kubeconfig" "ueransim" "$pod_name" "ip address show"
         for worker in $(sudo docker ps --filter "name=edge01-md*" --format "{{.Names}}"); do
             sudo docker exec "$worker" dmesg -l warn,err
@@ -65,11 +67,14 @@ port=$(kubectl --kubeconfig "$regional_kubeconfig" -n free5gc-cp get svc webui-s
 debug "port: $port"
 
 # Register a subscriber with free5gc
-curl -d "@${TESTDIR}/007-subscriber.json" -H 'Token: admin' -H 'Content-Type: application/json' "http://${ip}:${port}/api/subscriber/imsi-208930000000003/20893"
+curl -v -d "@${TESTDIR}/007-subscriber.json" -H 'Token: admin' -H 'Content-Type: application/json' "http://${ip}:${port}/api/subscriber/imsi-208930000000003/20893"
+# List existing subscribers
+curl -s -X GET -H 'Token: admin' "http://${ip}:${port}/api/subscriber"
 
 # Deploy UERANSIM to edge01
 k8s_apply "$TESTDIR/007-edge01-ueransim.yaml"
 k8s_wait_ready "packagevariant" "edge01-ueransim"
+porch_wait_published_packagerev "ueransim" "edge01"
 edge01_kubeconfig=$(k8s_get_capi_kubeconfig "edge01")
 k8s_wait_ready_replicas "deployment" "ueransimgnb-edge01" "$edge01_kubeconfig" "ueransim"
 k8s_wait_ready_replicas "deployment" "ueransimue-edge01" "$edge01_kubeconfig" "ueransim"

e2e/tests/oai/004-ran-network.sh

@@ -36,16 +36,19 @@ function _wait_for_ran {
     timeout=600
 
     temp_file=$(mktemp)
-    kubectl logs "$(kubectl get pods -n oai-ran-cucp --kubeconfig "$kubeconfig" -l app.kubernetes.io/name=oai-gnb-cu-cp -o jsonpath='{.items[*].metadata.name}')" -n oai-ran-cucp -c gnbcucp --kubeconfig "$kubeconfig" > temp_file
-    while grep -q "$wait_msg" temp_file;status=$?; [[ $status != 0 ]]
-    	do
-        	if [[ $timeout -lt 0 ]]; then
-            		kubectl logs -l app.kubernetes.io/name=oai-gnb-cu-cp -n oai-ran-cucp -c gnbcucp --kubeconfig "$kubeconfig" --tail 50
-            		error "Timed out waiting for $link_name link to be established"
-        	fi
-        	timeout=$((timeout - 5))
-        	sleep 5
-		kubectl logs "$(kubectl get pods -n oai-ran-cucp --kubeconfig "$kubeconfig" -l app.kubernetes.io/name=oai-gnb-cu-cp -o jsonpath='{.items[*].metadata.name}')" -n oai-ran-cucp -c gnbcucp --kubeconfig "$kubeconfig" > temp_file
+    kubectl logs "$(kubectl get pods -n oai-ran-cucp --kubeconfig "$kubeconfig" -l app.kubernetes.io/name=oai-gnb-cu-cp -o jsonpath='{.items[*].metadata.name}')" -n oai-ran-cucp -c gnbcucp --kubeconfig "$kubeconfig" >temp_file
+    while
+        grep -q "$wait_msg" temp_file
+        status=$?
+        [[ $status != 0 ]]
+    do
+        if [[ $timeout -lt 0 ]]; then
+            kubectl logs -l app.kubernetes.io/name=oai-gnb-cu-cp -n oai-ran-cucp -c gnbcucp --kubeconfig "$kubeconfig" --tail 50
+            error "Timed out waiting for $link_name link to be established"
+        fi
+        timeout=$((timeout - 5))
+        sleep 5
+        kubectl logs "$(kubectl get pods -n oai-ran-cucp --kubeconfig "$kubeconfig" -l app.kubernetes.io/name=oai-gnb-cu-cp -o jsonpath='{.items[*].metadata.name}')" -n oai-ran-cucp -c gnbcucp --kubeconfig "$kubeconfig" >temp_file
     done
     debug "timeout: $timeout"
     rm "${temp_file}"

e2e/tests/oai/005-ue.sh

@@ -35,16 +35,19 @@ function _wait_for_ue {
     info "waiting for $msg to be finished"
     timeout=600
     temp_file=$(mktemp)
-    kubectl logs "$(kubectl get pods -n oai-ue --kubeconfig "$kubeconfig" -l app.kubernetes.io/name=oai-nr-ue -o jsonpath='{.items[*].metadata.name}')" -n oai-ue -c nr-ue --kubeconfig "$kubeconfig" > temp_file
-    while grep -q "$log_msg" temp_file;status=$?; [ $status != 0 ]
-    	do
-        	if [[ $timeout -lt 0 ]]; then
-            		kubectl logs -l app.kubernetes.io/name=oai-nr-ue -n oai-ue -c nr-ue --kubeconfig "$kubeconfig" --tail 50
-            		error "Timed out waiting for $msg"
-        	fi
-        	timeout=$((timeout - 5))
-        	sleep 5
-		kubectl logs "$(kubectl get pods -n oai-ue --kubeconfig "$kubeconfig" -l app.kubernetes.io/name=oai-nr-ue -o jsonpath='{.items[*].metadata.name}')" -n oai-ue -c nr-ue --kubeconfig "$kubeconfig" > temp_file
+    kubectl logs "$(kubectl get pods -n oai-ue --kubeconfig "$kubeconfig" -l app.kubernetes.io/name=oai-nr-ue -o jsonpath='{.items[*].metadata.name}')" -n oai-ue -c nr-ue --kubeconfig "$kubeconfig" >temp_file
+    while
+        grep -q "$log_msg" temp_file
+        status=$?
+        [ $status != 0 ]
+    do
+        if [[ $timeout -lt 0 ]]; then
+            kubectl logs -l app.kubernetes.io/name=oai-nr-ue -n oai-ue -c nr-ue --kubeconfig "$kubeconfig" --tail 50
+            error "Timed out waiting for $msg"
+        fi
+        timeout=$((timeout - 5))
+        sleep 5
+        kubectl logs "$(kubectl get pods -n oai-ue --kubeconfig "$kubeconfig" -l app.kubernetes.io/name=oai-nr-ue -o jsonpath='{.items[*].metadata.name}')" -n oai-ue -c nr-ue --kubeconfig "$kubeconfig" >temp_file
     done
     debug "timeout: $timeout"
     rm "${temp_file}"