feat: User service K8S ingresses for reverse proxy routing (#1941)

## Description:
Similar to how we are adding Traefik labels for user services on Docker,
we are adding a K8S ingress for each user service with a path per HTTP
port so HTTP traffic can be routed to them.

## Is this change user facing?
NO

## References (if applicable):
#1871

.circleci/config.yml

@@ -30,7 +30,7 @@ steps_prepare_testing_k8s_k3s: &steps_prepare_testing_k8s_k3s
         name: Install K3D and create the K3D/K3S cluster on Docker
         command: |
           curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
-          k3d cluster create --servers 1 --no-lb --wait --verbose
+          k3d cluster create -p "<< pipeline.parameters.reverse-proxy-entrypoint-web-port >>:80@loadbalancer" --servers 1 --wait --verbose
     - run:
         name: Load Kurtosis images into the K3S cluster
         # First load the image into Docker and then import the images into K3S by taking the image from Docker
@@ -124,8 +124,7 @@ abort_job_if_only_docs_changes: &abort_job_if_only_docs_changes
 abort_job_if_kubernetes_backend: &abort_job_if_kubernetes_backend
   when:
     condition:
-      and:
-        - equal: [ "kubernetes", << parameters.cli-cluster-backend >> ]
+      equal: [ "kubernetes", << parameters.cli-cluster-backend >> ]
     steps:
       - run: circleci-agent step halt
 
@@ -540,8 +539,7 @@ jobs:
 
       - when:
           condition:
-            and:
-              - equal: [ "kubernetes", << parameters.cli-cluster-backend >> ]
+            equal: [ "kubernetes", << parameters.cli-cluster-backend >> ]
           <<: *steps_prepare_testing_k8s_k3s
 
       - when:
@@ -614,8 +612,7 @@ jobs:
       - run: "${KURTOSIS_BINPATH} analytics disable"
       - when:
           condition:
-            and:
-              - equal: [ "kubernetes", << parameters.cli-cluster-backend >> ]
+            equal: [ "kubernetes", << parameters.cli-cluster-backend >> ]
           <<: *steps_prepare_testing_k8s_k3s
 
       - when:
@@ -767,8 +764,7 @@ jobs:
       # When backend is 'kubernetes' install kubernetes and start a Kurtosis gateway
       - when:
           condition:
-            and:
-              - equal: [ "kubernetes", << parameters.cli-cluster-backend >> ]
+            equal: [ "kubernetes", << parameters.cli-cluster-backend >> ]
           <<: *steps_prepare_testing_k8s_k3s
 
       - when:
@@ -940,8 +936,6 @@ jobs:
     parameters:
       <<: *param_cli_cluster_backend
     steps:
-      - <<: *abort_job_if_kubernetes_backend
-
       - checkout
 
       - <<: *abort_job_if_only_docs_changes
@@ -961,7 +955,15 @@ jobs:
           echo 'export KURTOSIS_BINPATH="<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.cli-dist-home-relative-dirpath >>/<< pipeline.parameters.cli-linux-amd-64-binary-relative-filepath >>"' >> "${BASH_ENV}"
       - run: "${KURTOSIS_BINPATH} analytics disable"
 
-      - <<: *run_prepare_testing_docker
+      - when:
+          condition:
+            equal: [ "kubernetes", << parameters.cli-cluster-backend >> ]
+          <<: *steps_prepare_testing_k8s_k3s
+
+      - when:
+          condition:
+            equal: [ "docker", << parameters.cli-cluster-backend >> ]
+          <<: *steps_prepare_testing_docker
 
       # Start a service and send an http request to it via the reverse proxy
       - run: |
@@ -977,18 +979,22 @@ jobs:
             false
           fi
 
-      # Restart the engine and make sure Traefik restarted and reconfigured properly 
-      - run: |
-          ${KURTOSIS_BINPATH} engine restart
-          enclave_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | grep "^UUID:" | awk '{print $2}')
-          service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}')
-          # Give the reverse proxy enough time to discover the httpd user service
-          sleep 10
-          status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-$(echo $service_uuid)-$(echo $enclave_uuid)" | head -1 | awk '{print $2}')
-          if ! [ "${status_code}" -eq "200" ]; then
-            echo 'HTTP request status code returned is '${status_code}' instead of 200'
-            false
-          fi
+      - when:
+          condition:
+            equal: [ "docker", << parameters.cli-cluster-backend >> ]
+          steps:
+            # Restart the engine and make sure Traefik restarted and reconfigured properly 
+            - run: |
+                ${KURTOSIS_BINPATH} engine restart
+                enclave_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | grep "^UUID:" | awk '{print $2}')
+                service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}')
+                # Give the reverse proxy enough time to discover the httpd user service
+                sleep 10
+                status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-$(echo $service_uuid)-$(echo $enclave_uuid)" | head -1 | awk '{print $2}')
+                if ! [ "${status_code}" -eq "200" ]; then
+                  echo 'HTTP request status code returned is '${status_code}' instead of 200'
+                  false
+                fi
 
   test_ci_for_failure:
     executor: ubuntu_vm
@@ -1013,8 +1019,7 @@ jobs:
       # When backend is 'kubernetes' install kubernetes and start a Kurtosis gateway
       - when:
           condition:
-            and:
-              - equal: [ "kubernetes", << parameters.cli-cluster-backend >> ]
+            equal: [ "kubernetes", << parameters.cli-cluster-backend >> ]
           <<: *steps_prepare_testing_k8s_k3s
 
       - when:
@@ -1472,6 +1477,18 @@ workflows:
             - build_files_artifacts_expander
           <<: *filters_ignore_main
 
+      - test_reverse_proxy:
+          name: "Test reverse proxy against Kubernetes"
+          cli-cluster-backend: "kubernetes"
+          context:
+            - docker-user
+          requires:
+            - build_cli
+            - build_api_container_server
+            - build_engine_server
+            - build_files_artifacts_expander
+          <<: *filters_ignore_main
+
       # -- Artifact-publishing jobs --------------------------------
       - publish_kurtosis_sdk_rust:
           context:

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/create_engine.go

@@ -307,6 +307,7 @@ func createEngineClusterRole(
 				kubernetes_manager_consts.ServicesKubernetesResource,
 				kubernetes_manager_consts.PersistentVolumesKubernetesResource,
 				kubernetes_manager_consts.PersistentVolumeClaimsKubernetesResource,
+				kubernetes_manager_consts.IngressesKubernetesResource,
 				kubernetes_manager_consts.JobsKubernetesResource, // Necessary so that we can give the API container the permission
 			},
 		},

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend_api_container_functions.go

@@ -342,6 +342,7 @@ func (backend *KubernetesKurtosisBackend) CreateAPIContainer(
 				kubernetes_manager_consts.ServicesKubernetesResource,
 				kubernetes_manager_consts.JobsKubernetesResource,
 				kubernetes_manager_consts.PersistentVolumeClaimsKubernetesResource,
+				kubernetes_manager_consts.IngressesKubernetesResource,
 			},
 		},
 		{

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/shared_helpers/shared_helpers.go

@@ -33,6 +33,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 	apiv1 "k8s.io/api/core/v1"
+	netv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
@@ -140,6 +141,9 @@ type UserServiceKubernetesResources struct {
 
 	// This can be nil if the user hasn't started a pod for the service yet, or if the pod was deleted
 	Pod *apiv1.Pod
+
+	// This can be nil if the user hasn't started the service yet, or if the ingress was deleted
+	Ingress *netv1.Ingress
 }
 
 func GetEnclaveNamespaceName(
@@ -323,6 +327,7 @@ func GetUserServiceKubernetesResourcesMatchingGuids(
 			resultObj = &UserServiceKubernetesResources{
 				Service: nil,
 				Pod:     nil,
+				Ingress: nil,
 			}
 		}
 		resultObj.Service = kubernetesService
@@ -361,13 +366,48 @@ func GetUserServiceKubernetesResourcesMatchingGuids(
 				resultObj = &UserServiceKubernetesResources{
 					Service: nil,
 					Pod:     nil,
+					Ingress: nil,
 				}
 			}
 			resultObj.Pod = kubernetesPod
 			results[serviceUuid] = resultObj
 		}
 	}
 
+	// Get k8s ingresses
+	matchingKubernetesIngresses, err := kubernetes_resource_collectors.CollectMatchingIngresses(
+		ctx,
+		kubernetesManager,
+		namespaceName,
+		kubernetesResourceSearchLabels,
+		kubernetes_label_key.GUIDKubernetesLabelKey.GetString(),
+		postFilterLabelValues,
+	)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred getting Kubernetes ingresses matching service UUIDs: %+v", serviceUuids)
+	}
+	for serviceGuidStr, kubernetesIngressForGuid := range matchingKubernetesIngresses {
+		logrus.Tracef("Found Kubernetes ingress for GUID '%v': %+v", serviceGuidStr, kubernetesIngressForGuid)
+		serviceUuid := service.ServiceUUID(serviceGuidStr)
+
+		numIngressesForGuid := len(kubernetesIngressForGuid)
+		if numIngressesForGuid != 1 {
+			return nil, stacktrace.NewError("Found %v Kubernetes ingresses associated with service GUID '%v', but number of ingresses should be exactly 1; this is a bug in Kurtosis", numIngressesForGuid, serviceUuid)
+		}
+		kubernetesIngress := kubernetesIngressForGuid[0]
+
+		resultObj, found := results[serviceUuid]
+		if !found {
+			resultObj = &UserServiceKubernetesResources{
+				Service: nil,
+				Pod:     nil,
+				Ingress: nil,
+			}
+		}
+		resultObj.Ingress = kubernetesIngress
+		results[serviceUuid] = resultObj
+	}
+
 	return results, nil
 }
 

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/destroy_user_services.go

@@ -61,6 +61,18 @@ func DestroyUserServices(
 				continue
 			}
 		}
+		ingressToRemove := resources.Ingress
+		if ingressToRemove != nil {
+			if err := kubernetesManager.RemoveIngress(ctx, ingressToRemove); err != nil {
+				erroredGuids[serviceUuid] = stacktrace.Propagate(
+					err,
+					"An error occurred removing Kubernetes ingress '%v' in namespace '%v'",
+					ingressToRemove.Name,
+					namespaceName,
+				)
+				continue
+			}
+		}
 	}
 	return successfulGuids, erroredGuids, nil
 }

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/start_user_services.go

@@ -2,6 +2,10 @@ package user_services_functions
 
 import (
 	"context"
+	"fmt"
+	"strings"
+
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/consts"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/shared_helpers"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider"
@@ -18,10 +22,10 @@ import (
 	"github.com/kurtosis-tech/stacktrace"
 	"github.com/sirupsen/logrus"
 	apiv1 "k8s.io/api/core/v1"
+	netv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	applyconfigurationsv1 "k8s.io/client-go/applyconfigurations/core/v1"
-	"strings"
 )
 
 const (
@@ -39,6 +43,12 @@ const (
 	unboundPortNumber = 1
 
 	unlimitedReplacements = -1
+
+	ingressRulePathAllPaths = "/"
+)
+
+var (
+	ingressRulePathTypePrefix = netv1.PathTypePrefix
 )
 
 // Completeness enforced via unit test
@@ -424,6 +434,43 @@ func createStartServiceOperation(
 			}
 		}()
 
+		// Create the ingress for the reverse proxy
+		ingressAttributes, err := enclaveObjAttributesProvider.ForUserServiceIngress(serviceUuid, serviceName, privatePorts)
+		if err != nil {
+			return nil, stacktrace.Propagate(err, "An error occurred getting attributes for new ingress for service with UUID '%v'", serviceUuid)
+		}
+		ingressAnnotationsStrs := shared_helpers.GetStringMapFromAnnotationMap(ingressAttributes.GetAnnotations())
+
+		ingressRules, err := getUserServiceIngressRules(serviceRegistrationObj, privatePorts)
+		if err != nil {
+			return nil, stacktrace.Propagate(err, "An error occurred creating the user service ingress rules for service with UUID '%v'", serviceUuid)
+		}
+
+		shouldDestroyIngress := false
+		if ingressRules != nil {
+			ingressName := string(serviceName)
+			createdIngress, err := kubernetesManager.CreateIngress(
+				ctx,
+				namespaceName,
+				ingressName,
+				ingressAnnotationsStrs,
+				ingressRules,
+			)
+			if err != nil {
+				return nil, stacktrace.Propagate(err, "An error occurred creating ingress for service with UUID '%v'", serviceUuid)
+			}
+			shouldDestroyIngress = true
+			defer func() {
+				if !shouldDestroyIngress {
+					return
+				}
+				if err := kubernetesManager.RemoveIngress(ctx, createdIngress); err != nil {
+					logrus.Errorf("Starting service didn't complete successfully so we tried to remove the ingress we created but doing so threw an error:\n%v", err)
+					logrus.Errorf("ACTION REQUIRED: You'll need to remove ingress '%v' in '%v' manually!!!", ingressName, namespaceName)
+				}
+			}()
+		}
+
 		updatedService, undoServiceUpdateFunc, err := updateServiceWhenContainerStarted(ctx, namespaceName, kubernetesService, privatePorts, kubernetesManager)
 		if err != nil {
 			return nil, stacktrace.Propagate(err, "An error occurred updating service '%v' to reflect its new ports: %+v", kubernetesService.GetName(), privatePorts)
@@ -456,6 +503,7 @@ func createStartServiceOperation(
 		}
 
 		shouldDestroyPod = false
+		shouldDestroyIngress = false
 		shouldUndoServiceUpdate = false
 		shouldDestroyPersistentVolumesAndClaims = false
 		return objectsAndResources.Service, nil
@@ -858,3 +906,46 @@ func createRegisterUserServiceOperation(
 		return objectsAndResources.ServiceRegistration, nil
 	}
 }
+
+func getUserServiceIngressRules(
+	serviceRegistration *service.ServiceRegistration,
+	privatePorts map[string]*port_spec.PortSpec,
+) ([]netv1.IngressRule, error) {
+	var ingressRules []netv1.IngressRule
+	enclaveShortUuid := uuid_generator.ShortenedUUIDString(string(serviceRegistration.GetEnclaveID()))
+	serviceShortUuid := uuid_generator.ShortenedUUIDString(string(serviceRegistration.GetUUID()))
+	for _, portSpec := range privatePorts {
+		maybeApplicationProtocol := ""
+		if portSpec.GetMaybeApplicationProtocol() != nil {
+			maybeApplicationProtocol = *portSpec.GetMaybeApplicationProtocol()
+		}
+		if maybeApplicationProtocol == consts.HttpApplicationProtocol {
+			host := fmt.Sprintf("%d-%s-%s", portSpec.GetNumber(), serviceShortUuid, enclaveShortUuid)
+			ingressRule := netv1.IngressRule{
+				Host: host,
+				IngressRuleValue: netv1.IngressRuleValue{
+					HTTP: &netv1.HTTPIngressRuleValue{
+						Paths: []netv1.HTTPIngressPath{
+							{
+								Path:     ingressRulePathAllPaths,
+								PathType: &ingressRulePathTypePrefix,
+								Backend: netv1.IngressBackend{
+									Service: &netv1.IngressServiceBackend{
+										Name: string(serviceRegistration.GetName()),
+										Port: netv1.ServiceBackendPort{
+											Name:   "",
+											Number: int32(portSpec.GetNumber()),
+										},
+									},
+									Resource: nil,
+								},
+							},
+						},
+					},
+				},
+			}
+			ingressRules = append(ingressRules, ingressRule)
+		}
+	}
+	return ingressRules, nil
+}

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager/consts/kubernetes_consts.go

@@ -23,6 +23,7 @@ const (
 	NodesKubernetesResource                  = "nodes"
 	PersistentVolumesKubernetesResource      = "persistentvolumes"
 	PersistentVolumeClaimsKubernetesResource = "persistentvolumeclaims"
+	IngressesKubernetesResource              = "ingresses"
 
 	ClusterRoleKubernetesResourceType = "ClusterRole"
 	RoleKubernetesResourceType        = "Role"