diff --git a/.circleci/config.yml b/.circleci/config.yml index 32fa13a69b..820fade5a4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,7 +30,7 @@ steps_prepare_testing_k8s_k3s: &steps_prepare_testing_k8s_k3s name: Install K3D and create the K3D/K3S cluster on Docker command: | curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash - k3d cluster create --servers 1 --no-lb --wait --verbose + k3d cluster create -p "<< pipeline.parameters.reverse-proxy-entrypoint-web-port >>:80@loadbalancer" --servers 1 --wait --verbose - run: name: Load Kurtosis images into the K3S cluster # First load the image into Docker and then import the images into K3S by taking the image from Docker @@ -124,8 +124,7 @@ abort_job_if_only_docs_changes: &abort_job_if_only_docs_changes abort_job_if_kubernetes_backend: &abort_job_if_kubernetes_backend when: condition: - and: - - equal: [ "kubernetes", << parameters.cli-cluster-backend >> ] + equal: [ "kubernetes", << parameters.cli-cluster-backend >> ] steps: - run: circleci-agent step halt @@ -540,8 +539,7 @@ jobs: - when: condition: - and: - - equal: [ "kubernetes", << parameters.cli-cluster-backend >> ] + equal: [ "kubernetes", << parameters.cli-cluster-backend >> ] <<: *steps_prepare_testing_k8s_k3s - when: @@ -614,8 +612,7 @@ jobs: - run: "${KURTOSIS_BINPATH} analytics disable" - when: condition: - and: - - equal: [ "kubernetes", << parameters.cli-cluster-backend >> ] + equal: [ "kubernetes", << parameters.cli-cluster-backend >> ] <<: *steps_prepare_testing_k8s_k3s - when: @@ -767,8 +764,7 @@ jobs: # When backend is 'kubernetes' install kubernetes and start a Kurtosis gateway - when: condition: - and: - - equal: [ "kubernetes", << parameters.cli-cluster-backend >> ] + equal: [ "kubernetes", << parameters.cli-cluster-backend >> ] <<: *steps_prepare_testing_k8s_k3s - when: @@ -940,8 +936,6 @@ jobs: parameters: <<: *param_cli_cluster_backend steps: - - <<: *abort_job_if_kubernetes_backend - - checkout - <<: *abort_job_if_only_docs_changes @@ -961,7 +955,15 @@ jobs: echo 'export KURTOSIS_BINPATH="<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.cli-dist-home-relative-dirpath >>/<< pipeline.parameters.cli-linux-amd-64-binary-relative-filepath >>"' >> "${BASH_ENV}" - run: "${KURTOSIS_BINPATH} analytics disable" - - <<: *run_prepare_testing_docker + - when: + condition: + equal: [ "kubernetes", << parameters.cli-cluster-backend >> ] + <<: *steps_prepare_testing_k8s_k3s + + - when: + condition: + equal: [ "docker", << parameters.cli-cluster-backend >> ] + <<: *steps_prepare_testing_docker # Start a service and send an http request to it via the reverse proxy - run: | @@ -977,18 +979,22 @@ jobs: false fi - # Restart the engine and make sure Traefik restarted and reconfigured properly - - run: | - ${KURTOSIS_BINPATH} engine restart - enclave_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | grep "^UUID:" | awk '{print $2}') - service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') - # Give the reverse proxy enough time to discover the httpd user service - sleep 10 - status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-$(echo $service_uuid)-$(echo $enclave_uuid)" | head -1 | awk '{print $2}') - if ! [ "${status_code}" -eq "200" ]; then - echo 'HTTP request status code returned is '${status_code}' instead of 200' - false - fi + - when: + condition: + equal: [ "docker", << parameters.cli-cluster-backend >> ] + steps: + # Restart the engine and make sure Traefik restarted and reconfigured properly + - run: | + ${KURTOSIS_BINPATH} engine restart + enclave_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | grep "^UUID:" | awk '{print $2}') + service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') + # Give the reverse proxy enough time to discover the httpd user service + sleep 10 + status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-$(echo $service_uuid)-$(echo $enclave_uuid)" | head -1 | awk '{print $2}') + if ! [ "${status_code}" -eq "200" ]; then + echo 'HTTP request status code returned is '${status_code}' instead of 200' + false + fi test_ci_for_failure: executor: ubuntu_vm @@ -1013,8 +1019,7 @@ jobs: # When backend is 'kubernetes' install kubernetes and start a Kurtosis gateway - when: condition: - and: - - equal: [ "kubernetes", << parameters.cli-cluster-backend >> ] + equal: [ "kubernetes", << parameters.cli-cluster-backend >> ] <<: *steps_prepare_testing_k8s_k3s - when: @@ -1472,6 +1477,18 @@ workflows: - build_files_artifacts_expander <<: *filters_ignore_main + - test_reverse_proxy: + name: "Test reverse proxy against Kubernetes" + cli-cluster-backend: "kubernetes" + context: + - docker-user + requires: + - build_cli + - build_api_container_server + - build_engine_server + - build_files_artifacts_expander + <<: *filters_ignore_main + # -- Artifact-publishing jobs -------------------------------- - publish_kurtosis_sdk_rust: context: diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/create_engine.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/create_engine.go index b35ca9280c..babb10f7a0 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/create_engine.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/create_engine.go @@ -307,6 +307,7 @@ func createEngineClusterRole( kubernetes_manager_consts.ServicesKubernetesResource, kubernetes_manager_consts.PersistentVolumesKubernetesResource, kubernetes_manager_consts.PersistentVolumeClaimsKubernetesResource, + kubernetes_manager_consts.IngressesKubernetesResource, kubernetes_manager_consts.JobsKubernetesResource, // Necessary so that we can give the API container the permission }, }, diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend_api_container_functions.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend_api_container_functions.go index e87c01bfe5..77537e2a07 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend_api_container_functions.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend_api_container_functions.go @@ -342,6 +342,7 @@ func (backend *KubernetesKurtosisBackend) CreateAPIContainer( kubernetes_manager_consts.ServicesKubernetesResource, kubernetes_manager_consts.JobsKubernetesResource, kubernetes_manager_consts.PersistentVolumeClaimsKubernetesResource, + kubernetes_manager_consts.IngressesKubernetesResource, }, }, { diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/shared_helpers/shared_helpers.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/shared_helpers/shared_helpers.go index b57acbe5bb..72a70ba9ee 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/shared_helpers/shared_helpers.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/shared_helpers/shared_helpers.go @@ -33,6 +33,7 @@ import ( "github.com/sirupsen/logrus" apiv1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -140,6 +141,9 @@ type UserServiceKubernetesResources struct { // This can be nil if the user hasn't started a pod for the service yet, or if the pod was deleted Pod *apiv1.Pod + + // This can be nil if the user hasn't started the service yet, or if the ingress was deleted + Ingress *netv1.Ingress } func GetEnclaveNamespaceName( @@ -323,6 +327,7 @@ func GetUserServiceKubernetesResourcesMatchingGuids( resultObj = &UserServiceKubernetesResources{ Service: nil, Pod: nil, + Ingress: nil, } } resultObj.Service = kubernetesService @@ -361,6 +366,7 @@ func GetUserServiceKubernetesResourcesMatchingGuids( resultObj = &UserServiceKubernetesResources{ Service: nil, Pod: nil, + Ingress: nil, } } resultObj.Pod = kubernetesPod @@ -368,6 +374,40 @@ func GetUserServiceKubernetesResourcesMatchingGuids( } } + // Get k8s ingresses + matchingKubernetesIngresses, err := kubernetes_resource_collectors.CollectMatchingIngresses( + ctx, + kubernetesManager, + namespaceName, + kubernetesResourceSearchLabels, + kubernetes_label_key.GUIDKubernetesLabelKey.GetString(), + postFilterLabelValues, + ) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting Kubernetes ingresses matching service UUIDs: %+v", serviceUuids) + } + for serviceGuidStr, kubernetesIngressForGuid := range matchingKubernetesIngresses { + logrus.Tracef("Found Kubernetes ingress for GUID '%v': %+v", serviceGuidStr, kubernetesIngressForGuid) + serviceUuid := service.ServiceUUID(serviceGuidStr) + + numIngressesForGuid := len(kubernetesIngressForGuid) + if numIngressesForGuid != 1 { + return nil, stacktrace.NewError("Found %v Kubernetes ingresses associated with service GUID '%v', but number of ingresses should be exactly 1; this is a bug in Kurtosis", numIngressesForGuid, serviceUuid) + } + kubernetesIngress := kubernetesIngressForGuid[0] + + resultObj, found := results[serviceUuid] + if !found { + resultObj = &UserServiceKubernetesResources{ + Service: nil, + Pod: nil, + Ingress: nil, + } + } + resultObj.Ingress = kubernetesIngress + results[serviceUuid] = resultObj + } + return results, nil } diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/destroy_user_services.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/destroy_user_services.go index 4c894dde17..35c3c9c170 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/destroy_user_services.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/destroy_user_services.go @@ -61,6 +61,18 @@ func DestroyUserServices( continue } } + ingressToRemove := resources.Ingress + if ingressToRemove != nil { + if err := kubernetesManager.RemoveIngress(ctx, ingressToRemove); err != nil { + erroredGuids[serviceUuid] = stacktrace.Propagate( + err, + "An error occurred removing Kubernetes ingress '%v' in namespace '%v'", + ingressToRemove.Name, + namespaceName, + ) + continue + } + } } return successfulGuids, erroredGuids, nil } diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/start_user_services.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/start_user_services.go index ebc063e65a..408b982240 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/start_user_services.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions/start_user_services.go @@ -2,6 +2,10 @@ package user_services_functions import ( "context" + "fmt" + "strings" + + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/consts" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/shared_helpers" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider" @@ -18,10 +22,10 @@ import ( "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" apiv1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/intstr" applyconfigurationsv1 "k8s.io/client-go/applyconfigurations/core/v1" - "strings" ) const ( @@ -39,6 +43,12 @@ const ( unboundPortNumber = 1 unlimitedReplacements = -1 + + ingressRulePathAllPaths = "/" +) + +var ( + ingressRulePathTypePrefix = netv1.PathTypePrefix ) // Completeness enforced via unit test @@ -424,6 +434,43 @@ func createStartServiceOperation( } }() + // Create the ingress for the reverse proxy + ingressAttributes, err := enclaveObjAttributesProvider.ForUserServiceIngress(serviceUuid, serviceName, privatePorts) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting attributes for new ingress for service with UUID '%v'", serviceUuid) + } + ingressAnnotationsStrs := shared_helpers.GetStringMapFromAnnotationMap(ingressAttributes.GetAnnotations()) + + ingressRules, err := getUserServiceIngressRules(serviceRegistrationObj, privatePorts) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating the user service ingress rules for service with UUID '%v'", serviceUuid) + } + + shouldDestroyIngress := false + if ingressRules != nil { + ingressName := string(serviceName) + createdIngress, err := kubernetesManager.CreateIngress( + ctx, + namespaceName, + ingressName, + ingressAnnotationsStrs, + ingressRules, + ) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating ingress for service with UUID '%v'", serviceUuid) + } + shouldDestroyIngress = true + defer func() { + if !shouldDestroyIngress { + return + } + if err := kubernetesManager.RemoveIngress(ctx, createdIngress); err != nil { + logrus.Errorf("Starting service didn't complete successfully so we tried to remove the ingress we created but doing so threw an error:\n%v", err) + logrus.Errorf("ACTION REQUIRED: You'll need to remove ingress '%v' in '%v' manually!!!", ingressName, namespaceName) + } + }() + } + updatedService, undoServiceUpdateFunc, err := updateServiceWhenContainerStarted(ctx, namespaceName, kubernetesService, privatePorts, kubernetesManager) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred updating service '%v' to reflect its new ports: %+v", kubernetesService.GetName(), privatePorts) @@ -456,6 +503,7 @@ func createStartServiceOperation( } shouldDestroyPod = false + shouldDestroyIngress = false shouldUndoServiceUpdate = false shouldDestroyPersistentVolumesAndClaims = false return objectsAndResources.Service, nil @@ -858,3 +906,46 @@ func createRegisterUserServiceOperation( return objectsAndResources.ServiceRegistration, nil } } + +func getUserServiceIngressRules( + serviceRegistration *service.ServiceRegistration, + privatePorts map[string]*port_spec.PortSpec, +) ([]netv1.IngressRule, error) { + var ingressRules []netv1.IngressRule + enclaveShortUuid := uuid_generator.ShortenedUUIDString(string(serviceRegistration.GetEnclaveID())) + serviceShortUuid := uuid_generator.ShortenedUUIDString(string(serviceRegistration.GetUUID())) + for _, portSpec := range privatePorts { + maybeApplicationProtocol := "" + if portSpec.GetMaybeApplicationProtocol() != nil { + maybeApplicationProtocol = *portSpec.GetMaybeApplicationProtocol() + } + if maybeApplicationProtocol == consts.HttpApplicationProtocol { + host := fmt.Sprintf("%d-%s-%s", portSpec.GetNumber(), serviceShortUuid, enclaveShortUuid) + ingressRule := netv1.IngressRule{ + Host: host, + IngressRuleValue: netv1.IngressRuleValue{ + HTTP: &netv1.HTTPIngressRuleValue{ + Paths: []netv1.HTTPIngressPath{ + { + Path: ingressRulePathAllPaths, + PathType: &ingressRulePathTypePrefix, + Backend: netv1.IngressBackend{ + Service: &netv1.IngressServiceBackend{ + Name: string(serviceRegistration.GetName()), + Port: netv1.ServiceBackendPort{ + Name: "", + Number: int32(portSpec.GetNumber()), + }, + }, + Resource: nil, + }, + }, + }, + }, + }, + } + ingressRules = append(ingressRules, ingressRule) + } + } + return ingressRules, nil +} diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager/consts/kubernetes_consts.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager/consts/kubernetes_consts.go index 77fcc04ca9..ac5b5a7d8d 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager/consts/kubernetes_consts.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager/consts/kubernetes_consts.go @@ -23,6 +23,7 @@ const ( NodesKubernetesResource = "nodes" PersistentVolumesKubernetesResource = "persistentvolumes" PersistentVolumeClaimsKubernetesResource = "persistentvolumeclaims" + IngressesKubernetesResource = "ingresses" ClusterRoleKubernetesResourceType = "ClusterRole" RoleKubernetesResourceType = "Role" diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager/kubernetes_manager.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager/kubernetes_manager.go index 14c709662b..ae7e51ebb1 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager/kubernetes_manager.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager/kubernetes_manager.go @@ -25,6 +25,7 @@ import ( "github.com/sirupsen/logrus" terminal "golang.org/x/term" apiv1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -292,6 +293,32 @@ func (manager *KubernetesManager) GetServicesByLabels(ctx context.Context, names return &servicesNotMarkedForDeletionserviceList, nil } +func (manager *KubernetesManager) GetIngressesByLabels(ctx context.Context, namespace string, ingressLabels map[string]string) (*netv1.IngressList, error) { + ingressesClient := manager.kubernetesClientSet.NetworkingV1().Ingresses(namespace) + + opts := buildListOptionsFromLabels(ingressLabels) + ingressResult, err := ingressesClient.List(ctx, opts) + if err != nil { + return nil, stacktrace.Propagate(err, "Failed to list ingresses with labels '%+v' in namespace '%s'", ingressLabels, namespace) + } + + // Only return objects not tombstoned by Kubernetes + var ingressesNotMarkedForDeletionList []netv1.Ingress + for _, service := range ingressResult.Items { + deletionTimestamp := service.GetObjectMeta().GetDeletionTimestamp() + if deletionTimestamp == nil { + ingressesNotMarkedForDeletionList = append(ingressesNotMarkedForDeletionList, service) + } + } + ingressesNotMarkedForDeletionserviceList := netv1.IngressList{ + Items: ingressesNotMarkedForDeletionList, + TypeMeta: ingressResult.TypeMeta, + ListMeta: ingressResult.ListMeta, + } + + return &ingressesNotMarkedForDeletionserviceList, nil +} + // ---------------------------Volumes------------------------------------------------------------------------------ func (manager *KubernetesManager) RemovePersistentVolume( @@ -1638,6 +1665,67 @@ func (manager *KubernetesManager) HasComputeNodes(ctx context.Context) (bool, er return len(nodes.Items) != 0, nil } +// ---------------------------Ingresses------------------------------------------------------------------------------ + +func (manager *KubernetesManager) CreateIngress(ctx context.Context, namespace string, name string, annotations map[string]string, rules []netv1.IngressRule) (*netv1.Ingress, error) { + client := manager.kubernetesClientSet.NetworkingV1().Ingresses(namespace) + + ingress := &netv1.Ingress{ + TypeMeta: metav1.TypeMeta{ + Kind: "", + APIVersion: "", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + GenerateName: "", + Namespace: "", + SelfLink: "", + UID: "", + ResourceVersion: "", + Generation: 0, + CreationTimestamp: metav1.Time{ + Time: time.Time{}, + }, + DeletionTimestamp: nil, + DeletionGracePeriodSeconds: nil, + Labels: nil, + Annotations: annotations, + OwnerReferences: nil, + Finalizers: nil, + ManagedFields: nil, + }, + Spec: netv1.IngressSpec{ + IngressClassName: nil, + DefaultBackend: nil, + TLS: nil, + Rules: rules, + }, + Status: netv1.IngressStatus{ + LoadBalancer: netv1.IngressLoadBalancerStatus{ + Ingress: nil, + }, + }, + } + + ingressResult, err := client.Create(ctx, ingress, globalCreateOptions) + if err != nil { + return nil, stacktrace.Propagate(err, "Failed to create the ingress with name '%s' in namespace '%v'", name, namespace) + } + return ingressResult, nil +} + +func (manager *KubernetesManager) RemoveIngress(ctx context.Context, ingress *netv1.Ingress) error { + namespace := ingress.Namespace + ingressName := ingress.Name + ingressesClient := manager.kubernetesClientSet.NetworkingV1().Ingresses(namespace) + + if err := ingressesClient.Delete(ctx, ingressName, globalDeleteOptions); err != nil { + return stacktrace.Propagate(err, "Failed to delete ingress '%s' with delete options '%+v' in namespace '%s'", ingressName, globalDeleteOptions, namespace) + } + + return nil +} + // TODO Delete this after 2022-08-01 if we're not using Jobs /* func (manager *KubernetesManager) CreateJobWithContainerAndVolume(ctx context.Context, diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_resource_collectors/kubernetes_resource_collectors.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_resource_collectors/kubernetes_resource_collectors.go index a0a86e9bb3..8975195078 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_resource_collectors/kubernetes_resource_collectors.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_resource_collectors/kubernetes_resource_collectors.go @@ -5,6 +5,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager" "github.com/kurtosis-tech/stacktrace" apiv1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" ) @@ -115,6 +116,24 @@ func CollectMatchingServices( return postFilterKubernetesResources(getListOfPointersFromListOfElements(objects.Items), postFilterLabelKey, postFilterLabelValues) } +func CollectMatchingIngresses( + ctx context.Context, + kubernetesManager *kubernetes_manager.KubernetesManager, + namespace string, + searchLabels map[string]string, + postFilterLabelKey string, + postFilterLabelValues map[string]bool, +) ( + map[string][]*netv1.Ingress, + error, +) { + objects, err := kubernetesManager.GetIngressesByLabels(ctx, namespace, searchLabels) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting Kubernetes resources matching labels: %+v", searchLabels) + } + return postFilterKubernetesResources(getListOfPointersFromListOfElements(objects.Items), postFilterLabelKey, postFilterLabelValues) +} + func CollectMatchingClusterRoles( ctx context.Context, kubernetesManager *kubernetes_manager.KubernetesManager, diff --git a/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/enclave_object_attributes_provider.go b/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/enclave_object_attributes_provider.go index edd6599ea6..6ac31c84b8 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/enclave_object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/enclave_object_attributes_provider.go @@ -3,6 +3,8 @@ package object_attributes_provider import ( "crypto/md5" "encoding/hex" + "time" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/kubernetes_annotation_key" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/kubernetes_annotation_key_consts" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/kubernetes_annotation_value" @@ -16,7 +18,6 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service_directory" "github.com/kurtosis-tech/stacktrace" - "time" ) const ( @@ -25,6 +26,8 @@ const ( enclaveDataDirFragment = "enclave-data-dir" persistentServiceDirectoryNameFragment = "service-persistent-directory" + + traefikIngressRouterEntrypointsValue = "web" ) type KubernetesEnclaveObjectAttributesProvider interface { @@ -45,6 +48,11 @@ type KubernetesEnclaveObjectAttributesProvider interface { serviceUUID service.ServiceUUID, persistentKey service_directory.DirectoryPersistentKey, ) (KubernetesObjectAttributes, error) + ForUserServiceIngress( + uuid service.ServiceUUID, + id service.ServiceName, + privatePorts map[string]*port_spec.PortSpec, + ) (KubernetesObjectAttributes, error) } // Private so it can't be instantiated @@ -268,6 +276,43 @@ func (provider *kubernetesEnclaveObjectAttributesProviderImpl) ForSinglePersiste return objectAttributes, nil } +func (provider *kubernetesEnclaveObjectAttributesProviderImpl) ForUserServiceIngress( + serviceUUID service.ServiceUUID, + serviceName service.ServiceName, + privatePorts map[string]*port_spec.PortSpec, +) (KubernetesObjectAttributes, error) { + name, err := getKubernetesObjectName(serviceName) + if err != nil { + return nil, stacktrace.Propagate(err, "Failed to get name for user service ingress") + } + + labels, err := provider.getLabelsForEnclaveObjectWithIDAndGUID(string(serviceName), string(serviceUUID)) + if err != nil { + return nil, stacktrace.Propagate( + err, + "Failed to get labels for user service ingress with name '%s' and UUID '%s'", + serviceName, + serviceUUID, + ) + } + labels[kubernetes_label_key.KurtosisResourceTypeKubernetesLabelKey] = label_value_consts.UserServiceKurtosisResourceTypeKubernetesLabelValue + + traefikIngressRouterEntrypointsAnnotationValue, err := kubernetes_annotation_value.CreateNewKubernetesAnnotationValue(traefikIngressRouterEntrypointsValue) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating a new user custom Kubernetes label value '%s'", traefikIngressRouterEntrypointsValue) + } + annotations := map[*kubernetes_annotation_key.KubernetesAnnotationKey]*kubernetes_annotation_value.KubernetesAnnotationValue{ + kubernetes_annotation_key_consts.TraefikIngressRouterEntrypointsAnnotationKey: traefikIngressRouterEntrypointsAnnotationValue, + } + + objectAttributes, err := newKubernetesObjectAttributesImpl(name, labels, annotations) + if err != nil { + return nil, stacktrace.Propagate(err, "Failed to create user service ingress object attributes") + } + + return objectAttributes, nil +} + // ==================================================================================================== // // Private Helper Functions diff --git a/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/kubernetes_annotation_key_consts/kubernetes_annotation_key_consts.go b/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/kubernetes_annotation_key_consts/kubernetes_annotation_key_consts.go index 05323a49a5..87662881b4 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/kubernetes_annotation_key_consts/kubernetes_annotation_key_consts.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/kubernetes_annotation_key_consts/kubernetes_annotation_key_consts.go @@ -15,6 +15,10 @@ const ( enclaveCreationTimeKeyStr = labelKeyPrefixStr + "enclave-creation-time" enclaveNameKeyStr = labelKeyPrefixStr + "enclave-name" + + // Traefik ingress router + traefikKeyIngressRouterPrefixStr = "traefik.ingress.kubernetes.io/router." + traefikKeyEntrypointsStr = traefikKeyIngressRouterPrefixStr + "entrypoints" ) // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DO NOT CHANGE THESE VALUES !!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -23,3 +27,4 @@ const ( var PortSpecsKubernetesAnnotationKey = kubernetes_annotation_key.MustCreateNewKubernetesAnnotationKey(portSpecsAnnotationKeyStr) var EnclaveCreationTimeAnnotationKey = kubernetes_annotation_key.MustCreateNewKubernetesAnnotationKey(enclaveCreationTimeKeyStr) var EnclaveNameAnnotationKey = kubernetes_annotation_key.MustCreateNewKubernetesAnnotationKey(enclaveNameKeyStr) +var TraefikIngressRouterEntrypointsAnnotationKey = kubernetes_annotation_key.MustCreateNewKubernetesAnnotationKey(traefikKeyEntrypointsStr) diff --git a/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/kubernetes_annotation_key_consts/kubernetes_annotation_key_consts_test.go b/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/kubernetes_annotation_key_consts/kubernetes_annotation_key_consts_test.go index c041366f30..9a95aa63ef 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/kubernetes_annotation_key_consts/kubernetes_annotation_key_consts_test.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/kubernetes_annotation_key_consts/kubernetes_annotation_key_consts_test.go @@ -1,9 +1,10 @@ package kubernetes_annotation_key_consts import ( + "testing" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/kubernetes_annotation_key" "github.com/stretchr/testify/require" - "testing" ) var labelKeyStrsToEnsure = map[string]string{ @@ -11,12 +12,14 @@ var labelKeyStrsToEnsure = map[string]string{ portSpecsAnnotationKeyStr: "kurtosistech.com/ports", enclaveCreationTimeKeyStr: "kurtosistech.com/enclave-creation-time", enclaveNameKeyStr: "kurtosistech.com/enclave-name", + traefikKeyEntrypointsStr: "traefik.ingress.kubernetes.io/router.entrypoints", } var labelKeysToEnsure = map[*kubernetes_annotation_key.KubernetesAnnotationKey]string{ - PortSpecsKubernetesAnnotationKey: "kurtosistech.com/ports", - EnclaveCreationTimeAnnotationKey: "kurtosistech.com/enclave-creation-time", - EnclaveNameAnnotationKey: "kurtosistech.com/enclave-name", + PortSpecsKubernetesAnnotationKey: "kurtosistech.com/ports", + EnclaveCreationTimeAnnotationKey: "kurtosistech.com/enclave-creation-time", + EnclaveNameAnnotationKey: "kurtosistech.com/enclave-name", + TraefikIngressRouterEntrypointsAnnotationKey: "traefik.ingress.kubernetes.io/router.entrypoints", } // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IMPORTANT !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! diff --git a/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/label_value_consts/label_value_consts.go b/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/label_value_consts/label_value_consts.go index caae209310..b474d981aa 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/label_value_consts/label_value_consts.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/object_attributes_provider/label_value_consts/label_value_consts.go @@ -15,9 +15,10 @@ const ( engineKurtosisResourceTypeLabelValueStr = "kurtosis-engine" // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DO NOT CHANGE THESE VALUES !!!!!!!!!!!!!!!!!!!!!!!!!!!!! - enclaveKurtosisResourceTypeLabelValueStr = "enclave" - apiContainerKurtosisResourceTypeLabelValueStr = "api-container" - userServiceKurtosisResourceTypeLabelValueStr = "user-service" + enclaveKurtosisResourceTypeLabelValueStr = "enclave" + apiContainerKurtosisResourceTypeLabelValueStr = "api-container" + userServiceKurtosisResourceTypeLabelValueStr = "user-service" + userServiceIngressKurtosisResourceTypeLabelValueStr = "user-service-ingress" enclaveDataVolumeTypeLabelValueStr = "enclave-data" filesArtifactsExpansionVolumeTypeLabelValueStr = "files-artifacts-expansion" @@ -36,6 +37,7 @@ var EngineKurtosisResourceTypeKubernetesLabelValue = kubernetes_label_value.Must var EnclaveKurtosisResourceTypeKubernetesLabelValue = kubernetes_label_value.MustCreateNewKubernetesLabelValue(enclaveKurtosisResourceTypeLabelValueStr) var APIContainerKurtosisResourceTypeKubernetesLabelValue = kubernetes_label_value.MustCreateNewKubernetesLabelValue(apiContainerKurtosisResourceTypeLabelValueStr) var UserServiceKurtosisResourceTypeKubernetesLabelValue = kubernetes_label_value.MustCreateNewKubernetesLabelValue(userServiceKurtosisResourceTypeLabelValueStr) +var UserServiceIngressKurtosisResourceTypeKubernetesLabelValue = kubernetes_label_value.MustCreateNewKubernetesLabelValue(userServiceIngressKurtosisResourceTypeLabelValueStr) var EnclaveDataVolumeTypeKubernetesLabelValue = kubernetes_label_value.MustCreateNewKubernetesLabelValue(enclaveDataVolumeTypeLabelValueStr) var FilesArtifactsExpansionVolumeTypeKubernetesLabelValue = kubernetes_label_value.MustCreateNewKubernetesLabelValue(filesArtifactsExpansionVolumeTypeLabelValueStr)