feat: make kurtosis service logs pull from persistent volume

.circleci/config.yml

@@ -669,9 +669,9 @@ jobs:
               false
             fi
       - run:
-          name: "Verify Kurtosis cleaned up all its volumes"
+          name: "Verify Kurtosis cleaned up all its volumes (except for the log storage volume, so we can persist service logs)"
           command: |
-            if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l ) -eq 0 ]; then
+            if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l ) -eq 1 ]; then
               docker volume ls
               false
             fi
@@ -828,9 +828,9 @@ jobs:
                     false
                   fi
             - run:
-                name: "Verify Kurtosis cleaned up all its volumes"
+                name: "Verify Kurtosis cleaned up all its volumes (except for the log storage volume, so we can persist service logs)"
                 command: |
-                  if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l) -eq 0 ]; then
+                  if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l) -eq 1 ]; then
                     docker volume ls
                     false
                   fi

cli/cli/command_framework/highlevel/enclave_id_arg/enclave_id_arg.go

@@ -40,8 +40,7 @@ func NewEnclaveIdentifierArg(
 	}
 }
 
-// TODO we added this constructor for allowing 'service logs' command to disable the validation for consuming logs from removed or stopped enclaves
-// TODO after https://github.com/kurtosis-tech/kurtosis/issues/879
+// This constructor is for allowing 'service logs' command to disable the validation for consuming logs from removed or stopped enclaves
 func NewHistoricalEnclaveIdentifiersArgWithValidationDisabled(
 	argKey string,
 	isOptional bool,

cli/cli/command_framework/highlevel/service_identifier_arg/service_identifier_arg.go

@@ -36,8 +36,7 @@ func NewServiceIdentifierArg(
 	}
 }
 
-// TODO we added this constructor for allowing 'service logs' command to disable the validation for consuming logs from removed or stopped enclaves
-// TODO after https://github.com/kurtosis-tech/kurtosis/issues/879 is done
+// This constructor is for allowing 'service logs' command to disable the validation for consuming logs from removed or stopped enclaves
 func NewHistoricalServiceIdentifierArgWithValidationDisabled(
 	serviceIdentifierArgKey string,
 	enclaveIdentifierArgKey string,

cli/cli/commands/service/logs/logs.go

@@ -103,15 +103,12 @@ var ServiceLogsCmd = &engine_consuming_kurtosis_command.EngineConsumingKurtosisC
 		},
 	},
 	Args: []*args.ArgConfig{
-		//TODO disabling enclaveID validation and serviceUUID validation for allowing consuming logs from removed or stopped enclaves
-		//TODO we should enable them when #879 is ready: https://github.com/kurtosis-tech/kurtosis/issues/879
-		enclave_id_arg.NewEnclaveIdentifierArg(
+		enclave_id_arg.NewHistoricalEnclaveIdentifiersArgWithValidationDisabled(
 			enclaveIdentifierArgKey,
-			engineClientCtxKey,
 			isEnclaveIdArgOptional,
 			isEnclaveIdArgGreedy,
 		),
-		service_identifier_arg.NewServiceIdentifierArg(
+		service_identifier_arg.NewHistoricalServiceIdentifierArgWithValidationDisabled(
 			serviceIdentifierArgKey,
 			enclaveIdentifierArgKey,
 			isServiceIdentifierArgGreedy,

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"github.com/docker/go-connections/nat"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts"
-	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types"
@@ -148,15 +147,7 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer(
 		labelStrs[labelKey.GetString()] = labelValue.GetString()
 	}
 
-	//The APIContainer will be configured to send the logs to the Fluentbit logs collector server
-	logCollectorAddr, err := enclaveLogsCollector.GetEnclaveNetworkAddressString()
-	if err != nil {
-		return nil, stacktrace.Propagate(err, "An error occurred retrieving the log collector address.")
-	}
-	fluentdLoggingDriverCnfg := docker_manager.NewFluentdLoggingDriver(
-		logCollectorAddr,
-		logs_collector_functions.GetKurtosisTrackedLogsCollectorLabels(),
-	)
+	// TODO: configure the APIContainer to send the logs to the Fluentbit logs collector server
 
 	createAndStartArgs := docker_manager.NewCreateAndStartContainerArgsBuilder(
 		image,
@@ -174,8 +165,6 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer(
 		ipAddr,
 	).WithLabels(
 		labelStrs,
-	).WithLoggingDriver(
-		fluentdLoggingDriverCnfg,
 	).Build()
 
 	if err = backend.dockerManager.FetchImage(ctx, image); err != nil {

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go

@@ -25,6 +25,7 @@ const (
 	frontendPortSpec                            = 9711
 	maxWaitForEngineAvailabilityRetries         = 10
 	timeBetweenWaitForEngineAvailabilityRetries = 1 * time.Second
+	logsStorageDirpath                          = "/var/log/kurtosis/"
 )
 
 func CreateEngine(
@@ -72,6 +73,22 @@ func CreateEngine(
 	targetNetworkId := engineNetwork.GetId()
 
 	logrus.Infof("Starting the centralized logs components...")
+	logsStorageAttrs, err := objAttrsProvider.ForLogsStorageVolume()
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred retrieving logs storage object attributes.")
+	}
+	logsStorageVolNameStr := logsStorageAttrs.GetName().GetString()
+	volumeLabelStrs := map[string]string{}
+	for labelKey, labelValue := range logsStorageAttrs.GetLabels() {
+		volumeLabelStrs[labelKey.GetString()] = labelValue.GetString()
+	}
+
+	// Creation of volume should be idempotent because the volume with persisted logs in it could already exist
+	// Thus, we don't defer an undo volume if this operation fails
+	if err = dockerManager.CreateVolume(ctx, logsStorageVolNameStr, volumeLabelStrs); err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred creating logs storage.")
+	}
+
 	logsAggregatorContainer := vector.NewVectorLogsAggregatorContainer() // Declaring implementation
 	_, removeLogsAggregatorFunc, err := logs_aggregator_functions.CreateLogsAggregator(
 		ctx,
@@ -134,6 +151,10 @@ func CreateEngine(
 		consts.DockerSocketFilepath: consts.DockerSocketFilepath,
 	}
 
+	volumeMounts := map[string]string{
+		logsStorageVolNameStr: logsStorageDirpath,
+	}
+
 	if serverArgs.OnBastionHost {
 		// Mount the host engine config directory so the engine can access files like the remote backend config.
 		bindMounts[consts.HostEngineConfigDirToMount] = consts.EngineConfigLocalDir
@@ -158,6 +179,8 @@ func CreateEngine(
 		envVars,
 	).WithBindMounts(
 		bindMounts,
+	).WithVolumeMounts(
+		volumeMounts,
 	).WithUsedPorts(
 		usedPorts,
 	).WithLabels(

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go

@@ -1,42 +1,47 @@
 package vector
 
 const (
-	configDirpath                = "/etc/vector/"
-	healthCheckEndpoint          = "health"
-	defaultGraphQlApiHttpPortNum = uint16(8686)
-	httpProtocolStr              = "http"
+	configDirpath = "/etc/vector/"
 
 	////////////////////////--VECTOR CONTAINER CONFIGURATION SECTION--/////////////////////////////
 	containerImage = "timberio/vector:0.31.0-debian"
 
 	configFilepath = configDirpath + "vector.toml"
 	binaryFilepath = "/usr/bin/vector"
 	configFileFlag = "-c"
+
+	logsStorageDirpath = "/var/log/kurtosis/"
 	////////////////////////--FINISH VECTOR CONTAINER CONFIGURATION SECTION--/////////////////////////////
 
 	////////////////////////--VECTOR CONFIGURATION SECTION--/////////////////////////////
 	fluentBitSourceId        = "\"fluent_bit\""
 	fluentBitSourceType      = "\"fluent\""
 	fluentBitSourceIpAddress = "0.0.0.0"
 
-	// TODO: change output when persistent volume is implemented
-	stdoutSinkID = "\"stdout\""
-	stdoutTypeId = "\"console\""
+	fileSinkId = "\"file\""
+	fileTypeId = "\"file\""
+
+	// We store log files in the volume per-enclave, per-service
+	// To construct the filepath, we utilize vectors template syntax that allows us to reference fields in log events
+	// https://vector.dev/docs/reference/configuration/template-syntax/
+	logsFilepath = "\"" + logsStorageDirpath + "{{ enclave_uuid }}/{{ service_uuid }}.json\""
 
 	configFileTemplateName = "vectorConfigFileTemplate"
-	configFileTemplate     = `
-[api]
-enabled = true
-address = "0.0.0.0:8686"
 
+	// Note: we set buffer to block so that we don't drop any logs, however this could apply backpressure up the topology
+	// if we start noticing slowdown due to vector buffer blocking, we might want to revisit our architecture
+	configFileTemplate = `
 [sources.{{ .Source.Id }}]
 type = {{ .Source.Type }}
 address = "{{ .Source.Address }}"
 
 [sinks.{{ .Sink.Id }}]
 type = {{ .Sink.Type }}
 inputs = {{ .Sink.Inputs }}
+path = {{ .Sink.Filepath }}
 encoding.codec = "json"
+buffer.when_full = "block"
 `
+
 	////////////////////////--FINISH--VECTOR CONFIGURATION SECTION--/////////////////////////////
 )

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go

@@ -17,9 +17,10 @@ type Source struct {
 }
 
 type Sink struct {
-	Id     string
-	Type   string
-	Inputs []string
+	Id       string
+	Type     string
+	Inputs   []string
+	Filepath string
 }
 
 func newDefaultVectorConfig(listeningPortNumber uint16) *VectorConfig {
@@ -30,9 +31,10 @@ func newDefaultVectorConfig(listeningPortNumber uint16) *VectorConfig {
 			Address: fmt.Sprintf("%s:%s", fluentBitSourceIpAddress, strconv.Itoa(int(listeningPortNumber))),
 		},
 		Sink: &Sink{
-			Id:     stdoutSinkID,
-			Type:   stdoutTypeId,
-			Inputs: []string{fluentBitSourceId},
+			Id:       fileSinkId,
+			Type:     fileTypeId,
+			Inputs:   []string{fluentBitSourceId},
+			Filepath: logsFilepath,
 		},
 	}
 }

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go

@@ -26,7 +26,13 @@ func (vector *vectorContainerConfigProvider) GetContainerArgs(
 	containerName string,
 	containerLabels map[string]string,
 	networkId string,
+	logsStorageVolumeName string,
 ) (*docker_manager.CreateAndStartContainerArgs, error) {
+
+	volumeMounts := map[string]string{
+		logsStorageVolumeName: logsStorageDirpath,
+	}
+
 	logsAggregatorConfigContentStr, err := vector.getConfigFileContent()
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "An error occurred getting the Loki server's configuration content")
@@ -54,6 +60,8 @@ func (vector *vectorContainerConfigProvider) GetContainerArgs(
 		networkId,
 	).WithLabels(
 		containerLabels,
+	).WithVolumeMounts(
+		volumeMounts,
 	).WithEntrypointArgs(
 		[]string{
 			shBinaryFilepath,

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_logs_aggregator_container.go

@@ -32,7 +32,15 @@ func (vectorContainer *vectorLogsAggregatorContainer) CreateAndStart(
 	for labelKey, labelValue := range logsAggregatorAttrs.GetLabels() {
 		containerLabelStrs[labelKey.GetString()] = labelValue.GetString()
 	}
-	createAndStartArgs, err := vectorContainerConfigProviderObj.GetContainerArgs(containerName, containerLabelStrs, targetNetworkId)
+
+	// Engine handles creating the volume, but we need to mount the aggregator can send logs to logs storage
+	logsStorageAttrs, err := objAttrsProvider.ForLogsStorageVolume()
+	if err != nil {
+		return "", nil, nil, stacktrace.Propagate(err, "An error occurred getting the logs storage volume attributes.")
+	}
+	logsStorageVolNameStr := logsStorageAttrs.GetName().GetString()
+
+	createAndStartArgs, err := vectorContainerConfigProviderObj.GetContainerArgs(containerName, containerLabelStrs, targetNetworkId, logsStorageVolNameStr)
 	if err != nil {
 		return "", nil, nil, err
 	}

container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key/docker_label_key.go

@@ -6,7 +6,7 @@ import (
 )
 
 const (
-	dockerLabelKeyRegexStr = "^[a-z0-9-.]+$"
+	dockerLabelKeyRegexStr = "^[a-z0-9-._]+$"
 
 	// It doesn't seem Docker actually has a label key length limit, but we implement one of our own for practicality
 	maxLabelLength = 256

container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go

@@ -8,9 +8,9 @@ import (
 // The following docker labels will be added into the logs stream
 // These are necessary for propagating information for log filtering and retrieval through the logging pipeline
 var LogsDatabaseKurtosisTrackedDockerLabelsForIdentifyLogsStream = []*docker_label_key.DockerLabelKey{
-	label_key_consts.GUIDDockerLabelKey,
 	label_key_consts.ContainerTypeDockerLabelKey,
-	label_key_consts.EnclaveUUIDDockerLabelKey,
+	label_key_consts.LogsEnclaveUUIDDockerLabelKey,
+	label_key_consts.LogsServiceUUIDDockerLabelKey,
 }
 
 // These are all the logs database Kurtosis tracked Docker Labels used

container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go

@@ -467,7 +467,8 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getNameForUserService
 
 func (provider *dockerEnclaveObjectAttributesProviderImpl) getLabelsForEnclaveObject() map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue {
 	return map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue{
-		label_key_consts.EnclaveUUIDDockerLabelKey: provider.enclaveId,
+		label_key_consts.EnclaveUUIDDockerLabelKey:     provider.enclaveId,
+		label_key_consts.LogsEnclaveUUIDDockerLabelKey: provider.enclaveId,
 	}
 }
 
@@ -478,6 +479,7 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getLabelsForEnclaveOb
 		return nil, stacktrace.Propagate(err, "An error occurred creating a Docker label value from GUID string '%v'", guid)
 	}
 	labels[label_key_consts.GUIDDockerLabelKey] = guidLabelValue
+	labels[label_key_consts.LogsServiceUUIDDockerLabelKey] = guidLabelValue
 	return labels, nil
 }
 

container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go

@@ -29,12 +29,19 @@ const (
 	portSpecsLabelKeyStr = labelNamespaceStr + "ports"
 
 	enclaveIdLabelKeyStr = labelNamespaceStr + "enclave-id"
+
 	// TODO deprecate this in favor of storing in DB
 	enclaveNameLabelKeyStr = labelNamespaceStr + "enclave-name"
 
 	enclaveCreationTime = labelNamespaceStr + "enclave-creation-time"
 
 	privateIpAddrLabelKeyStr = labelNamespaceStr + "private-ip"
+
+	// We create a duplicate of the enclave uuid and service uuid label key because:
+	// the logs aggregator (vector) needs the enclave uuid and service uuid label keys to create the filepath where logs are stored in persistent volume
+	// but vectors template syntax can't interpret the "com.kurtosistech." prefix, so we can't use the existing label keys
+	logsEnclaveUuidLabelKeyStr    = "enclave_uuid"
+	logsServiceUuidDockerLabelKey = "service_uuid"
 )
 
 // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DO NOT CHANGE THESE VALUES !!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -57,3 +64,5 @@ var EnclaveNameDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(enc
 var EnclaveCreationTimeLabelKey = docker_label_key.MustCreateNewDockerLabelKey(enclaveCreationTime)
 var PrivateIPDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(privateIpAddrLabelKeyStr)
 var UserServiceGUIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(userServiceGuidDockerLabelKeyStr)
+var LogsEnclaveUUIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(logsEnclaveUuidLabelKeyStr)
+var LogsServiceUUIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(logsServiceUuidDockerLabelKey)

container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go

@@ -24,7 +24,7 @@ const (
 	enclaveDataVolumeTypeLabelValueStr            = "enclave-data"
 	filesArtifactExpansionVolumeTypeLabelValueStr = "files-artifacts-expansion"
 	persistentDirectoryVolumeTypeLabelValueStr    = "persistent-directory"
-	logsDatabaseVolumeTypeLabelValueStr           = "logs-db"
+	logsStorageVolumeTypeLabelValueStr            = "kurtosis-logs-storage"
 	logsCollectorVolumeTypeLabelValueStr          = "logs-collector-data"
 )
 
@@ -48,5 +48,5 @@ var FilesArtifactExpanderContainerTypeDockerLabelValue = docker_label_value.Must
 var EnclaveDataVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(enclaveDataVolumeTypeLabelValueStr)
 var FilesArtifactExpansionVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(filesArtifactExpansionVolumeTypeLabelValueStr)
 var PersistentDirectoryVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(persistentDirectoryVolumeTypeLabelValueStr)
-var LogsDatabaseVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsDatabaseVolumeTypeLabelValueStr)
+var LogsStorageVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsStorageVolumeTypeLabelValueStr)
 var LogsCollectorVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsCollectorVolumeTypeLabelValueStr)