From 9dfad8eb5f5bc35ea46c4d9c3568224dbb49db33 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 17 Aug 2023 10:39:40 -0400 Subject: [PATCH] add logs storage attr provider --- .../engine_functions/create_engine.go | 16 +++++++++++++--- .../start_user_services.go | 1 - .../label_value_consts/label_value_consts.go | 4 ++-- .../object_attributes_provider.go | 19 +++++++++++++++++++ 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go index 8cacefbdba..9b9649d9f4 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go @@ -25,6 +25,7 @@ const ( frontendPortSpec = 9711 maxWaitForEngineAvailabilityRetries = 10 timeBetweenWaitForEngineAvailabilityRetries = 1 * time.Second + logsStorageDirpath = "/var/log/kurtosis" ) func CreateEngine( @@ -72,10 +73,19 @@ func CreateEngine( targetNetworkId := engineNetwork.GetId() logrus.Infof("Starting the centralized logs components...") + logsStorageObjAttrProvider, err := objAttrsProvider.ForLogsStorageVolume() + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred retrieving logs storage object attributes.") + } + logsStorageVolumeNameStr := logsStorageObjAttrProvider.GetName().GetString() + volumeLabelStrs := map[string]string{} + for labelKey, labelValue := range logsStorageObjAttrProvider.GetLabels() { + volumeLabelStrs[labelKey.GetString()] = labelValue.GetString() + } + // Creation of volume should be idempotent because the volume with persisted logs in it could already exist // Thus, we don't defer an undo volume if this operation fails - // TODO: retrieve name and labels from objsAttrProvider - if err = dockerManager.CreateVolume(ctx, "kurtosis-logs-storage", map[string]string{}); err != nil { + if err = dockerManager.CreateVolume(ctx, logsStorageVolumeNameStr, volumeLabelStrs); err != nil { return nil, stacktrace.Propagate(err, "An error occurred creating logs storage.") } @@ -142,7 +152,7 @@ func CreateEngine( } volumeMounts := map[string]string{ - "kurtosis-logs-storage": "/var/log/kurtosis/", + logsStorageVolumeNameStr: logsStorageDirpath, } if serverArgs.OnBastionHost { diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go index 56515b752d..69bf3795b1 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go @@ -595,7 +595,6 @@ func createStartServiceOperation( return nil, stacktrace.NewError("Expected to have a logs collector server address value to send the user service logs, but it is empty") } - logrus.Debugf("LOG COLLECTOR LABELS: %v", logsCollectorLabels) // The container will be configured to send the logs to the Fluentbit logs collector server fluentdLoggingDriverCnfg := docker_manager.NewFluentdLoggingDriver( logsCollectorAddress, diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go index f1152af699..f44294d616 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go @@ -24,7 +24,7 @@ const ( enclaveDataVolumeTypeLabelValueStr = "enclave-data" filesArtifactExpansionVolumeTypeLabelValueStr = "files-artifacts-expansion" persistentDirectoryVolumeTypeLabelValueStr = "persistent-directory" - logsDatabaseVolumeTypeLabelValueStr = "logs-db" + logsStorageVolumeTypeLabelValueStr = "kurtosis-logs-storage" logsCollectorVolumeTypeLabelValueStr = "logs-collector-data" ) @@ -48,5 +48,5 @@ var FilesArtifactExpanderContainerTypeDockerLabelValue = docker_label_value.Must var EnclaveDataVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(enclaveDataVolumeTypeLabelValueStr) var FilesArtifactExpansionVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(filesArtifactExpansionVolumeTypeLabelValueStr) var PersistentDirectoryVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(persistentDirectoryVolumeTypeLabelValueStr) -var LogsDatabaseVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsDatabaseVolumeTypeLabelValueStr) +var LogsStorageVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsStorageVolumeTypeLabelValueStr) var LogsCollectorVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsCollectorVolumeTypeLabelValueStr) diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go index b350062099..8263316234 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go @@ -17,6 +17,7 @@ import ( const ( engineServerNamePrefix = "kurtosis-engine" logsAggregatorName = "kurtosis-logs-aggregator" + logsStorageVolumeName = "kurtosis-logs-storage" ) type DockerObjectAttributesProvider interface { @@ -27,6 +28,7 @@ type DockerObjectAttributesProvider interface { ) (DockerObjectAttributes, error) ForEnclave(enclaveUuid enclave.EnclaveUUID) (DockerEnclaveObjectAttributesProvider, error) ForLogsAggregator() (DockerObjectAttributes, error) + ForLogsStorageVolume() (DockerObjectAttributes, error) } func GetDockerObjectAttributesProvider() DockerObjectAttributesProvider { @@ -116,3 +118,20 @@ func (provider *dockerObjectAttributesProviderImpl) ForLogsAggregator() (DockerO } return objectAttributes, nil } + +func (provider *dockerObjectAttributesProviderImpl) ForLogsStorageVolume() (DockerObjectAttributes, error) { + name, err := docker_object_name.CreateNewDockerObjectName(logsStorageVolumeName) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating a Docker object name object from string '%v'", logsStorageVolumeName) + } + + labels := map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue{ + label_key_consts.VolumeTypeDockerLabelKey: label_value_consts.LogsStorageVolumeTypeDockerLabelValue, + } + + objectAttributes, err := newDockerObjectAttributesImpl(name, labels) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while creating the ObjectAttributesImpl with the name '%s' and labels '%+v'", name, labels) + } + return objectAttributes, nil +}