From e4ba5ea3456fbf2df27fac5fdf6744d86ccc177f Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Wed, 30 Aug 2023 20:35:19 -0400 Subject: [PATCH 1/3] add service name and short uuid to logs label stream --- .../docker_labels_for_logs/docker_labels_for_logs.go | 2 ++ .../enclave_object_attributes_provider.go | 7 +++++++ .../label_key_consts/label_key_consts.go | 8 ++++++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go index dd0537cf7a..030f3dc59d 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go @@ -11,6 +11,8 @@ var LogsDatabaseKurtosisTrackedDockerLabelsForIdentifyLogsStream = []*docker_lab label_key_consts.ContainerTypeDockerLabelKey, label_key_consts.LogsEnclaveUUIDDockerLabelKey, label_key_consts.LogsServiceUUIDDockerLabelKey, + label_key_consts.LogsServiceShortUUIDDockerLabelKey, + label_key_consts.LogsServiceNameDockerLabelKey, } // These are all the logs database Kurtosis tracked Docker Labels used diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go index 8c76477b23..6dcacbc8e8 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go @@ -477,8 +477,14 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getLabelsForEnclaveOb if err != nil { return nil, stacktrace.Propagate(err, "An error occurred creating a Docker label value from GUID string '%v'", guid) } + shortGuidStr := uuid_generator.ShortenedUUIDString(guid) + shortGuidLabelValue, err := docker_label_value.CreateNewDockerLabelValue(shortGuidStr) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating a short GUID Docker label value from GUID string '%v'", guid) + } labels[label_key_consts.GUIDDockerLabelKey] = guidLabelValue labels[label_key_consts.LogsServiceUUIDDockerLabelKey] = guidLabelValue + labels[label_key_consts.LogsServiceShortUUIDDockerLabelKey] = shortGuidLabelValue return labels, nil } @@ -492,6 +498,7 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getLabelsForEnclaveOb return nil, stacktrace.Propagate(err, "An error occurred creating a Docker label value from ID string '%v'", id) } labels[label_key_consts.IDDockerLabelKey] = idLabelValue + labels[label_key_consts.LogsServiceNameDockerLabelKey] = idLabelValue return labels, nil } diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go index 0027d1e82f..c5a6cf4a5d 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go @@ -41,8 +41,10 @@ const ( // We create a duplicate of the enclave uuid and service uuid label key because: // the logs aggregator (vector) needs the enclave uuid and service uuid label keys to create the filepath where logs are stored in persistent volume // but vectors template syntax can't interpret the "com.kurtosistech." prefix, so we can't use the existing label keys - logsEnclaveUuidLabelKeyStr = "enclave_uuid" - logsServiceUuidDockerLabelKey = "service_uuid" + logsEnclaveUuidLabelKeyStr = "enclave_uuid" + logsServiceUuidDockerLabelKey = "service_uuid" + logsServiceShortUuidDockerLabelKey = "service_short_uuid" + logsServiceNameDockerLabelKey = "service_name" ) // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DO NOT CHANGE THESE VALUES !!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -67,3 +69,5 @@ var PrivateIPDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(priva var UserServiceGUIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(userServiceGuidDockerLabelKeyStr) var LogsEnclaveUUIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(logsEnclaveUuidLabelKeyStr) var LogsServiceUUIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(logsServiceUuidDockerLabelKey) +var LogsServiceShortUUIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(logsServiceShortUuidDockerLabelKey) +var LogsServiceNameDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(logsServiceNameDockerLabelKey) From be0e7e7185462fc3b20516d1de2fcca4e580c4b4 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 31 Aug 2023 10:28:10 -0400 Subject: [PATCH 2/3] modify vector config --- .../implementations/vector/consts.go | 32 +++++++---- .../implementations/vector/vector_config.go | 55 +++++++++++++++++-- .../vector_container_config_provider.go | 21 +------ 3 files changed, 70 insertions(+), 38 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go index 6340d469a7..d4624136cf 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go @@ -24,24 +24,32 @@ const ( // We store log files in the volume per-enclave, per-service // To construct the filepath, we utilize vectors template syntax that allows us to reference fields in log events // https://vector.dev/docs/reference/configuration/template-syntax/ - logsFilepath = "\"" + logsStorageDirpath + "{{ enclave_uuid }}/{{ service_uuid }}.json\"" + uuidLogsFilepath = "\"" + logsStorageDirpath + "{{ enclave_uuid }}/{{ service_uuid }}.json\"" - configFileTemplateName = "vectorConfigFileTemplate" + // Right now, we store duplicate logs files by service name and service's shortened uuid + // This is to enable retrieving logs by name and short uuid after enclaves have been stopped + // As right now, when an enclave is stopped all identifier info is lost + // TODO: Find a better way to capture or persist identifier info without storing duplicate log files + nameLogsFilepath = "\"" + logsStorageDirpath + "{{ enclave_uuid }}/{{ service_name }}.json\"" + shortUUIDLogsFilepath = "\"" + logsStorageDirpath + "{{ enclave_uuid }}/{{ service_short_uuid }}.json\"" + + sourceConfigFileTemplateName = "srcVectorConfigFileTemplate" + sinkConfigFileTemplateName = "sinkVectorConfigFileTemplate" // Note: we set buffer to block so that we don't drop any logs, however this could apply backpressure up the topology // if we start noticing slowdown due to vector buffer blocking, we might want to revisit our architecture - configFileTemplate = ` -[sources.{{ .Source.Id }}] -type = {{ .Source.Type }} -address = "{{ .Source.Address }}" - -[sinks.{{ .Sink.Id }}] -type = {{ .Sink.Type }} -inputs = {{ .Sink.Inputs }} -path = {{ .Sink.Filepath }} + srcConfigFileTemplate = ` +[sources.{{ .Id }}] +type = {{ .Type }} +address = "{{ .Address }}" +` + sinkConfigFileTemplate = ` +[sinks.{{ .Id }}] +type = {{ .Type }} +inputs = {{ .Inputs }} +path = {{ .Filepath }} encoding.codec = "json" buffer.when_full = "block" ` - ////////////////////////--FINISH--VECTOR CONFIGURATION SECTION--///////////////////////////// ) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go index cee5f5efed..7e769b0630 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go @@ -1,13 +1,16 @@ package vector import ( + "bytes" "fmt" + "github.com/kurtosis-tech/stacktrace" "strconv" + "text/template" ) type VectorConfig struct { Source *Source - Sink *Sink + Sinks []*Sink } type Source struct { @@ -30,11 +33,51 @@ func newDefaultVectorConfig(listeningPortNumber uint16) *VectorConfig { Type: fluentBitSourceType, Address: fmt.Sprintf("%s:%s", fluentBitSourceIpAddress, strconv.Itoa(int(listeningPortNumber))), }, - Sink: &Sink{ - Id: fileSinkId, - Type: fileTypeId, - Inputs: []string{fluentBitSourceId}, - Filepath: logsFilepath, + Sinks: []*Sink{ + { + Id: fileSinkId, + Type: fileTypeId, + Inputs: []string{fluentBitSourceId}, + Filepath: uuidLogsFilepath, + }, + { + Id: fileSinkId, + Type: fileTypeId, + Inputs: []string{fluentBitSourceId}, + Filepath: nameLogsFilepath, + }, + { + Id: fileSinkId, + Type: fileTypeId, + Inputs: []string{fluentBitSourceId}, + Filepath: shortUUIDLogsFilepath, + }, }, } } + +func (cfg *VectorConfig) getConfigFileContent() (string, error) { + srcCfgFileTemplate, err := template.New(sourceConfigFileTemplateName).Parse(srcConfigFileTemplate) + if err != nil { + return "", stacktrace.Propagate(err, "An error occurred parsing Vector's source config template.") + } + sinkCfgFileTemplate, err := template.New(sinkConfigFileTemplateName).Parse(sinkConfigFileTemplate) + if err != nil { + return "", stacktrace.Propagate(err, "An error occurred parsing Vector's sink config template.") + } + + templateStrBuffer := &bytes.Buffer{} + + if err := srcCfgFileTemplate.Execute(templateStrBuffer, cfg.Source); err != nil { + return "", stacktrace.Propagate(err, "An error occurred executing Vector's source config file template.") + } + for _, sink := range cfg.Sinks { + if err := sinkCfgFileTemplate.Execute(templateStrBuffer, sink); err != nil { + return "", stacktrace.Propagate(err, "An error occurred executing Vector's sink config file template.") + } + } + + templateStr := templateStrBuffer.String() + + return templateStr, nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go index 503340cff9..f7822deca2 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go @@ -1,11 +1,9 @@ package vector import ( - "bytes" "fmt" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" "github.com/kurtosis-tech/stacktrace" - "text/template" ) const ( @@ -33,7 +31,7 @@ func (vector *vectorContainerConfigProvider) GetContainerArgs( logsStorageVolumeName: logsStorageDirpath, } - logsAggregatorConfigContentStr, err := vector.getConfigFileContent() + logsAggregatorConfigContentStr, err := vector.config.getConfigFileContent() if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting the Loki server's configuration content") } @@ -72,20 +70,3 @@ func (vector *vectorContainerConfigProvider) GetContainerArgs( return createAndStartArgs, nil } - -func (vector *vectorContainerConfigProvider) getConfigFileContent() (string, error) { - cngFileTemplate, err := template.New(configFileTemplateName).Parse(configFileTemplate) - if err != nil { - return "", stacktrace.Propagate(err, "An error occurred parsing Vector config template '%v'", configFileTemplate) - } - - templateStrBuffer := &bytes.Buffer{} - - if err := cngFileTemplate.Execute(templateStrBuffer, vector.config); err != nil { - return "", stacktrace.Propagate(err, "An error occurred executing the Vector config file template") - } - - templateStr := templateStrBuffer.String() - - return templateStr, nil -} From 4cdc050f1c15037031aecd6e4f37cb8f50b07551 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 31 Aug 2023 10:35:26 -0400 Subject: [PATCH 3/3] debug --- .../implementations/vector/consts.go | 4 ++-- .../implementations/vector/vector_config.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go index d4624136cf..179e4841af 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go @@ -18,8 +18,8 @@ const ( fluentBitSourceType = "\"fluent\"" fluentBitSourceIpAddress = "0.0.0.0" - fileSinkId = "\"file\"" - fileTypeId = "\"file\"" + fileSinkIdSuffix = "file" + fileTypeId = "\"file\"" // We store log files in the volume per-enclave, per-service // To construct the filepath, we utilize vectors template syntax that allows us to reference fields in log events diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go index 7e769b0630..54c5dad80a 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go @@ -35,19 +35,19 @@ func newDefaultVectorConfig(listeningPortNumber uint16) *VectorConfig { }, Sinks: []*Sink{ { - Id: fileSinkId, + Id: "uuid_" + fileSinkIdSuffix, Type: fileTypeId, Inputs: []string{fluentBitSourceId}, Filepath: uuidLogsFilepath, }, { - Id: fileSinkId, + Id: "name_" + fileSinkIdSuffix, Type: fileTypeId, Inputs: []string{fluentBitSourceId}, Filepath: nameLogsFilepath, }, { - Id: fileSinkId, + Id: "short_uuid_" + fileSinkIdSuffix, Type: fileTypeId, Inputs: []string{fluentBitSourceId}, Filepath: shortUUIDLogsFilepath,