From 539ba39cb570f2ac48f6a0f8aab435aa27812ca1 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 11 Aug 2023 11:47:15 -0400 Subject: [PATCH 01/40] create logs storage volume --- .../engine_functions/create_engine.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go index 92d98dca9e..5c972d1e9a 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go @@ -72,6 +72,13 @@ func CreateEngine( targetNetworkId := engineNetwork.GetId() logrus.Infof("Starting the centralized logs components...") + // Creation of volume should be idempotent because the volume with persisted logs in it could already exist + // Thus, we don't defer an undo volume if this operation fails + // TODO: retrieve name and labels from objsAttrProvider + if err = dockerManager.CreateVolume(ctx, "kurtosis-logs-storage", map[string]string{}); err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating logs storage.") + } + logsAggregatorContainer := vector.NewVectorLogsAggregatorContainer() // Declaring implementation _, removeLogsAggregatorFunc, err := logs_aggregator_functions.CreateLogsAggregator( ctx, From 9ae153955e71e64a5e90c0b4f993efb6d8055a26 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 11 Aug 2023 12:14:28 -0400 Subject: [PATCH 02/40] mount volume to aggregator --- .../implementations/vector/consts.go | 2 ++ .../vector/vector_container_config_provider.go | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go index 6c00ae4fd3..e55df0ac1f 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go @@ -12,6 +12,8 @@ const ( configFilepath = configDirpath + "vector.toml" binaryFilepath = "/usr/bin/vector" configFileFlag = "-c" + + logsStorageDirpath = "/etc/vector/" ////////////////////////--FINISH VECTOR CONTAINER CONFIGURATION SECTION--///////////////////////////// ////////////////////////--VECTOR CONFIGURATION SECTION--///////////////////////////// diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go index b3d22a60d5..c843820de4 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go @@ -27,6 +27,11 @@ func (vector *vectorContainerConfigProvider) GetContainerArgs( containerLabels map[string]string, networkId string, ) (*docker_manager.CreateAndStartContainerArgs, error) { + // TODO: get volume name from arg pass in via objs attr provider + volumeMounts := map[string]string{ + "kurtosis-logs-storage": logsStorageDirpath, + } + logsAggregatorConfigContentStr, err := vector.getConfigFileContent() if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting the Loki server's configuration content") @@ -54,6 +59,8 @@ func (vector *vectorContainerConfigProvider) GetContainerArgs( networkId, ).WithLabels( containerLabels, + ).WithVolumeMounts( + volumeMounts, ).WithEntrypointArgs( []string{ shBinaryFilepath, From ff8735f6a96fe24af72f867aa0fdc04bacbbeaa2 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 11 Aug 2023 12:58:15 -0400 Subject: [PATCH 03/40] mount volume to engine --- .../engine_functions/create_engine.go | 6 +++++ .../implementations/vector/consts.go | 24 ++++++++++++++++--- .../implementations/vector/vector_config.go | 14 ++++++----- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go index 5c972d1e9a..56db8d1d68 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go @@ -141,6 +141,10 @@ func CreateEngine( consts.DockerSocketFilepath: consts.DockerSocketFilepath, } + volumeMounts := map[string]string{ + "kurtosis-logs-storage": "/tmp/", + } + if serverArgs.OnBastionHost { // Mount the host engine config directory so the engine can access files like the remote backend config. bindMounts[consts.HostEngineConfigDirToMount] = consts.EngineConfigLocalDir @@ -165,6 +169,8 @@ func CreateEngine( envVars, ).WithBindMounts( bindMounts, + ).WithVolumeMounts( + volumeMounts, ).WithUsedPorts( usedPorts, ).WithLabels( diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go index e55df0ac1f..99b40346f9 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go @@ -13,7 +13,7 @@ const ( binaryFilepath = "/usr/bin/vector" configFileFlag = "-c" - logsStorageDirpath = "/etc/vector/" + logsStorageDirpath = "/tmp/" ////////////////////////--FINISH VECTOR CONTAINER CONFIGURATION SECTION--///////////////////////////// ////////////////////////--VECTOR CONFIGURATION SECTION--///////////////////////////// @@ -25,8 +25,26 @@ const ( stdoutSinkID = "\"stdout\"" stdoutTypeId = "\"console\"" + fileSinkId = "\"file\"" + fileTypeId = "\"file\"" + filepathForLogs = "/tmp/vector.txt" + configFileTemplateName = "vectorConfigFileTemplate" - configFileTemplate = ` + // configFileTemplate = ` + //[api] + //enabled = true + //address = "0.0.0.0:8686" + // + //[sources.{{ .Source.Id }}] + //type = {{ .Source.Type }} + //address = "{{ .Source.Address }}" + // + //[sinks.{{ .Sink.Id }}] + //type = {{ .Sink.Type }} + //inputs = {{ .Sink.Inputs }} + //encoding.codec = "json" + + configFileTemplate = ` [api] enabled = true address = "0.0.0.0:8686" @@ -38,7 +56,7 @@ address = "{{ .Source.Address }}" [sinks.{{ .Sink.Id }}] type = {{ .Sink.Type }} inputs = {{ .Sink.Inputs }} -encoding.codec = "json" +path = {{ .Sink.Filepath }} ` ////////////////////////--FINISH--VECTOR CONFIGURATION SECTION--///////////////////////////// ) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go index 133e568b60..ae4d0b6244 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go @@ -17,9 +17,10 @@ type Source struct { } type Sink struct { - Id string - Type string - Inputs []string + Id string + Type string + Inputs []string + Filepath string } func newDefaultVectorConfig(listeningPortNumber uint16) *VectorConfig { @@ -30,9 +31,10 @@ func newDefaultVectorConfig(listeningPortNumber uint16) *VectorConfig { Address: fmt.Sprintf("%s:%s", fluentBitSourceIpAddress, strconv.Itoa(int(listeningPortNumber))), }, Sink: &Sink{ - Id: stdoutSinkID, - Type: stdoutTypeId, - Inputs: []string{fluentBitSourceId}, + Id: fileSinkId, + Type: fileTypeId, + Inputs: []string{fluentBitSourceId}, + Filepath: filepathForLogs, }, } } From e5d1580f08192a4f7688d067a45ca3aac887bfad Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Mon, 14 Aug 2023 12:22:25 -0400 Subject: [PATCH 04/40] configure logs to output to volume --- .../implementations/vector/consts.go | 27 ++++--------------- .../implementations/vector/vector_config.go | 2 +- 2 files changed, 6 insertions(+), 23 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go index 99b40346f9..ee62674e89 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go @@ -21,30 +21,12 @@ const ( fluentBitSourceType = "\"fluent\"" fluentBitSourceIpAddress = "0.0.0.0" - // TODO: change output when persistent volume is implemented - stdoutSinkID = "\"stdout\"" - stdoutTypeId = "\"console\"" - - fileSinkId = "\"file\"" - fileTypeId = "\"file\"" - filepathForLogs = "/tmp/vector.txt" + fileSinkId = "\"file\"" + fileTypeId = "\"file\"" + logsFilepath = "\"/tmp/vector.json\"" configFileTemplateName = "vectorConfigFileTemplate" - // configFileTemplate = ` - //[api] - //enabled = true - //address = "0.0.0.0:8686" - // - //[sources.{{ .Source.Id }}] - //type = {{ .Source.Type }} - //address = "{{ .Source.Address }}" - // - //[sinks.{{ .Sink.Id }}] - //type = {{ .Sink.Type }} - //inputs = {{ .Sink.Inputs }} - //encoding.codec = "json" - - configFileTemplate = ` + configFileTemplate = ` [api] enabled = true address = "0.0.0.0:8686" @@ -57,6 +39,7 @@ address = "{{ .Source.Address }}" type = {{ .Sink.Type }} inputs = {{ .Sink.Inputs }} path = {{ .Sink.Filepath }} +encoding.codec = "json" ` ////////////////////////--FINISH--VECTOR CONFIGURATION SECTION--///////////////////////////// ) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go index ae4d0b6244..cee5f5efed 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_config.go @@ -34,7 +34,7 @@ func newDefaultVectorConfig(listeningPortNumber uint16) *VectorConfig { Id: fileSinkId, Type: fileTypeId, Inputs: []string{fluentBitSourceId}, - Filepath: filepathForLogs, + Filepath: logsFilepath, }, } } From d4f3a31a8918cbbbb08390e45c162434ca3307d1 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 15 Aug 2023 11:20:31 -0400 Subject: [PATCH 05/40] add persistent volume logs client --- .../persistent_volume_logs_database_client.go | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go new file mode 100644 index 0000000000..f313ad77cc --- /dev/null +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go @@ -0,0 +1,106 @@ +package persistent_volume + +import ( + "context" + "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/enclaves" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/logline" + "github.com/kurtosis-tech/stacktrace" +) + +const ( + // Location of logs on the filesystem of the engine + logsFilepath = "var/log/kurtosis/logs.json" +) + +// persistentVolumeLogsDatabaseClient pulls logs from a Docker volume the engine is mounted to +type persistentVolumeLogsDatabaseClient struct { + kurtosisBackend backend_interface.KurtosisBackend +} + +func NewPersistentVolumeLogsDatabaseClient(kurtosisBackend backend_interface.KurtosisBackend) *persistentVolumeLogsDatabaseClient { + return &persistentVolumeLogsDatabaseClient{ + kurtosisBackend: kurtosisBackend, + } +} + +func (client *persistentVolumeLogsDatabaseClient) StreamUserServiceLogs( + ctx context.Context, + enclaveUuid enclaves.EnclaveUUID, + userServiceUuids map[service.ServiceUUID]bool, + conjunctiveLogLineFilters logline.ConjunctiveLogLineFilters, + shouldFollowLogs bool, +) ( + chan map[service.ServiceUUID][]logline.LogLine, + chan error, + context.CancelFunc, + error, +) { + ctx, cancelCtxFunc := context.WithCancel(ctx) + + // create user service filers + + // create log filters + + // grab logs + // return error if smth happens + + // create err chan + // return err if anything happened + + // create go routine to stream logs for each requested service + + // create go routine to handle stream cancellation + // wait for all senders to end + // close all resources + // cancel context + + // return everything + return nil, nil, cancelCtxFunc, nil +} + +func (client *persistentVolumeLogsDatabaseClient) FilterExistingServiceUuids( + ctx context.Context, + enclaveUuid enclave.EnclaveUUID, + userServiceUuids map[service.ServiceUUID]bool, +) (map[service.ServiceUUID]bool, error) { + userServiceFilters := &service.ServiceFilters{ + Names: nil, + UUIDs: userServiceUuids, + Statuses: nil, + } + + existingServicesByUuids, err := client.kurtosisBackend.GetUserServices(ctx, enclaveUuid, userServiceFilters) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting user services for enclave with UUID '%v' and using filters '%+v'", enclaveUuid, userServiceFilters) + } + + filteredServiceUuidsSet := map[service.ServiceUUID]bool{} + for serviceUuid := range userServiceUuids { + if _, found := existingServicesByUuids[serviceUuid]; found { + filteredServiceUuidsSet[serviceUuid] = true + } + } + return filteredServiceUuidsSet, nil +} + +// ==================================================================================================== +// +// Private helper functions +// +// ==================================================================================================== +func streamServiceLogLines( + ctx context.Context, + logsByKurtosisUserServiceUuidChan chan map[service.ServiceUUID][]logline.LogLine, + streamErrChan chan error, + // conjunctiveLogLines []LogLinesFilterWithRegex, +) { + // for + // return if context was canceled + // read a log line + // turn it into a log line object + // filter the log line based on the conjunctive filter regex + // send the log line +} From 2f03d7eeb13bbdc2d9e6c71a330a81ec42867773 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 15 Aug 2023 11:31:10 -0400 Subject: [PATCH 06/40] refactor logline filters with regex --- .../kurtosis_backend_logs_database_client.go | 70 ++----------------- .../logline_filter_with_regex.go | 15 ---- .../centralized_logs/logline/logline.go | 46 +++++++++++- .../logline/logline_filter_with_regex.go | 34 +++++++++ 4 files changed, 83 insertions(+), 82 deletions(-) delete mode 100644 engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/logline_filter_with_regex.go create mode 100644 engine/server/engine/centralized_logs/logline/logline_filter_with_regex.go diff --git a/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/kurtosis_backend_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/kurtosis_backend_logs_database_client.go index 81eafcc168..bc080ed7de 100644 --- a/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/kurtosis_backend_logs_database_client.go +++ b/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/kurtosis_backend_logs_database_client.go @@ -12,7 +12,6 @@ import ( "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" "io" - "regexp" "strings" "sync" ) @@ -49,7 +48,7 @@ func (client *kurtosisBackendLogsDatabaseClient) StreamUserServiceLogs( userServiceFilters := newUserServiceFilters(userServiceUuids) - conjunctiveLogFiltersWithRegex, err := newConjunctiveLogFiltersWithRegex(conjunctiveLogLineFilters) + conjunctiveLogFiltersWithRegex, err := logline.NewConjunctiveLogFiltersWithRegex(conjunctiveLogLineFilters) if err != nil { cancelCtxFunc() return nil, nil, nil, stacktrace.Propagate(err, "An error occurred creating conjunctive log line filter with regex from filters '%+v'", conjunctiveLogLineFilters) @@ -171,7 +170,7 @@ func streamServiceLogLines( streamErrChan chan error, serviceUuid service.ServiceUUID, userServiceReadCloserLog io.ReadCloser, - conjunctiveLogLinesFiltersWithRegex []LogLineFilterWithRegex, + conjunctiveLogLinesFiltersWithRegex []logline.LogLineFilterWithRegex, ) { defer wgSenders.Done() @@ -179,7 +178,7 @@ func streamServiceLogLines( for { select { - //client cancel ctx case + // client cancel ctx case case <-ctx.Done(): logrus.Debugf("Context was canceled, stopping streaming service logs for service '%v'", serviceUuid) return @@ -199,7 +198,7 @@ func streamServiceLogLines( logLine := logline.NewLogLine(logLineStr) //filtering it - shouldReturnLogLine, err := shouldReturnLogLineBaseOnFilters(logLine, conjunctiveLogLinesFiltersWithRegex) + shouldReturnLogLine, err := logLine.IsValidLogLineBaseOnFilters(conjunctiveLogLinesFiltersWithRegex) if err != nil { streamErrChan <- stacktrace.Propagate(err, "An error occurred filtering log line '%+v' using filters '%+v'", logLine, conjunctiveLogLinesFiltersWithRegex) break @@ -217,64 +216,3 @@ func streamServiceLogLines( } } } - -func shouldReturnLogLineBaseOnFilters( - logLine *logline.LogLine, - conjunctiveLogLinesFiltersWithRegex []LogLineFilterWithRegex, -) (bool, error) { - - shouldReturnIt := true - - for _, logLineFilter := range conjunctiveLogLinesFiltersWithRegex { - operator := logLineFilter.GetOperator() - - logLineContent := logLine.GetContent() - logLineContentLowerCase := strings.ToLower(logLineContent) - textPatternLowerCase := strings.ToLower(logLineFilter.GetTextPattern()) - - switch operator { - case logline.LogLineOperator_DoesContainText: - if !strings.Contains(logLineContentLowerCase, textPatternLowerCase) { - shouldReturnIt = false - } - case logline.LogLineOperator_DoesNotContainText: - if strings.Contains(logLineContentLowerCase, textPatternLowerCase) { - shouldReturnIt = false - } - case logline.LogLineOperator_DoesContainMatchRegex: - if !logLineFilter.compiledRegexPattern.MatchString(logLineContent) { - shouldReturnIt = false - } - case logline.LogLineOperator_DoesNotContainMatchRegex: - if logLineFilter.compiledRegexPattern.MatchString(logLineContent) { - shouldReturnIt = false - } - default: - return false, stacktrace.NewError("Unrecognized log line filter operator '%v' in filter '%v'; this is a bug in Kurtosis", operator, logLineFilter) - } - if !shouldReturnIt { - break - } - } - - return shouldReturnIt, nil -} - -func newConjunctiveLogFiltersWithRegex(conjunctiveLogLineFilters logline.ConjunctiveLogLineFilters) ([]LogLineFilterWithRegex, error) { - conjunctiveLogFiltersWithRegex := []LogLineFilterWithRegex{} - for _, logLineFilter := range conjunctiveLogLineFilters { - logLineFilterWithRegex := newLogLineFilterWithRegex(logLineFilter, nil) - - if logLineFilter.IsRegexFilter() { - filterRegexPattern := logLineFilter.GetTextPattern() - logLineRegexPattern, err := regexp.Compile(filterRegexPattern) - if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred compiling regex string '%v' for log line filter '%+v'", filterRegexPattern, logLineFilter) - } - logLineFilterWithRegex.compiledRegexPattern = logLineRegexPattern - } - conjunctiveLogFiltersWithRegex = append(conjunctiveLogFiltersWithRegex, *logLineFilterWithRegex) - } - - return conjunctiveLogFiltersWithRegex, nil -} diff --git a/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/logline_filter_with_regex.go b/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/logline_filter_with_regex.go deleted file mode 100644 index 2fbcc0da50..0000000000 --- a/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/logline_filter_with_regex.go +++ /dev/null @@ -1,15 +0,0 @@ -package kurtosis_backend - -import ( - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/logline" - "regexp" -) - -type LogLineFilterWithRegex struct { - logline.LogLineFilter - compiledRegexPattern *regexp.Regexp -} - -func newLogLineFilterWithRegex(logLineFilter logline.LogLineFilter, compiledRegexPattern *regexp.Regexp) *LogLineFilterWithRegex { - return &LogLineFilterWithRegex{LogLineFilter: logLineFilter, compiledRegexPattern: compiledRegexPattern} -} diff --git a/engine/server/engine/centralized_logs/logline/logline.go b/engine/server/engine/centralized_logs/logline/logline.go index 3b6ae7981e..c23d66315c 100644 --- a/engine/server/engine/centralized_logs/logline/logline.go +++ b/engine/server/engine/centralized_logs/logline/logline.go @@ -1,6 +1,9 @@ package logline -import "strings" +import ( + "github.com/kurtosis-tech/stacktrace" + "strings" +) const ( newlineChar = "\n" @@ -19,3 +22,44 @@ func NewLogLine(content string) *LogLine { func (logLine LogLine) GetContent() string { return logLine.content } + +func (logLine LogLine) IsValidLogLineBaseOnFilters( + conjunctiveLogLinesFiltersWithRegex []LogLineFilterWithRegex, +) (bool, error) { + + shouldReturnIt := true + + for _, logLineFilter := range conjunctiveLogLinesFiltersWithRegex { + operator := logLineFilter.GetOperator() + + logLineContent := logLine.GetContent() + logLineContentLowerCase := strings.ToLower(logLineContent) + textPatternLowerCase := strings.ToLower(logLineFilter.GetTextPattern()) + + switch operator { + case LogLineOperator_DoesContainText: + if !strings.Contains(logLineContentLowerCase, textPatternLowerCase) { + shouldReturnIt = false + } + case LogLineOperator_DoesNotContainText: + if strings.Contains(logLineContentLowerCase, textPatternLowerCase) { + shouldReturnIt = false + } + case LogLineOperator_DoesContainMatchRegex: + if !logLineFilter.compiledRegexPattern.MatchString(logLineContent) { + shouldReturnIt = false + } + case LogLineOperator_DoesNotContainMatchRegex: + if logLineFilter.compiledRegexPattern.MatchString(logLineContent) { + shouldReturnIt = false + } + default: + return false, stacktrace.NewError("Unrecognized log line filter operator '%v' in filter '%v'; this is a bug in Kurtosis", operator, logLineFilter) + } + if !shouldReturnIt { + break + } + } + + return shouldReturnIt, nil +} diff --git a/engine/server/engine/centralized_logs/logline/logline_filter_with_regex.go b/engine/server/engine/centralized_logs/logline/logline_filter_with_regex.go new file mode 100644 index 0000000000..b7d691e981 --- /dev/null +++ b/engine/server/engine/centralized_logs/logline/logline_filter_with_regex.go @@ -0,0 +1,34 @@ +package logline + +import ( + "github.com/kurtosis-tech/stacktrace" + "regexp" +) + +type LogLineFilterWithRegex struct { + LogLineFilter + compiledRegexPattern *regexp.Regexp +} + +func NewLogLineFilterWithRegex(logLineFilter LogLineFilter, compiledRegexPattern *regexp.Regexp) *LogLineFilterWithRegex { + return &LogLineFilterWithRegex{LogLineFilter: logLineFilter, compiledRegexPattern: compiledRegexPattern} +} + +func NewConjunctiveLogFiltersWithRegex(conjunctiveLogLineFilters ConjunctiveLogLineFilters) ([]LogLineFilterWithRegex, error) { + conjunctiveLogFiltersWithRegex := []LogLineFilterWithRegex{} + for _, logLineFilter := range conjunctiveLogLineFilters { + logLineFilterWithRegex := NewLogLineFilterWithRegex(logLineFilter, nil) + + if logLineFilter.IsRegexFilter() { + filterRegexPattern := logLineFilter.GetTextPattern() + logLineRegexPattern, err := regexp.Compile(filterRegexPattern) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred compiling regex string '%v' for log line filter '%+v'", filterRegexPattern, logLineFilter) + } + logLineFilterWithRegex.compiledRegexPattern = logLineRegexPattern + } + conjunctiveLogFiltersWithRegex = append(conjunctiveLogFiltersWithRegex, *logLineFilterWithRegex) + } + + return conjunctiveLogFiltersWithRegex, nil +} From 9a6294da89d32b4f21b460045498baa7f45fd0fa Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 15 Aug 2023 13:19:33 -0400 Subject: [PATCH 07/40] impl persistent volume logs client --- .../kurtosis_backend_logs_database_client.go | 15 +- .../persistent_volume_logs_database_client.go | 175 +++++++++++++++--- 2 files changed, 156 insertions(+), 34 deletions(-) diff --git a/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/kurtosis_backend_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/kurtosis_backend_logs_database_client.go index bc080ed7de..73a6c8c12d 100644 --- a/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/kurtosis_backend_logs_database_client.go +++ b/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/kurtosis_backend_logs_database_client.go @@ -46,7 +46,11 @@ func (client *kurtosisBackendLogsDatabaseClient) StreamUserServiceLogs( ctx, cancelCtxFunc := context.WithCancel(ctx) - userServiceFilters := newUserServiceFilters(userServiceUuids) + userServiceFilters := &service.ServiceFilters{ + Names: nil, + UUIDs: userServiceUuids, + Statuses: nil, + } conjunctiveLogFiltersWithRegex, err := logline.NewConjunctiveLogFiltersWithRegex(conjunctiveLogLineFilters) if err != nil { @@ -154,15 +158,6 @@ func (client *kurtosisBackendLogsDatabaseClient) FilterExistingServiceUuids( // Private helper functions // // ==================================================================================================== -func newUserServiceFilters(userServiceGuids map[service.ServiceUUID]bool) *service.ServiceFilters { - userServiceFilters := &service.ServiceFilters{ - Names: nil, - UUIDs: userServiceGuids, - Statuses: nil, - } - return userServiceFilters -} - func streamServiceLogLines( ctx context.Context, wgSenders *sync.WaitGroup, diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go index f313ad77cc..1f08fb47a0 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go @@ -1,20 +1,37 @@ package persistent_volume import ( + "bufio" "context" - "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/enclaves" + "encoding/json" + "errors" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/logline" "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" + "io" + "os" + "sync" ) const ( + oneSenderAdded = 1 + // Location of logs on the filesystem of the engine logsFilepath = "var/log/kurtosis/logs.json" + + newlineRune = '\n' + serviceUUIDLogLabel = "service_id" + enclaveUUIDLogLabel = "enclave_id" + logLabel = "log" + + maxNumLogsToReturn = 200 ) +type JsonLog map[string]string + // persistentVolumeLogsDatabaseClient pulls logs from a Docker volume the engine is mounted to type persistentVolumeLogsDatabaseClient struct { kurtosisBackend backend_interface.KurtosisBackend @@ -28,7 +45,7 @@ func NewPersistentVolumeLogsDatabaseClient(kurtosisBackend backend_interface.Kur func (client *persistentVolumeLogsDatabaseClient) StreamUserServiceLogs( ctx context.Context, - enclaveUuid enclaves.EnclaveUUID, + enclaveUuid enclave.EnclaveUUID, userServiceUuids map[service.ServiceUUID]bool, conjunctiveLogLineFilters logline.ConjunctiveLogLineFilters, shouldFollowLogs bool, @@ -40,25 +57,55 @@ func (client *persistentVolumeLogsDatabaseClient) StreamUserServiceLogs( ) { ctx, cancelCtxFunc := context.WithCancel(ctx) - // create user service filers - - // create log filters - - // grab logs - // return error if smth happens + logsFile, err := os.Open(logsFilepath) + if err != nil { + cancelCtxFunc() + return nil, nil, nil, stacktrace.Propagate(err, "An error occurred opening logs file while attempting to stream logs.") + } - // create err chan - // return err if anything happened + conjunctiveLogFiltersWithRegex, err := logline.NewConjunctiveLogFiltersWithRegex(conjunctiveLogLineFilters) + if err != nil { + cancelCtxFunc() + return nil, nil, nil, stacktrace.Propagate(err, "An error occurred creating conjunctive log line filter with regex from filters '%+v'", conjunctiveLogLineFilters) + } - // create go routine to stream logs for each requested service + // this channel return an error if the stream fails at some point + streamErrChan := make(chan error) + + // this channel will return the user service log lines by service UUID + logsByKurtosisUserServiceUuidChan := make(chan map[service.ServiceUUID][]logline.LogLine) + + wgSenders := &sync.WaitGroup{} + wgSenders.Add(oneSenderAdded) + go streamServiceLogLines( + ctx, + wgSenders, + logsByKurtosisUserServiceUuidChan, + streamErrChan, + enclaveUuid, + userServiceUuids, + logsFile, + conjunctiveLogFiltersWithRegex, + shouldFollowLogs, + ) + + // this go routine handles the stream cancellation + go func() { + //wait for stream go routine to end + wgSenders.Wait() + + //close resources first + if err := logsFile.Close(); err != nil { + logrus.Warnf("An error occurred attempting to close the user service logs file after streaming:\n%v", err) + } + close(logsByKurtosisUserServiceUuidChan) + close(streamErrChan) - // create go routine to handle stream cancellation - // wait for all senders to end - // close all resources - // cancel context + //then cancel the context + cancelCtxFunc() + }() - // return everything - return nil, nil, cancelCtxFunc, nil + return logsByKurtosisUserServiceUuidChan, streamErrChan, cancelCtxFunc, nil } func (client *persistentVolumeLogsDatabaseClient) FilterExistingServiceUuids( @@ -93,14 +140,94 @@ func (client *persistentVolumeLogsDatabaseClient) FilterExistingServiceUuids( // ==================================================================================================== func streamServiceLogLines( ctx context.Context, + wgSenders *sync.WaitGroup, logsByKurtosisUserServiceUuidChan chan map[service.ServiceUUID][]logline.LogLine, streamErrChan chan error, - // conjunctiveLogLines []LogLinesFilterWithRegex, + enclaveUuid enclave.EnclaveUUID, + userServiceUuids map[service.ServiceUUID]bool, + logsFile io.Reader, + conjunctiveLogLinesFiltersWithRegex []logline.LogLineFilterWithRegex, + shouldFollowLogs bool, ) { - // for - // return if context was canceled - // read a log line - // turn it into a log line object - // filter the log line based on the conjunctive filter regex - // send the log line + defer wgSenders.Done() + + logsReader := bufio.NewReader(logsFile) + + numLogsReturned := 0 + for shouldFollowLogs || numLogsReturned < maxNumLogsToReturn { + select { + case <-ctx.Done(): + logrus.Debugf("Context was canceled, stopping streaming service logs for services '%v'", userServiceUuids) + return + default: + jsonLogStr, err := logsReader.ReadString(newlineRune) + if err != nil && errors.Is(err, io.EOF) { + // exiting stream + logrus.Debugf("EOF error returned when reading logs for services '%v'", userServiceUuids) + return + } + if err != nil { + streamErrChan <- stacktrace.Propagate(err, "An error occurred reading the logs file '%v'", userServiceUuids) + return + } + + // each logLineStr is of the following structure: {"label1": "...", "label2":"...", "log": "...", "timestamp", ... } + // eg. {"container_type":"api-container","enclave_id":"ffd1c0ba29e1a464c","container_id":"8f8558ba", + // "container_name":"/kurtosis-api--ffd", "log":"hi","timestamp":"2023-08-14T14:57:49Z"} + + // First, we decode the line + var jsonLog JsonLog + err = json.Unmarshal([]byte(jsonLogStr), &jsonLog) + if err != nil { + streamErrChan <- stacktrace.Propagate(err, "An error occurred reading the logs file '%v'", userServiceUuids) + return + } + + // Then we extract the actual log message using the "log" field + logLineStr, found := jsonLog[logLabel] + if !found { + streamErrChan <- stacktrace.NewError("An error retrieving the log field from logs json file. This is a bug in Kurtosis.") + return + } + logLine := logline.NewLogLine(logLineStr) + + // We also extract enclave uuid and service uuid + logEnclaveUuidStr, found := jsonLog[enclaveUUIDLogLabel] + if !found { + streamErrChan <- stacktrace.NewError("An error retrieving the enclave uuid field from logs json file. This is a bug in Kurtosis.") + return + } + logServiceUuidStr, found := jsonLog[serviceUUIDLogLabel] + if !found { + streamErrChan <- stacktrace.NewError("An error retrieving the enclave uuid field from logs json file. This is a bug in Kurtosis.") + return + } + + logEnclaveUuid := enclave.EnclaveUUID(logEnclaveUuidStr) + logServiceUuid := service.ServiceUUID(logServiceUuidStr) + + // Then we filter by checking: + // 1. if the log message is valid based on requested filters + // 2. if the log is associated with the requested enclave and one of the requested services + // we check this bc currently all logs are in one file + isValidBasedOnFilters, err := logLine.IsValidLogLineBaseOnFilters(conjunctiveLogLinesFiltersWithRegex) + if err != nil { + streamErrChan <- stacktrace.Propagate(err, "An error occurred filtering log line '%+v' using filters '%+v'", logLine, conjunctiveLogLinesFiltersWithRegex) + break + } + isValidBasedOnEnclaveAndServiceUuid := (enclaveUuid == logEnclaveUuid) && (userServiceUuids[logServiceUuid]) + shouldReturnLogLine := isValidBasedOnFilters && isValidBasedOnEnclaveAndServiceUuid + if !shouldReturnLogLine { + break + } + + // send the log line + logLines := []logline.LogLine{*logLine} + userServicesLogLinesMap := map[service.ServiceUUID][]logline.LogLine{ + logServiceUuid: logLines, + } + logsByKurtosisUserServiceUuidChan <- userServicesLogLinesMap + numLogsReturned++ + } + } } From f33f73bd1314efa07db6b16778f56468a9934468 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 15 Aug 2023 14:35:19 -0400 Subject: [PATCH 08/40] change logs location --- .../engine_functions/create_engine.go | 2 +- .../implementations/vector/consts.go | 12 ++++++++---- .../persistent_volume_logs_database_client.go | 9 +++++---- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go index 56db8d1d68..8cacefbdba 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go @@ -142,7 +142,7 @@ func CreateEngine( } volumeMounts := map[string]string{ - "kurtosis-logs-storage": "/tmp/", + "kurtosis-logs-storage": "/var/log/kurtosis/", } if serverArgs.OnBastionHost { diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go index ee62674e89..9ac52e05f0 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go @@ -13,7 +13,7 @@ const ( binaryFilepath = "/usr/bin/vector" configFileFlag = "-c" - logsStorageDirpath = "/tmp/" + logsStorageDirpath = "/var/log/kurtosis/" ////////////////////////--FINISH VECTOR CONTAINER CONFIGURATION SECTION--///////////////////////////// ////////////////////////--VECTOR CONFIGURATION SECTION--///////////////////////////// @@ -21,9 +21,13 @@ const ( fluentBitSourceType = "\"fluent\"" fluentBitSourceIpAddress = "0.0.0.0" - fileSinkId = "\"file\"" - fileTypeId = "\"file\"" - logsFilepath = "\"/tmp/vector.json\"" + fileSinkId = "\"file\"" + fileTypeId = "\"file\"" + + // We store log files per-enclave, per-service + // To construct the filepath, we utilize vectors template syntax that allows us to reference fields in log events + // https://vector.dev/docs/reference/configuration/template-syntax/ + logsFilepath = "\"" + logsStorageDirpath + "{{ timestamp }}/{{ container_name }}-logs.json\"" configFileTemplateName = "vectorConfigFileTemplate" configFileTemplate = ` diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go index 1f08fb47a0..bc1afbc33d 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go @@ -22,9 +22,10 @@ const ( // Location of logs on the filesystem of the engine logsFilepath = "var/log/kurtosis/logs.json" - newlineRune = '\n' - serviceUUIDLogLabel = "service_id" - enclaveUUIDLogLabel = "enclave_id" + newlineRune = '\n' + + serviceUUIDLogLabel = "container_id" + enclaveUUIDLogLabel = "com.kurtosistech.enclave-id" logLabel = "log" maxNumLogsToReturn = 200 @@ -60,7 +61,7 @@ func (client *persistentVolumeLogsDatabaseClient) StreamUserServiceLogs( logsFile, err := os.Open(logsFilepath) if err != nil { cancelCtxFunc() - return nil, nil, nil, stacktrace.Propagate(err, "An error occurred opening logs file while attempting to stream logs.") + return nil, nil, nil, stacktrace.Propagate(err, "An error occurred opening logs file.") } conjunctiveLogFiltersWithRegex, err := logline.NewConjunctiveLogFiltersWithRegex(conjunctiveLogLineFilters) From debd20a2b4adafe1241cfb01eb7f4d9b4d998d20 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 15 Aug 2023 15:05:45 -0400 Subject: [PATCH 09/40] add enclave name to log stream --- .../docker_labels_for_logs/docker_labels_for_logs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go index 8d4b5d37c1..7ba502a286 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go @@ -11,6 +11,7 @@ var LogsDatabaseKurtosisTrackedDockerLabelsForIdentifyLogsStream = []*docker_lab label_key_consts.GUIDDockerLabelKey, label_key_consts.ContainerTypeDockerLabelKey, label_key_consts.EnclaveUUIDDockerLabelKey, + label_key_consts.EnclaveNameDockerLabelKey, } // These are all the logs database Kurtosis tracked Docker Labels used From d542a8ed98f69b4e2abd5a1ccc04486d6bbdec88 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 15 Aug 2023 15:46:53 -0400 Subject: [PATCH 10/40] use persistent volume client --- .../implementations/vector/consts.go | 13 ++++--------- .../persistent_volume_logs_database_client.go | 2 +- engine/server/engine/main.go | 5 +++-- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go index 9ac52e05f0..83e6fd4d4c 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go @@ -1,10 +1,7 @@ package vector const ( - configDirpath = "/etc/vector/" - healthCheckEndpoint = "health" - defaultGraphQlApiHttpPortNum = uint16(8686) - httpProtocolStr = "http" + configDirpath = "/etc/vector/" ////////////////////////--VECTOR CONTAINER CONFIGURATION SECTION--///////////////////////////// containerImage = "timberio/vector:0.31.0-debian" @@ -26,15 +23,13 @@ const ( // We store log files per-enclave, per-service // To construct the filepath, we utilize vectors template syntax that allows us to reference fields in log events + // https://vector.dev/docs/reference/configuration/template-syntax/ - logsFilepath = "\"" + logsStorageDirpath + "{{ timestamp }}/{{ container_name }}-logs.json\"" + //logsFilepath = "\"" + logsStorageDirpath + "/{{ container_name }}.json\"" + logsFilepath = "\"" + logsStorageDirpath + "logs.json\"" configFileTemplateName = "vectorConfigFileTemplate" configFileTemplate = ` -[api] -enabled = true -address = "0.0.0.0:8686" - [sources.{{ .Source.Id }}] type = {{ .Source.Type }} address = "{{ .Source.Address }}" diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go index bc1afbc33d..ccbc6836e1 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go @@ -20,7 +20,7 @@ const ( oneSenderAdded = 1 // Location of logs on the filesystem of the engine - logsFilepath = "var/log/kurtosis/logs.json" + logsFilepath = "/var/log/kurtosis/logs.json" newlineRune = '\n' diff --git a/engine/server/engine/main.go b/engine/server/engine/main.go index c1f36208f8..97ab63205f 100644 --- a/engine/server/engine/main.go +++ b/engine/server/engine/main.go @@ -17,7 +17,7 @@ import ( "github.com/kurtosis-tech/kurtosis/core/launcher/api_container_launcher" "github.com/kurtosis-tech/kurtosis/engine/launcher/args" "github.com/kurtosis-tech/kurtosis/engine/launcher/args/kurtosis_backend_config" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume" "github.com/kurtosis-tech/kurtosis/engine/server/engine/enclave_manager" "github.com/kurtosis-tech/kurtosis/engine/server/engine/server" minimal_grpc_server "github.com/kurtosis-tech/minimal-grpc-server/golang/server" @@ -135,7 +135,8 @@ func runMain() error { } // TODO: replace with persistent client so that we can get logs even after enclave is stopped - logsDatabaseClient := kurtosis_backend.NewKurtosisBackendLogsDatabaseClient(kurtosisBackend) + logsDatabaseClient := persistent_volume.NewPersistentVolumeLogsDatabaseClient(kurtosisBackend) + //logsDatabaseClient := kurtosis_backend.NewKurtosisBackendLogsDatabaseClient(kurtosisBackend) engineServerService := server.NewEngineServerService( serverArgs.ImageVersionTag, From ae75356f3685dbcec3e19e75e6db1404b5ef7f51 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 15 Aug 2023 22:56:38 -0400 Subject: [PATCH 11/40] adjust docker logs labels --- .../docker_labels_for_logs/docker_labels_for_logs.go | 1 - 1 file changed, 1 deletion(-) diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go index 7ba502a286..566a89d572 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go @@ -8,7 +8,6 @@ import ( // The following docker labels will be added into the logs stream // These are necessary for propagating information for log filtering and retrieval through the logging pipeline var LogsDatabaseKurtosisTrackedDockerLabelsForIdentifyLogsStream = []*docker_label_key.DockerLabelKey{ - label_key_consts.GUIDDockerLabelKey, label_key_consts.ContainerTypeDockerLabelKey, label_key_consts.EnclaveUUIDDockerLabelKey, label_key_consts.EnclaveNameDockerLabelKey, From 59ba80e6d099023c06db9b57ae2374868c9e0a72 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Wed, 16 Aug 2023 09:06:07 -0400 Subject: [PATCH 12/40] rm log aggregator volume --- .../object_attributes_provider.go | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go index f9b5286478..b350062099 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go @@ -17,10 +17,6 @@ import ( const ( engineServerNamePrefix = "kurtosis-engine" logsAggregatorName = "kurtosis-logs-aggregator" - - //We always use the same name because we are going to have only one instance of this volume, - //so when the engine is restarted it mounts the same volume with the previous logs - logsAggregatorVolumeName = logsAggregatorName + "-vol" ) type DockerObjectAttributesProvider interface { @@ -31,7 +27,6 @@ type DockerObjectAttributesProvider interface { ) (DockerObjectAttributes, error) ForEnclave(enclaveUuid enclave.EnclaveUUID) (DockerEnclaveObjectAttributesProvider, error) ForLogsAggregator() (DockerObjectAttributes, error) - ForLogsAggregatorVolume() (DockerObjectAttributes, error) } func GetDockerObjectAttributesProvider() DockerObjectAttributesProvider { @@ -121,21 +116,3 @@ func (provider *dockerObjectAttributesProviderImpl) ForLogsAggregator() (DockerO } return objectAttributes, nil } - -func (provider *dockerObjectAttributesProviderImpl) ForLogsAggregatorVolume() (DockerObjectAttributes, error) { - name, err := docker_object_name.CreateNewDockerObjectName(logsAggregatorVolumeName) - if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred creating a Docker object name object from string '%v'", logsAggregatorVolumeName) - } - - labels := map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue{ - label_key_consts.VolumeTypeDockerLabelKey: label_value_consts.LogsAggregatorTypeDockerLabelValue, - } - - objectAttributes, err := newDockerObjectAttributesImpl(name, labels) - if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred while creating the ObjectAttributesImpl with the name '%s' and labels '%+v'", name, labels) - } - - return objectAttributes, nil -} From 0cdaff23ae6ab8d79eedd590f64803607cc00502 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Wed, 16 Aug 2023 09:08:18 -0400 Subject: [PATCH 13/40] add user service guid to label --- .../docker_labels_for_logs/docker_labels_for_logs.go | 1 + .../persistent_volume_logs_database_client.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go index 566a89d572..cecb9904cf 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go @@ -9,6 +9,7 @@ import ( // These are necessary for propagating information for log filtering and retrieval through the logging pipeline var LogsDatabaseKurtosisTrackedDockerLabelsForIdentifyLogsStream = []*docker_label_key.DockerLabelKey{ label_key_consts.ContainerTypeDockerLabelKey, + label_key_consts.UserServiceGUIDDockerLabelKey, label_key_consts.EnclaveUUIDDockerLabelKey, label_key_consts.EnclaveNameDockerLabelKey, } diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go index ccbc6836e1..f15e0e6fb1 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go @@ -56,6 +56,9 @@ func (client *persistentVolumeLogsDatabaseClient) StreamUserServiceLogs( context.CancelFunc, error, ) { + logrus.Debugf("ENCLAVE UUID: %v", enclaveUuid) + logrus.Debugf("USER SERVICE UUIDS: %v", userServiceUuids) + ctx, cancelCtxFunc := context.WithCancel(ctx) logsFile, err := os.Open(logsFilepath) From f0bf20951758ae99ad41c063cc473c38e9f27cab Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Wed, 16 Aug 2023 09:30:31 -0400 Subject: [PATCH 14/40] adjust filtering of logs --- .../persistent_volume_logs_database_client.go | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go index f15e0e6fb1..599b24c0fe 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go @@ -13,6 +13,7 @@ import ( "github.com/sirupsen/logrus" "io" "os" + "strings" "sync" ) @@ -24,7 +25,7 @@ const ( newlineRune = '\n' - serviceUUIDLogLabel = "container_id" + serviceUUIDLogLabel = "container_name" enclaveUUIDLogLabel = "com.kurtosistech.enclave-id" logLabel = "log" @@ -201,14 +202,27 @@ func streamServiceLogLines( streamErrChan <- stacktrace.NewError("An error retrieving the enclave uuid field from logs json file. This is a bug in Kurtosis.") return } - logServiceUuidStr, found := jsonLog[serviceUUIDLogLabel] + containerNameStr, found := jsonLog[serviceUUIDLogLabel] if !found { streamErrChan <- stacktrace.NewError("An error retrieving the enclave uuid field from logs json file. This is a bug in Kurtosis.") return } + logrus.Debugf("ENCLAVE UUID STR: %v", logEnclaveUuidStr) + logrus.Debugf("CONTAINER NAME STR: %v", containerNameStr) + logEnclaveUuid := enclave.EnclaveUUID(logEnclaveUuidStr) - logServiceUuid := service.ServiceUUID(logServiceUuidStr) + + var serviceUUID service.ServiceUUID + doesServiceMatch := false + for uuid := range userServiceUuids { + uuidStr := string(uuid) + if strings.Contains(containerNameStr, uuidStr) { + doesServiceMatch = true + serviceUUID = uuid + break + } + } // Then we filter by checking: // 1. if the log message is valid based on requested filters @@ -219,7 +233,9 @@ func streamServiceLogLines( streamErrChan <- stacktrace.Propagate(err, "An error occurred filtering log line '%+v' using filters '%+v'", logLine, conjunctiveLogLinesFiltersWithRegex) break } - isValidBasedOnEnclaveAndServiceUuid := (enclaveUuid == logEnclaveUuid) && (userServiceUuids[logServiceUuid]) + logrus.Debugf("ENCLAVE MATCHES: %v", enclaveUuid == logEnclaveUuid) + logrus.Debugf("SERVICE MATCHES: %v", doesServiceMatch) + isValidBasedOnEnclaveAndServiceUuid := (enclaveUuid == logEnclaveUuid) && doesServiceMatch shouldReturnLogLine := isValidBasedOnFilters && isValidBasedOnEnclaveAndServiceUuid if !shouldReturnLogLine { break @@ -228,7 +244,7 @@ func streamServiceLogLines( // send the log line logLines := []logline.LogLine{*logLine} userServicesLogLinesMap := map[service.ServiceUUID][]logline.LogLine{ - logServiceUuid: logLines, + serviceUUID: logLines, } logsByKurtosisUserServiceUuidChan <- userServicesLogLinesMap numLogsReturned++ From 77a8d6dac920ba7299125f274e4b8d96d1522eef Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Wed, 16 Aug 2023 09:39:59 -0400 Subject: [PATCH 15/40] add test script --- logs.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100755 logs.sh diff --git a/logs.sh b/logs.sh new file mode 100755 index 0000000000..932761eb2f --- /dev/null +++ b/logs.sh @@ -0,0 +1,8 @@ +#!/bin/zsh + +cd /Users/tewodrosmitiku/Desktop/kurtosis +kurtosis clean -a +kurtosis engine stop +./cli/cli/scripts/launch-cli.sh engine start --version=$1 +./cli/cli/scripts/launch-cli.sh run github.com/kurtosis-tech/mongodb-package + From f44dd22caaff384e931f1e2e23ef2f3f8526574d Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Wed, 16 Aug 2023 13:49:02 -0400 Subject: [PATCH 16/40] change log filepath --- .../implementations/vector/consts.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go index 83e6fd4d4c..1574eb8c68 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go @@ -25,8 +25,8 @@ const ( // To construct the filepath, we utilize vectors template syntax that allows us to reference fields in log events // https://vector.dev/docs/reference/configuration/template-syntax/ - //logsFilepath = "\"" + logsStorageDirpath + "/{{ container_name }}.json\"" - logsFilepath = "\"" + logsStorageDirpath + "logs.json\"" + logsFilepath = "\"" + logsStorageDirpath + "/{{ container_name }}.json\"" + //logsFilepath = "\"" + logsStorageDirpath + "logs.json\"" configFileTemplateName = "vectorConfigFileTemplate" configFileTemplate = ` From e77a7f06207cef5453ece402b354c865ee0ec0d2 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Wed, 16 Aug 2023 14:50:12 -0400 Subject: [PATCH 17/40] add label key for logs --- .../implementations/vector/consts.go | 3 +-- .../docker_labels_for_logs/docker_labels_for_logs.go | 1 + .../enclave_object_attributes_provider.go | 2 ++ .../label_key_consts/label_key_consts.go | 7 +++++++ .../persistent_volume_logs_database_client.go | 2 +- 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go index 1574eb8c68..ba8e38a453 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go @@ -25,8 +25,7 @@ const ( // To construct the filepath, we utilize vectors template syntax that allows us to reference fields in log events // https://vector.dev/docs/reference/configuration/template-syntax/ - logsFilepath = "\"" + logsStorageDirpath + "/{{ container_name }}.json\"" - //logsFilepath = "\"" + logsStorageDirpath + "logs.json\"" + logsFilepath = "\"" + logsStorageDirpath + "/{{ enclave-id }}-{{ container_name }}.json\"" configFileTemplateName = "vectorConfigFileTemplate" configFileTemplate = ` diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go index cecb9904cf..96e0f545df 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go @@ -10,6 +10,7 @@ import ( var LogsDatabaseKurtosisTrackedDockerLabelsForIdentifyLogsStream = []*docker_label_key.DockerLabelKey{ label_key_consts.ContainerTypeDockerLabelKey, label_key_consts.UserServiceGUIDDockerLabelKey, + label_key_consts.LogsEnclaveIDDockerLabelKey, label_key_consts.EnclaveUUIDDockerLabelKey, label_key_consts.EnclaveNameDockerLabelKey, } diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go index 2e117fd9c8..4ad277b314 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go @@ -112,6 +112,7 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) ForEnclaveNetwork(enc labels[label_key_consts.EnclaveCreationTimeLabelKey] = creationTimeLabelValue labels[label_key_consts.EnclaveNameDockerLabelKey] = enclaveNameLabelValue + labels[label_key_consts.LogsEnclaveIDDockerLabelKey] = provider.enclaveId objectAttributes, err := newDockerObjectAttributesImpl(name, labels) if err != nil { @@ -236,6 +237,7 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) ForUserServiceContain labels[label_key_consts.ContainerTypeDockerLabelKey] = label_value_consts.UserServiceContainerTypeDockerLabelValue labels[label_key_consts.PortSpecsDockerLabelKey] = serializedPortsSpec labels[label_key_consts.PrivateIPDockerLabelKey] = privateIpLabelValue + labels[label_key_consts.LogsEnclaveIDDockerLabelKey] = provider.enclaveId objectAttributes, err := newDockerObjectAttributesImpl(name, labels) if err != nil { diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go index f72c81c608..df80bdae78 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go @@ -29,6 +29,12 @@ const ( portSpecsLabelKeyStr = labelNamespaceStr + "ports" enclaveIdLabelKeyStr = labelNamespaceStr + "enclave-id" + + // We create a duplicate of the enclave id label key because: + // the logs aggregator (vector) needs the enclave id label to create the filepath where logs are stored in persistent volume + // but vectors template syntax can't interpret "com.kurtosistech.enclave-id" + logsEnclaveIdLabelKeyStr = "enclave-id" + // TODO deprecate this in favor of storing in DB enclaveNameLabelKeyStr = labelNamespaceStr + "enclave-name" @@ -53,6 +59,7 @@ var IDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(idLabelKeySt var GUIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(guidLabelKeyStr) var PortSpecsDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(portSpecsLabelKeyStr) var EnclaveUUIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(enclaveIdLabelKeyStr) +var LogsEnclaveIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(logsEnclaveIdLabelKeyStr) var EnclaveNameDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(enclaveNameLabelKeyStr) var EnclaveCreationTimeLabelKey = docker_label_key.MustCreateNewDockerLabelKey(enclaveCreationTime) var PrivateIPDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(privateIpAddrLabelKeyStr) diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go index 599b24c0fe..b47862a1d6 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go @@ -26,7 +26,7 @@ const ( newlineRune = '\n' serviceUUIDLogLabel = "container_name" - enclaveUUIDLogLabel = "com.kurtosistech.enclave-id" + enclaveUUIDLogLabel = "enclave-id" logLabel = "log" maxNumLogsToReturn = 200 From 14e4aab2e981e1d847f895f1824478882b1db14e Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 17 Aug 2023 09:17:59 -0400 Subject: [PATCH 18/40] iterate on log file structure --- .../implementations/vector/consts.go | 5 ++--- .../vector/vector_container_config_provider.go | 2 +- .../user_services_functions/start_user_services.go | 1 + .../docker_label_key/docker_label_key.go | 2 +- .../docker_labels_for_logs.go | 4 +--- .../enclave_object_attributes_provider.go | 6 +++--- .../label_key_consts/label_key_consts.go | 14 ++++++++------ .../persistent_volume_logs_database_client.go | 4 +++- 8 files changed, 20 insertions(+), 18 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go index ba8e38a453..270ceafe3e 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go @@ -21,11 +21,10 @@ const ( fileSinkId = "\"file\"" fileTypeId = "\"file\"" - // We store log files per-enclave, per-service + // We store log files in the volume per-enclave, per-service // To construct the filepath, we utilize vectors template syntax that allows us to reference fields in log events - // https://vector.dev/docs/reference/configuration/template-syntax/ - logsFilepath = "\"" + logsStorageDirpath + "/{{ enclave-id }}-{{ container_name }}.json\"" + logsFilepath = "\"" + logsStorageDirpath + "/{{ enclave_uuid }}/{{ service_uuid }}.json\"" configFileTemplateName = "vectorConfigFileTemplate" configFileTemplate = ` diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go index c843820de4..07c8c8ecdf 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go @@ -27,7 +27,7 @@ func (vector *vectorContainerConfigProvider) GetContainerArgs( containerLabels map[string]string, networkId string, ) (*docker_manager.CreateAndStartContainerArgs, error) { - // TODO: get volume name from arg pass in via objs attr provider + //j volumeMounts := map[string]string{ "kurtosis-logs-storage": logsStorageDirpath, } diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go index 69bf3795b1..56515b752d 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go @@ -595,6 +595,7 @@ func createStartServiceOperation( return nil, stacktrace.NewError("Expected to have a logs collector server address value to send the user service logs, but it is empty") } + logrus.Debugf("LOG COLLECTOR LABELS: %v", logsCollectorLabels) // The container will be configured to send the logs to the Fluentbit logs collector server fluentdLoggingDriverCnfg := docker_manager.NewFluentdLoggingDriver( logsCollectorAddress, diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key/docker_label_key.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key/docker_label_key.go index ff431eddf7..b07435fa16 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key/docker_label_key.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key/docker_label_key.go @@ -6,7 +6,7 @@ import ( ) const ( - dockerLabelKeyRegexStr = "^[a-z0-9-.]+$" + dockerLabelKeyRegexStr = "^[a-z0-9-._]+$" // It doesn't seem Docker actually has a label key length limit, but we implement one of our own for practicality maxLabelLength = 256 diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go index 96e0f545df..dc36e24f90 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go @@ -9,10 +9,8 @@ import ( // These are necessary for propagating information for log filtering and retrieval through the logging pipeline var LogsDatabaseKurtosisTrackedDockerLabelsForIdentifyLogsStream = []*docker_label_key.DockerLabelKey{ label_key_consts.ContainerTypeDockerLabelKey, - label_key_consts.UserServiceGUIDDockerLabelKey, label_key_consts.LogsEnclaveIDDockerLabelKey, - label_key_consts.EnclaveUUIDDockerLabelKey, - label_key_consts.EnclaveNameDockerLabelKey, + label_key_consts.LogsServiceIDDockerLabelKey, } // These are all the logs database Kurtosis tracked Docker Labels used diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go index 4ad277b314..87b7bbf297 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go @@ -112,7 +112,6 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) ForEnclaveNetwork(enc labels[label_key_consts.EnclaveCreationTimeLabelKey] = creationTimeLabelValue labels[label_key_consts.EnclaveNameDockerLabelKey] = enclaveNameLabelValue - labels[label_key_consts.LogsEnclaveIDDockerLabelKey] = provider.enclaveId objectAttributes, err := newDockerObjectAttributesImpl(name, labels) if err != nil { @@ -237,7 +236,6 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) ForUserServiceContain labels[label_key_consts.ContainerTypeDockerLabelKey] = label_value_consts.UserServiceContainerTypeDockerLabelValue labels[label_key_consts.PortSpecsDockerLabelKey] = serializedPortsSpec labels[label_key_consts.PrivateIPDockerLabelKey] = privateIpLabelValue - labels[label_key_consts.LogsEnclaveIDDockerLabelKey] = provider.enclaveId objectAttributes, err := newDockerObjectAttributesImpl(name, labels) if err != nil { @@ -469,7 +467,8 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getNameForUserService func (provider *dockerEnclaveObjectAttributesProviderImpl) getLabelsForEnclaveObject() map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue { return map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue{ - label_key_consts.EnclaveUUIDDockerLabelKey: provider.enclaveId, + label_key_consts.EnclaveUUIDDockerLabelKey: provider.enclaveId, + label_key_consts.LogsEnclaveIDDockerLabelKey: provider.enclaveId, } } @@ -480,6 +479,7 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getLabelsForEnclaveOb return nil, stacktrace.Propagate(err, "An error occurred creating a Docker label value from GUID string '%v'", guid) } labels[label_key_consts.GUIDDockerLabelKey] = guidLabelValue + labels[label_key_consts.LogsServiceIDDockerLabelKey] = guidLabelValue return labels, nil } diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go index df80bdae78..29cfec6522 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go @@ -30,17 +30,18 @@ const ( enclaveIdLabelKeyStr = labelNamespaceStr + "enclave-id" - // We create a duplicate of the enclave id label key because: - // the logs aggregator (vector) needs the enclave id label to create the filepath where logs are stored in persistent volume - // but vectors template syntax can't interpret "com.kurtosistech.enclave-id" - logsEnclaveIdLabelKeyStr = "enclave-id" - // TODO deprecate this in favor of storing in DB enclaveNameLabelKeyStr = labelNamespaceStr + "enclave-name" enclaveCreationTime = labelNamespaceStr + "enclave-creation-time" privateIpAddrLabelKeyStr = labelNamespaceStr + "private-ip" + + // We create a duplicate of the enclave id label key because: + // the logs aggregator (vector) needs the enclave id label to create the filepath where logs are stored in persistent volume + // but vectors template syntax can't interpret "com.kurtosistech.enclave-id" + logsEnclaveUuidLabelKeyStr = "enclave_uuid" + logsServiceUuidDockerLabelKey = "service_uuid" ) // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DO NOT CHANGE THESE VALUES !!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -59,8 +60,9 @@ var IDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(idLabelKeySt var GUIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(guidLabelKeyStr) var PortSpecsDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(portSpecsLabelKeyStr) var EnclaveUUIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(enclaveIdLabelKeyStr) -var LogsEnclaveIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(logsEnclaveIdLabelKeyStr) var EnclaveNameDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(enclaveNameLabelKeyStr) var EnclaveCreationTimeLabelKey = docker_label_key.MustCreateNewDockerLabelKey(enclaveCreationTime) var PrivateIPDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(privateIpAddrLabelKeyStr) var UserServiceGUIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(userServiceGuidDockerLabelKeyStr) +var LogsEnclaveIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(logsEnclaveUuidLabelKeyStr) +var LogsServiceIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(logsServiceUuidDockerLabelKey) diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go index b47862a1d6..6ed732930e 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go @@ -21,7 +21,7 @@ const ( oneSenderAdded = 1 // Location of logs on the filesystem of the engine - logsFilepath = "/var/log/kurtosis/logs.json" + logsStorageDirpath = "/var/log/kurtosis/" newlineRune = '\n' @@ -251,3 +251,5 @@ func streamServiceLogLines( } } } + +func constructLogsFilepath(userSer) From 8e707ef8e90aea0af82394c6e0e85eb4fca40d3c Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 17 Aug 2023 09:49:35 -0400 Subject: [PATCH 19/40] pull logs per enclave per service --- .../persistent_volume_logs_database_client.go | 103 ++++++------------ 1 file changed, 31 insertions(+), 72 deletions(-) diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go index 6ed732930e..d27f7d9838 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" @@ -13,7 +14,6 @@ import ( "github.com/sirupsen/logrus" "io" "os" - "strings" "sync" ) @@ -22,12 +22,11 @@ const ( // Location of logs on the filesystem of the engine logsStorageDirpath = "/var/log/kurtosis/" + filetype = ".json" newlineRune = '\n' - serviceUUIDLogLabel = "container_name" - enclaveUUIDLogLabel = "enclave-id" - logLabel = "log" + logLabel = "log" maxNumLogsToReturn = 200 ) @@ -62,12 +61,6 @@ func (client *persistentVolumeLogsDatabaseClient) StreamUserServiceLogs( ctx, cancelCtxFunc := context.WithCancel(ctx) - logsFile, err := os.Open(logsFilepath) - if err != nil { - cancelCtxFunc() - return nil, nil, nil, stacktrace.Propagate(err, "An error occurred opening logs file.") - } - conjunctiveLogFiltersWithRegex, err := logline.NewConjunctiveLogFiltersWithRegex(conjunctiveLogLineFilters) if err != nil { cancelCtxFunc() @@ -81,28 +74,25 @@ func (client *persistentVolumeLogsDatabaseClient) StreamUserServiceLogs( logsByKurtosisUserServiceUuidChan := make(chan map[service.ServiceUUID][]logline.LogLine) wgSenders := &sync.WaitGroup{} - wgSenders.Add(oneSenderAdded) - go streamServiceLogLines( - ctx, - wgSenders, - logsByKurtosisUserServiceUuidChan, - streamErrChan, - enclaveUuid, - userServiceUuids, - logsFile, - conjunctiveLogFiltersWithRegex, - shouldFollowLogs, - ) + for serviceUuid := range userServiceUuids { + wgSenders.Add(oneSenderAdded) + go streamServiceLogLines( + ctx, + wgSenders, + logsByKurtosisUserServiceUuidChan, + streamErrChan, + enclaveUuid, + serviceUuid, + conjunctiveLogFiltersWithRegex, + shouldFollowLogs, + ) + } // this go routine handles the stream cancellation go func() { //wait for stream go routine to end wgSenders.Wait() - //close resources first - if err := logsFile.Close(); err != nil { - logrus.Warnf("An error occurred attempting to close the user service logs file after streaming:\n%v", err) - } close(logsByKurtosisUserServiceUuidChan) close(streamErrChan) @@ -149,30 +139,36 @@ func streamServiceLogLines( logsByKurtosisUserServiceUuidChan chan map[service.ServiceUUID][]logline.LogLine, streamErrChan chan error, enclaveUuid enclave.EnclaveUUID, - userServiceUuids map[service.ServiceUUID]bool, - logsFile io.Reader, + serviceUuid service.ServiceUUID, conjunctiveLogLinesFiltersWithRegex []logline.LogLineFilterWithRegex, shouldFollowLogs bool, ) { defer wgSenders.Done() + // logs are stored per enclave id, per service uuid, eg. /123440231421/54325342w2341.json + logsFilepath := fmt.Sprintf("%s%s/%s%s", logsStorageDirpath, string(enclaveUuid), string(serviceUuid), filetype) + logsFile, err := os.Open(logsFilepath) + if err != nil { + streamErrChan <- stacktrace.Propagate(err, "An error occurred opening the logs file for service '%v' in enclave '%v' at the following path: %v.", serviceUuid, enclaveUuid, logsFilepath) + return + } logsReader := bufio.NewReader(logsFile) numLogsReturned := 0 for shouldFollowLogs || numLogsReturned < maxNumLogsToReturn { select { case <-ctx.Done(): - logrus.Debugf("Context was canceled, stopping streaming service logs for services '%v'", userServiceUuids) + logrus.Debugf("Context was canceled, stopping streaming service logs for service '%v' in enclave '%v", serviceUuid, enclaveUuid) return default: jsonLogStr, err := logsReader.ReadString(newlineRune) if err != nil && errors.Is(err, io.EOF) { // exiting stream - logrus.Debugf("EOF error returned when reading logs for services '%v'", userServiceUuids) + logrus.Debugf("EOF error returned when reading logs for service '%v' in enclave '%v'", serviceUuid, enclaveUuid) return } if err != nil { - streamErrChan <- stacktrace.Propagate(err, "An error occurred reading the logs file '%v'", userServiceUuids) + streamErrChan <- stacktrace.Propagate(err, "An error occurred reading the logs file for service '%v' in enclave '%v' at the following path: %v.", serviceUuid, enclaveUuid, logsFilepath) return } @@ -184,7 +180,7 @@ func streamServiceLogLines( var jsonLog JsonLog err = json.Unmarshal([]byte(jsonLogStr), &jsonLog) if err != nil { - streamErrChan <- stacktrace.Propagate(err, "An error occurred reading the logs file '%v'", userServiceUuids) + streamErrChan <- stacktrace.Propagate(err, "An error occurred parsing the json logs file for service '%v' in enclave '%v' at the following path: %v.", serviceUuid, enclaveUuid, logsFilepath) return } @@ -196,47 +192,12 @@ func streamServiceLogLines( } logLine := logline.NewLogLine(logLineStr) - // We also extract enclave uuid and service uuid - logEnclaveUuidStr, found := jsonLog[enclaveUUIDLogLabel] - if !found { - streamErrChan <- stacktrace.NewError("An error retrieving the enclave uuid field from logs json file. This is a bug in Kurtosis.") - return - } - containerNameStr, found := jsonLog[serviceUUIDLogLabel] - if !found { - streamErrChan <- stacktrace.NewError("An error retrieving the enclave uuid field from logs json file. This is a bug in Kurtosis.") - return - } - - logrus.Debugf("ENCLAVE UUID STR: %v", logEnclaveUuidStr) - logrus.Debugf("CONTAINER NAME STR: %v", containerNameStr) - - logEnclaveUuid := enclave.EnclaveUUID(logEnclaveUuidStr) - - var serviceUUID service.ServiceUUID - doesServiceMatch := false - for uuid := range userServiceUuids { - uuidStr := string(uuid) - if strings.Contains(containerNameStr, uuidStr) { - doesServiceMatch = true - serviceUUID = uuid - break - } - } - - // Then we filter by checking: - // 1. if the log message is valid based on requested filters - // 2. if the log is associated with the requested enclave and one of the requested services - // we check this bc currently all logs are in one file - isValidBasedOnFilters, err := logLine.IsValidLogLineBaseOnFilters(conjunctiveLogLinesFiltersWithRegex) + // Then we filter by checking if the log message is valid based on requested filters + shouldReturnLogLine, err := logLine.IsValidLogLineBaseOnFilters(conjunctiveLogLinesFiltersWithRegex) if err != nil { streamErrChan <- stacktrace.Propagate(err, "An error occurred filtering log line '%+v' using filters '%+v'", logLine, conjunctiveLogLinesFiltersWithRegex) break } - logrus.Debugf("ENCLAVE MATCHES: %v", enclaveUuid == logEnclaveUuid) - logrus.Debugf("SERVICE MATCHES: %v", doesServiceMatch) - isValidBasedOnEnclaveAndServiceUuid := (enclaveUuid == logEnclaveUuid) && doesServiceMatch - shouldReturnLogLine := isValidBasedOnFilters && isValidBasedOnEnclaveAndServiceUuid if !shouldReturnLogLine { break } @@ -244,12 +205,10 @@ func streamServiceLogLines( // send the log line logLines := []logline.LogLine{*logLine} userServicesLogLinesMap := map[service.ServiceUUID][]logline.LogLine{ - serviceUUID: logLines, + serviceUuid: logLines, } logsByKurtosisUserServiceUuidChan <- userServicesLogLinesMap numLogsReturned++ } } } - -func constructLogsFilepath(userSer) From 4598a6ff041d134a909215296b8f27fa023f6d44 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 17 Aug 2023 09:57:55 -0400 Subject: [PATCH 20/40] clean up --- .../persistent_volume_logs_database_client.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go index d27f7d9838..f906cc4513 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go @@ -56,9 +56,6 @@ func (client *persistentVolumeLogsDatabaseClient) StreamUserServiceLogs( context.CancelFunc, error, ) { - logrus.Debugf("ENCLAVE UUID: %v", enclaveUuid) - logrus.Debugf("USER SERVICE UUIDS: %v", userServiceUuids) - ctx, cancelCtxFunc := context.WithCancel(ctx) conjunctiveLogFiltersWithRegex, err := logline.NewConjunctiveLogFiltersWithRegex(conjunctiveLogLineFilters) From 5174a6d980667ff58df55bde5cf67a39d4f43506 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 17 Aug 2023 10:44:40 -0400 Subject: [PATCH 21/40] add logs storage attrs provider --- .../engine_functions/create_engine.go | 16 +++++++++++++--- .../vector_container_config_provider.go | 5 +++-- .../vector_logs_aggregator_container.go | 10 +++++++++- .../start_user_services.go | 1 - .../label_value_consts/label_value_consts.go | 4 ++-- .../object_attributes_provider.go | 19 +++++++++++++++++++ 6 files changed, 46 insertions(+), 9 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go index 8cacefbdba..73aab82c7a 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go @@ -25,6 +25,7 @@ const ( frontendPortSpec = 9711 maxWaitForEngineAvailabilityRetries = 10 timeBetweenWaitForEngineAvailabilityRetries = 1 * time.Second + logsStorageDirpath = "/var/log/kurtosis" ) func CreateEngine( @@ -72,10 +73,19 @@ func CreateEngine( targetNetworkId := engineNetwork.GetId() logrus.Infof("Starting the centralized logs components...") + logsStorageAttrs, err := objAttrsProvider.ForLogsStorageVolume() + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred retrieving logs storage object attributes.") + } + logsStorageVolumeNameStr := logsStorageAttrs.GetName().GetString() + volumeLabelStrs := map[string]string{} + for labelKey, labelValue := range logsStorageAttrs.GetLabels() { + volumeLabelStrs[labelKey.GetString()] = labelValue.GetString() + } + // Creation of volume should be idempotent because the volume with persisted logs in it could already exist // Thus, we don't defer an undo volume if this operation fails - // TODO: retrieve name and labels from objsAttrProvider - if err = dockerManager.CreateVolume(ctx, "kurtosis-logs-storage", map[string]string{}); err != nil { + if err = dockerManager.CreateVolume(ctx, logsStorageVolumeNameStr, volumeLabelStrs); err != nil { return nil, stacktrace.Propagate(err, "An error occurred creating logs storage.") } @@ -142,7 +152,7 @@ func CreateEngine( } volumeMounts := map[string]string{ - "kurtosis-logs-storage": "/var/log/kurtosis/", + logsStorageVolumeNameStr: logsStorageDirpath, } if serverArgs.OnBastionHost { diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go index 07c8c8ecdf..503340cff9 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_container_config_provider.go @@ -26,10 +26,11 @@ func (vector *vectorContainerConfigProvider) GetContainerArgs( containerName string, containerLabels map[string]string, networkId string, + logsStorageVolumeName string, ) (*docker_manager.CreateAndStartContainerArgs, error) { - //j + volumeMounts := map[string]string{ - "kurtosis-logs-storage": logsStorageDirpath, + logsStorageVolumeName: logsStorageDirpath, } logsAggregatorConfigContentStr, err := vector.getConfigFileContent() diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_logs_aggregator_container.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_logs_aggregator_container.go index 0934e67f54..405ef596a0 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_logs_aggregator_container.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/vector_logs_aggregator_container.go @@ -32,7 +32,15 @@ func (vectorContainer *vectorLogsAggregatorContainer) CreateAndStart( for labelKey, labelValue := range logsAggregatorAttrs.GetLabels() { containerLabelStrs[labelKey.GetString()] = labelValue.GetString() } - createAndStartArgs, err := vectorContainerConfigProviderObj.GetContainerArgs(containerName, containerLabelStrs, targetNetworkId) + + // Engine handles creating the volume, but we need to mount the aggregator can send logs to logs storage + logsStorageAttrs, err := objAttrsProvider.ForLogsStorageVolume() + if err != nil { + return "", nil, nil, stacktrace.Propagate(err, "An error occurred getting the logs storage volume attributes.") + } + logsStorageVolNameStr := logsStorageAttrs.GetName().GetString() + + createAndStartArgs, err := vectorContainerConfigProviderObj.GetContainerArgs(containerName, containerLabelStrs, targetNetworkId, logsStorageVolNameStr) if err != nil { return "", nil, nil, err } diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go index 56515b752d..69bf3795b1 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go @@ -595,7 +595,6 @@ func createStartServiceOperation( return nil, stacktrace.NewError("Expected to have a logs collector server address value to send the user service logs, but it is empty") } - logrus.Debugf("LOG COLLECTOR LABELS: %v", logsCollectorLabels) // The container will be configured to send the logs to the Fluentbit logs collector server fluentdLoggingDriverCnfg := docker_manager.NewFluentdLoggingDriver( logsCollectorAddress, diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go index f1152af699..f44294d616 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go @@ -24,7 +24,7 @@ const ( enclaveDataVolumeTypeLabelValueStr = "enclave-data" filesArtifactExpansionVolumeTypeLabelValueStr = "files-artifacts-expansion" persistentDirectoryVolumeTypeLabelValueStr = "persistent-directory" - logsDatabaseVolumeTypeLabelValueStr = "logs-db" + logsStorageVolumeTypeLabelValueStr = "kurtosis-logs-storage" logsCollectorVolumeTypeLabelValueStr = "logs-collector-data" ) @@ -48,5 +48,5 @@ var FilesArtifactExpanderContainerTypeDockerLabelValue = docker_label_value.Must var EnclaveDataVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(enclaveDataVolumeTypeLabelValueStr) var FilesArtifactExpansionVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(filesArtifactExpansionVolumeTypeLabelValueStr) var PersistentDirectoryVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(persistentDirectoryVolumeTypeLabelValueStr) -var LogsDatabaseVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsDatabaseVolumeTypeLabelValueStr) +var LogsStorageVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsStorageVolumeTypeLabelValueStr) var LogsCollectorVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsCollectorVolumeTypeLabelValueStr) diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go index b350062099..8263316234 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go @@ -17,6 +17,7 @@ import ( const ( engineServerNamePrefix = "kurtosis-engine" logsAggregatorName = "kurtosis-logs-aggregator" + logsStorageVolumeName = "kurtosis-logs-storage" ) type DockerObjectAttributesProvider interface { @@ -27,6 +28,7 @@ type DockerObjectAttributesProvider interface { ) (DockerObjectAttributes, error) ForEnclave(enclaveUuid enclave.EnclaveUUID) (DockerEnclaveObjectAttributesProvider, error) ForLogsAggregator() (DockerObjectAttributes, error) + ForLogsStorageVolume() (DockerObjectAttributes, error) } func GetDockerObjectAttributesProvider() DockerObjectAttributesProvider { @@ -116,3 +118,20 @@ func (provider *dockerObjectAttributesProviderImpl) ForLogsAggregator() (DockerO } return objectAttributes, nil } + +func (provider *dockerObjectAttributesProviderImpl) ForLogsStorageVolume() (DockerObjectAttributes, error) { + name, err := docker_object_name.CreateNewDockerObjectName(logsStorageVolumeName) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating a Docker object name object from string '%v'", logsStorageVolumeName) + } + + labels := map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue{ + label_key_consts.VolumeTypeDockerLabelKey: label_value_consts.LogsStorageVolumeTypeDockerLabelValue, + } + + objectAttributes, err := newDockerObjectAttributesImpl(name, labels) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while creating the ObjectAttributesImpl with the name '%s' and labels '%+v'", name, labels) + } + return objectAttributes, nil +} From 091120b302e7618c88e90517a6b3f368928788b4 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 17 Aug 2023 10:58:34 -0400 Subject: [PATCH 22/40] more clean up --- .../docker_labels_for_logs/docker_labels_for_logs.go | 4 ++-- .../enclave_object_attributes_provider.go | 6 +++--- .../label_key_consts/label_key_consts.go | 10 +++++----- .../persistent_volume_logs_database_client.go | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go index dc36e24f90..a2d9c5fd74 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go @@ -9,8 +9,8 @@ import ( // These are necessary for propagating information for log filtering and retrieval through the logging pipeline var LogsDatabaseKurtosisTrackedDockerLabelsForIdentifyLogsStream = []*docker_label_key.DockerLabelKey{ label_key_consts.ContainerTypeDockerLabelKey, - label_key_consts.LogsEnclaveIDDockerLabelKey, - label_key_consts.LogsServiceIDDockerLabelKey, + label_key_consts.LogsEnclaveUUIDDockerLabelKey, + label_key_consts.LogsEnclaveUUIDDockerLabelKey, } // These are all the logs database Kurtosis tracked Docker Labels used diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go index 87b7bbf297..77e4e63b80 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go @@ -467,8 +467,8 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getNameForUserService func (provider *dockerEnclaveObjectAttributesProviderImpl) getLabelsForEnclaveObject() map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue { return map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue{ - label_key_consts.EnclaveUUIDDockerLabelKey: provider.enclaveId, - label_key_consts.LogsEnclaveIDDockerLabelKey: provider.enclaveId, + label_key_consts.EnclaveUUIDDockerLabelKey: provider.enclaveId, + label_key_consts.LogsEnclaveUUIDDockerLabelKey: provider.enclaveId, } } @@ -479,7 +479,7 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getLabelsForEnclaveOb return nil, stacktrace.Propagate(err, "An error occurred creating a Docker label value from GUID string '%v'", guid) } labels[label_key_consts.GUIDDockerLabelKey] = guidLabelValue - labels[label_key_consts.LogsServiceIDDockerLabelKey] = guidLabelValue + labels[label_key_consts.LogsServiceUUIDDockerLabelKey] = guidLabelValue return labels, nil } diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go index 29cfec6522..9339e33f28 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_key_consts/label_key_consts.go @@ -37,9 +37,9 @@ const ( privateIpAddrLabelKeyStr = labelNamespaceStr + "private-ip" - // We create a duplicate of the enclave id label key because: - // the logs aggregator (vector) needs the enclave id label to create the filepath where logs are stored in persistent volume - // but vectors template syntax can't interpret "com.kurtosistech.enclave-id" + // We create a duplicate of the enclave uuid and service uuid label key because: + // the logs aggregator (vector) needs the enclave uuid and service uuid label keys to create the filepath where logs are stored in persistent volume + // but vectors template syntax can't interpret the "com.kurtosistech." prefix, so we can't use the existing label keys logsEnclaveUuidLabelKeyStr = "enclave_uuid" logsServiceUuidDockerLabelKey = "service_uuid" ) @@ -64,5 +64,5 @@ var EnclaveNameDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(enc var EnclaveCreationTimeLabelKey = docker_label_key.MustCreateNewDockerLabelKey(enclaveCreationTime) var PrivateIPDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(privateIpAddrLabelKeyStr) var UserServiceGUIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(userServiceGuidDockerLabelKeyStr) -var LogsEnclaveIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(logsEnclaveUuidLabelKeyStr) -var LogsServiceIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(logsServiceUuidDockerLabelKey) +var LogsEnclaveUUIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(logsEnclaveUuidLabelKeyStr) +var LogsServiceUUIDDockerLabelKey = docker_label_key.MustCreateNewDockerLabelKey(logsServiceUuidDockerLabelKey) diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go index f906cc4513..8069e0b83b 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go @@ -169,9 +169,9 @@ func streamServiceLogLines( return } - // each logLineStr is of the following structure: {"label1": "...", "label2":"...", "log": "...", "timestamp", ... } - // eg. {"container_type":"api-container","enclave_id":"ffd1c0ba29e1a464c","container_id":"8f8558ba", - // "container_name":"/kurtosis-api--ffd", "log":"hi","timestamp":"2023-08-14T14:57:49Z"} + // each logLineStr is of the following structure: {"enclave_uuid": "...", "service_uuid":"...", "log": "...",.. "timestamp":"..."} + // eg. {"container_type":"api-container", "container_id":"8f8558ba", "container_name":"/kurtosis-api--ffd", + // "log":"hi","timestamp":"2023-08-14T14:57:49Z"} // First, we decode the line var jsonLog JsonLog From 75e35ab97fa4cab91e836f2cdb5d47608b1d0c36 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 17 Aug 2023 11:24:49 -0400 Subject: [PATCH 23/40] disable enclave and service id validation --- .../highlevel/enclave_id_arg/enclave_id_arg.go | 3 +-- .../service_identifier_arg/service_identifier_arg.go | 3 +-- cli/cli/commands/service/logs/logs.go | 7 ++----- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/cli/cli/command_framework/highlevel/enclave_id_arg/enclave_id_arg.go b/cli/cli/command_framework/highlevel/enclave_id_arg/enclave_id_arg.go index 0870a32b2d..4363bacdc4 100644 --- a/cli/cli/command_framework/highlevel/enclave_id_arg/enclave_id_arg.go +++ b/cli/cli/command_framework/highlevel/enclave_id_arg/enclave_id_arg.go @@ -40,8 +40,7 @@ func NewEnclaveIdentifierArg( } } -// TODO we added this constructor for allowing 'service logs' command to disable the validation for consuming logs from removed or stopped enclaves -// TODO after https://github.com/kurtosis-tech/kurtosis/issues/879 +// This constructor is for allowing 'service logs' command to disable the validation for consuming logs from removed or stopped enclaves func NewHistoricalEnclaveIdentifiersArgWithValidationDisabled( argKey string, isOptional bool, diff --git a/cli/cli/command_framework/highlevel/service_identifier_arg/service_identifier_arg.go b/cli/cli/command_framework/highlevel/service_identifier_arg/service_identifier_arg.go index 85764352f2..85c7274a8b 100644 --- a/cli/cli/command_framework/highlevel/service_identifier_arg/service_identifier_arg.go +++ b/cli/cli/command_framework/highlevel/service_identifier_arg/service_identifier_arg.go @@ -36,8 +36,7 @@ func NewServiceIdentifierArg( } } -// TODO we added this constructor for allowing 'service logs' command to disable the validation for consuming logs from removed or stopped enclaves -// TODO after https://github.com/kurtosis-tech/kurtosis/issues/879 is done +// This constructor is for allowing 'service logs' command to disable the validation for consuming logs from removed or stopped enclaves func NewHistoricalServiceIdentifierArgWithValidationDisabled( serviceIdentifierArgKey string, enclaveIdentifierArgKey string, diff --git a/cli/cli/commands/service/logs/logs.go b/cli/cli/commands/service/logs/logs.go index afeeff87fb..9fcbf630ba 100644 --- a/cli/cli/commands/service/logs/logs.go +++ b/cli/cli/commands/service/logs/logs.go @@ -103,15 +103,12 @@ var ServiceLogsCmd = &engine_consuming_kurtosis_command.EngineConsumingKurtosisC }, }, Args: []*args.ArgConfig{ - //TODO disabling enclaveID validation and serviceUUID validation for allowing consuming logs from removed or stopped enclaves - //TODO we should enable them when #879 is ready: https://github.com/kurtosis-tech/kurtosis/issues/879 - enclave_id_arg.NewEnclaveIdentifierArg( + enclave_id_arg.NewHistoricalEnclaveIdentifiersArgWithValidationDisabled( enclaveIdentifierArgKey, - engineClientCtxKey, isEnclaveIdArgOptional, isEnclaveIdArgGreedy, ), - service_identifier_arg.NewServiceIdentifierArg( + service_identifier_arg.NewHistoricalServiceIdentifierArgWithValidationDisabled( serviceIdentifierArgKey, enclaveIdentifierArgKey, isServiceIdentifierArgGreedy, From edae14d5aa3773ee31f23aff06726a25980d1ce0 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 17 Aug 2023 11:44:18 -0400 Subject: [PATCH 24/40] lint --- .../engine_functions/create_engine.go | 8 ++++---- .../implementations/vector/consts.go | 2 +- .../docker_labels_for_logs/docker_labels_for_logs.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go index 73aab82c7a..f1425e74c1 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go @@ -25,7 +25,7 @@ const ( frontendPortSpec = 9711 maxWaitForEngineAvailabilityRetries = 10 timeBetweenWaitForEngineAvailabilityRetries = 1 * time.Second - logsStorageDirpath = "/var/log/kurtosis" + logsStorageDirpath = "/var/log/kurtosis/" ) func CreateEngine( @@ -77,7 +77,7 @@ func CreateEngine( if err != nil { return nil, stacktrace.Propagate(err, "An error occurred retrieving logs storage object attributes.") } - logsStorageVolumeNameStr := logsStorageAttrs.GetName().GetString() + logsStorageVolNameStr := logsStorageAttrs.GetName().GetString() volumeLabelStrs := map[string]string{} for labelKey, labelValue := range logsStorageAttrs.GetLabels() { volumeLabelStrs[labelKey.GetString()] = labelValue.GetString() @@ -85,7 +85,7 @@ func CreateEngine( // Creation of volume should be idempotent because the volume with persisted logs in it could already exist // Thus, we don't defer an undo volume if this operation fails - if err = dockerManager.CreateVolume(ctx, logsStorageVolumeNameStr, volumeLabelStrs); err != nil { + if err = dockerManager.CreateVolume(ctx, logsStorageVolNameStr, volumeLabelStrs); err != nil { return nil, stacktrace.Propagate(err, "An error occurred creating logs storage.") } @@ -152,7 +152,7 @@ func CreateEngine( } volumeMounts := map[string]string{ - logsStorageVolumeNameStr: logsStorageDirpath, + logsStorageVolNameStr: logsStorageDirpath, } if serverArgs.OnBastionHost { diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go index 270ceafe3e..39db477eff 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go @@ -24,7 +24,7 @@ const ( // We store log files in the volume per-enclave, per-service // To construct the filepath, we utilize vectors template syntax that allows us to reference fields in log events // https://vector.dev/docs/reference/configuration/template-syntax/ - logsFilepath = "\"" + logsStorageDirpath + "/{{ enclave_uuid }}/{{ service_uuid }}.json\"" + logsFilepath = "\"" + logsStorageDirpath + "{{ enclave_uuid }}/{{ service_uuid }}.json\"" configFileTemplateName = "vectorConfigFileTemplate" configFileTemplate = ` diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go index a2d9c5fd74..dd0537cf7a 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_labels_for_logs/docker_labels_for_logs.go @@ -10,7 +10,7 @@ import ( var LogsDatabaseKurtosisTrackedDockerLabelsForIdentifyLogsStream = []*docker_label_key.DockerLabelKey{ label_key_consts.ContainerTypeDockerLabelKey, label_key_consts.LogsEnclaveUUIDDockerLabelKey, - label_key_consts.LogsEnclaveUUIDDockerLabelKey, + label_key_consts.LogsServiceUUIDDockerLabelKey, } // These are all the logs database Kurtosis tracked Docker Labels used From a45713352c64e51c3d2d71f343313899bc782b4c Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 18 Aug 2023 09:30:27 -0400 Subject: [PATCH 25/40] block --- .../logs_aggregator_functions/implementations/vector/consts.go | 1 + 1 file changed, 1 insertion(+) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go index 39db477eff..454d23a2b9 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go @@ -37,6 +37,7 @@ type = {{ .Sink.Type }} inputs = {{ .Sink.Inputs }} path = {{ .Sink.Filepath }} encoding.codec = "json" +buffer.when_full = "block" ` ////////////////////////--FINISH--VECTOR CONFIGURATION SECTION--///////////////////////////// ) From 7a0e934c133c0782443f7fd759a4498686f9fe90 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 18 Aug 2023 09:37:29 -0400 Subject: [PATCH 26/40] rm fluentd driver temporarily from apic --- ...cker_kurtosis_backend_api_container_functions.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go index fcb1d7bf06..0ee30669d9 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go @@ -4,7 +4,6 @@ import ( "context" "github.com/docker/go-connections/nat" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types" @@ -153,10 +152,10 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer( if err != nil { return nil, stacktrace.Propagate(err, "An error occurred retrieving the log collector address.") } - fluentdLoggingDriverCnfg := docker_manager.NewFluentdLoggingDriver( - logCollectorAddr, - logs_collector_functions.GetKurtosisTrackedLogsCollectorLabels(), - ) + //fluentdLoggingDriverCnfg := docker_manager.NewFluentdLoggingDriver( + // logCollectorAddr, + // logs_collector_functions.GetKurtosisTrackedLogsCollectorLabels(), + //) createAndStartArgs := docker_manager.NewCreateAndStartContainerArgsBuilder( image, @@ -174,8 +173,8 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer( ipAddr, ).WithLabels( labelStrs, - ).WithLoggingDriver( - fluentdLoggingDriverCnfg, + //).WithLoggingDriver( + // fluentdLoggingDriverCnfg, ).Build() if err = backend.dockerManager.FetchImage(ctx, image); err != nil { From 3c10ee8b9e167700cca1dfae14fde0d290b6e029 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 18 Aug 2023 10:18:24 -0400 Subject: [PATCH 27/40] add waits for logs to tests --- ...ocker_kurtosis_backend_api_container_functions.go | 12 +----------- .../testsuite/search_logs_test/search_logs_test.go | 3 +++ .../testsuite/stream_logs_test/stream_logs_test.go | 3 +++ 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go index 0ee30669d9..906740e6d4 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go @@ -147,15 +147,7 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer( labelStrs[labelKey.GetString()] = labelValue.GetString() } - //The APIContainer will be configured to send the logs to the Fluentbit logs collector server - logCollectorAddr, err := enclaveLogsCollector.GetEnclaveNetworkAddressString() - if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred retrieving the log collector address.") - } - //fluentdLoggingDriverCnfg := docker_manager.NewFluentdLoggingDriver( - // logCollectorAddr, - // logs_collector_functions.GetKurtosisTrackedLogsCollectorLabels(), - //) + // TODO: configure the APIContainer to send the logs to the Fluentbit logs collector server createAndStartArgs := docker_manager.NewCreateAndStartContainerArgsBuilder( image, @@ -173,8 +165,6 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer( ipAddr, ).WithLabels( labelStrs, - //).WithLoggingDriver( - // fluentdLoggingDriverCnfg, ).Build() if err = backend.dockerManager.FetchImage(ctx, image); err != nil { diff --git a/internal_testsuites/golang/testsuite/search_logs_test/search_logs_test.go b/internal_testsuites/golang/testsuite/search_logs_test/search_logs_test.go index 6152240c7f..a32a77b933 100644 --- a/internal_testsuites/golang/testsuite/search_logs_test/search_logs_test.go +++ b/internal_testsuites/golang/testsuite/search_logs_test/search_logs_test.go @@ -37,6 +37,8 @@ const ( logLine2 = "Starting feature 'enclave pool'" logLine3 = "Starting feature 'enclave pool with size 2'" logLine4 = "The data have being loaded" + + secondsToWaitForLogs = 1 * time.Second ) var ( @@ -118,6 +120,7 @@ func TestSearchLogs(t *testing.T) { require.NoError(t, err, "An error occurred adding services with log lines '%+v'", logLinesByService) require.Equal(t, len(logLinesByService), len(serviceList)) + time.Sleep(secondsToWaitForLogs) // ------------------------------------- TEST RUN ------------------------------------------------- enclaveUuid := enclaveCtx.GetEnclaveUuid() diff --git a/internal_testsuites/golang/testsuite/stream_logs_test/stream_logs_test.go b/internal_testsuites/golang/testsuite/stream_logs_test/stream_logs_test.go index 282290c46f..5a87e38930 100644 --- a/internal_testsuites/golang/testsuite/stream_logs_test/stream_logs_test.go +++ b/internal_testsuites/golang/testsuite/stream_logs_test/stream_logs_test.go @@ -31,6 +31,8 @@ const ( secondLogLine = "test" thirdLogLine = "running" lastLogLine = "successfully" + + secondsToWaitForLogs = 1 * time.Second ) var ( @@ -76,6 +78,7 @@ func TestStreamLogs(t *testing.T) { serviceList, err := test_helpers.AddServicesWithLogLines(ctx, enclaveCtx, logLinesByService) require.NoError(t, err, "An error occurred adding the datastore service") + time.Sleep(secondsToWaitForLogs) // ------------------------------------- TEST RUN ---------------------------------------------- enclaveUuid := enclaveCtx.GetEnclaveUuid() From 6b351ffd2c7a9c99a18479eb99b8851a549b01c9 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 18 Aug 2023 10:58:57 -0400 Subject: [PATCH 28/40] adjust circleci tests --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b6a8806002..f198e0f515 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -669,9 +669,9 @@ jobs: false fi - run: - name: "Verify Kurtosis cleaned up all its volumes" + name: "Verify Kurtosis cleaned up all its volumes (except for the log storage volume, so we can persiste enclaves)" command: | - if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l ) -eq 0 ]; then + if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l ) -eq 1 ]; then docker volume ls false fi @@ -810,9 +810,9 @@ jobs: false fi - run: - name: "Verify Kurtosis cleaned up all its volumes" + name: "Verify Kurtosis cleaned up all its volumes (except for the logs storage volume)" command: | - if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l) -eq 0 ]; then + if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l) -eq 1 ]; then docker volume ls false fi From cbc4e9009721b9e2400f8e231fcc8efd4dcdeec3 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 18 Aug 2023 11:37:20 -0400 Subject: [PATCH 29/40] add waits to ts log tests --- .../src/testsuite/search_logs_test/search_logs.test.ts | 4 ++++ .../src/testsuite/stream_log_test/stream_logs.test.ts | 4 ++++ internal_testsuites/typescript/tsconfig.json | 8 +++++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/internal_testsuites/typescript/src/testsuite/search_logs_test/search_logs.test.ts b/internal_testsuites/typescript/src/testsuite/search_logs_test/search_logs.test.ts index 9967cc7e83..37f90ff6f8 100644 --- a/internal_testsuites/typescript/src/testsuite/search_logs_test/search_logs.test.ts +++ b/internal_testsuites/typescript/src/testsuite/search_logs_test/search_logs.test.ts @@ -84,6 +84,9 @@ const SHOULD_FOLLOW_LOGS_VALUES_BY_REQUEST: boolean[] = [ SHOULD_NOT_FOLLOW_LOGS, ] +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const MILLISECONDS_TO_WAIT_FOR_LOGS = 1000 + jest.setTimeout(180000); test("Test Search Logs", TestSearchLogs); @@ -113,6 +116,7 @@ async function TestSearchLogs() { throw new Error(`Expected number of added services '${LOG_LINES_BY_SERVICE.size}', but the actual number of added services is '${serviceList.size}'`); } + await sleep(MILLISECONDS_TO_WAIT_FOR_LOGS); // ------------------------------------- TEST RUN ---------------------------------------------- const newKurtosisContextResult = await KurtosisContext.newKurtosisContextFromLocalEngine(); diff --git a/internal_testsuites/typescript/src/testsuite/stream_log_test/stream_logs.test.ts b/internal_testsuites/typescript/src/testsuite/stream_log_test/stream_logs.test.ts index 000078e3e5..32ad32f878 100644 --- a/internal_testsuites/typescript/src/testsuite/stream_log_test/stream_logs.test.ts +++ b/internal_testsuites/typescript/src/testsuite/stream_log_test/stream_logs.test.ts @@ -40,6 +40,9 @@ const LOG_LINES_BY_SERVICE = new Map([ [EXAMPLE_SERVICE_NAME, EXAMPLE_SERVICE_LOG_LINES], ]) +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const MILLISECONDS_TO_WAIT_FOR_LOGS = 1000 + class ServiceLogsRequestInfoAndExpectedResults { readonly requestedEnclaveUUID: EnclaveUUID; readonly requestedServiceUuids: Set; @@ -100,6 +103,7 @@ async function TestStreamLogs() { throw new Error(`Expected number of added services '${LOG_LINES_BY_SERVICE.size}', but the actual number of added services is '${serviceList.size}'`); } + await sleep(MILLISECONDS_TO_WAIT_FOR_LOGS); // ------------------------------------- TEST RUN ---------------------------------------------- const enclaveID: EnclaveUUID = enclaveContext.getEnclaveUuid(); diff --git a/internal_testsuites/typescript/tsconfig.json b/internal_testsuites/typescript/tsconfig.json index 182ebcf1f8..72f87125db 100644 --- a/internal_testsuites/typescript/tsconfig.json +++ b/internal_testsuites/typescript/tsconfig.json @@ -4,7 +4,13 @@ "outDir": "build", "downlevelIteration": true, "declaration": true, - "strict": true + "strict": true, + "lib": [ + "dom", + "es5", + "scripthost", + "es2015.promise" + ] }, "include": [ "src/**/*" From 23d569788b69c271c0d129ff8cd6f5b4f407b344 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 18 Aug 2023 12:40:46 -0400 Subject: [PATCH 30/40] add volume filesystem --- .../persistent_volume_logs_database_client.go | 10 +++-- .../persistent_volume/volume_filesystem.go | 38 +++++++++++++++++++ engine/server/engine/main.go | 6 +-- 3 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem.go diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go index 8069e0b83b..0186e353f2 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go @@ -13,7 +13,6 @@ import ( "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" "io" - "os" "sync" ) @@ -36,11 +35,14 @@ type JsonLog map[string]string // persistentVolumeLogsDatabaseClient pulls logs from a Docker volume the engine is mounted to type persistentVolumeLogsDatabaseClient struct { kurtosisBackend backend_interface.KurtosisBackend + + filesystem VolumeFilesystem } -func NewPersistentVolumeLogsDatabaseClient(kurtosisBackend backend_interface.KurtosisBackend) *persistentVolumeLogsDatabaseClient { +func NewPersistentVolumeLogsDatabaseClient(kurtosisBackend backend_interface.KurtosisBackend, filesystem VolumeFilesystem) *persistentVolumeLogsDatabaseClient { return &persistentVolumeLogsDatabaseClient{ kurtosisBackend: kurtosisBackend, + filesystem: filesystem, } } @@ -75,6 +77,7 @@ func (client *persistentVolumeLogsDatabaseClient) StreamUserServiceLogs( wgSenders.Add(oneSenderAdded) go streamServiceLogLines( ctx, + client.filesystem, wgSenders, logsByKurtosisUserServiceUuidChan, streamErrChan, @@ -132,6 +135,7 @@ func (client *persistentVolumeLogsDatabaseClient) FilterExistingServiceUuids( // ==================================================================================================== func streamServiceLogLines( ctx context.Context, + fs VolumeFilesystem, wgSenders *sync.WaitGroup, logsByKurtosisUserServiceUuidChan chan map[service.ServiceUUID][]logline.LogLine, streamErrChan chan error, @@ -144,7 +148,7 @@ func streamServiceLogLines( // logs are stored per enclave id, per service uuid, eg. /123440231421/54325342w2341.json logsFilepath := fmt.Sprintf("%s%s/%s%s", logsStorageDirpath, string(enclaveUuid), string(serviceUuid), filetype) - logsFile, err := os.Open(logsFilepath) + logsFile, err := fs.Open(logsFilepath) if err != nil { streamErrChan <- stacktrace.Propagate(err, "An error occurred opening the logs file for service '%v' in enclave '%v' at the following path: %v.", serviceUuid, enclaveUuid, logsFilepath) return diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem.go new file mode 100644 index 0000000000..722450d97c --- /dev/null +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem.go @@ -0,0 +1,38 @@ +package persistent_volume + +import ( + "io" + "os" + "testing/fstest" +) + +// VolumeFilesystem interface is an abstraction of the disk filesystem +// primarily for the purpose of enabling unit testing PersistentVolumeLogsDatabaseClient +type VolumeFilesystem interface { + Open(name string) (VolumeFile, error) +} + +type VolumeFile interface { + io.Reader +} + +// OsVolumeFilesystem is an implementation of the filesystem using disk +type OsVolumeFilesystem struct{} + +func NewOsVolumeFilesystem() *OsVolumeFilesystem { + return &OsVolumeFilesystem{} +} + +func (fs *OsVolumeFilesystem) Open(name string) (VolumeFile, error) { return os.Open(name) } + +// MockedVolumeFilesystem is an implementation used for unit testing +type MockedVolumeFilesystem struct { + // we use an underlying map filesystem that's easy to mock file data with + mapFS fstest.MapFS +} + +func NewMockedVolumeFilesystem(fs fstest.MapFS) *MockedVolumeFilesystem { + return &MockedVolumeFilesystem{mapFS: fs} +} + +func (fs *MockedVolumeFilesystem) Open(name string) (VolumeFile, error) { return fs.mapFS.Open(name) } diff --git a/engine/server/engine/main.go b/engine/server/engine/main.go index 97ab63205f..36510ef539 100644 --- a/engine/server/engine/main.go +++ b/engine/server/engine/main.go @@ -134,9 +134,9 @@ func runMain() error { return stacktrace.Propagate(err, "Failed to create an enclave manager for backend type '%v' and config '%+v'", serverArgs.KurtosisBackendType, backendConfig) } - // TODO: replace with persistent client so that we can get logs even after enclave is stopped - logsDatabaseClient := persistent_volume.NewPersistentVolumeLogsDatabaseClient(kurtosisBackend) - //logsDatabaseClient := kurtosis_backend.NewKurtosisBackendLogsDatabaseClient(kurtosisBackend) + // osFs is a wrapper around disk + osFs := persistent_volume.NewOsVolumeFilesystem() + logsDatabaseClient := persistent_volume.NewPersistentVolumeLogsDatabaseClient(kurtosisBackend, osFs) engineServerService := server.NewEngineServerService( serverArgs.ImageVersionTag, From e1474c68e894d126d5e5b0bb806b6e9888234fcd Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 18 Aug 2023 14:36:00 -0400 Subject: [PATCH 31/40] add persistent volume unit test --- ...istent_volume_logs_database_client_test.go | 415 ++++++++++++++++++ 1 file changed, 415 insertions(+) create mode 100644 engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go new file mode 100644 index 0000000000..b085acc393 --- /dev/null +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go @@ -0,0 +1,415 @@ +package persistent_volume + +import ( + "context" + "fmt" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/logline" + "github.com/kurtosis-tech/stacktrace" + "github.com/stretchr/testify/require" + "strings" + "testing" + "testing/fstest" + "time" +) + +const ( + testEnclaveUuid = "test-enclave" + enclaveUuid = enclave.EnclaveUUID(testEnclaveUuid) + testUserService1Uuid = "test-user-service-1" + testUserService2Uuid = "test-user-service-2" + testUserService3Uuid = "test-user-service-3" + + logLine1 = "{\"log\":\"Starting feature 'centralized logs'\"}" + logLine2 = "{\"log\":\"Starting feature 'network partitioning'\"}" + logLine3 = "{\"log\":\"Starting feature 'network soft partitioning'\"}" + logLine4 = "{\"log\":\"Starting feature 'files storage'\"}" + logLine5 = "{\"log\":\"Starting feature 'files manager'\"}" + logLine6 = "{\"log\":\"The enclave was created\"}" + logLine7 = "{\"log\":\"User service started\"}" + logLine8 = "{\"log\":\"The data have being loaded\"}" + + firstFilterText = "feature" + secondFilterText = "Files" + notFoundedFilterText = "it shouldn't be found in the log lines" + firstMatchRegexFilterStr = "Starting.*partitioning'" + secondMatchRegexFilterStr = "[S].*manager" + + testTimeOut = 2 * time.Second + followLogs = true + doNotFollowLogs = false +) + +func TestStreamUserServiceLogs_WithFilters(t *testing.T) { + expectedServiceAmountLogLinesByServiceUuid := map[service.ServiceUUID]int{ + testUserService1Uuid: 2, + testUserService2Uuid: 2, + testUserService3Uuid: 2, + } + + firstTextFilter := logline.NewDoesContainTextLogLineFilter(firstFilterText) + secondTextFilter := logline.NewDoesNotContainTextLogLineFilter(secondFilterText) + regexFilter := logline.NewDoesContainMatchRegexLogLineFilter(firstMatchRegexFilterStr) + + logLinesFilters := []logline.LogLineFilter{ + *firstTextFilter, + *secondTextFilter, + *regexFilter, + } + + expectedFirstLogLine := logLine2 + + underlyingFs := createUnderlyingMapFilesystem() + + receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( + t, + logLinesFilters, + expectedServiceAmountLogLinesByServiceUuid, + doNotFollowLogs, + underlyingFs, + ) + + t.Logf("RECEIVER USER SERVICE LOGS BY UUID: %v", receivedUserServiceLogsByUuid) + for serviceUuid, serviceLogLines := range receivedUserServiceLogsByUuid { + expectedAmountLogLines, found := expectedServiceAmountLogLinesByServiceUuid[serviceUuid] + require.True(t, found) + require.Equal(t, expectedAmountLogLines, len(serviceLogLines)) + require.Equal(t, expectedFirstLogLine, serviceLogLines[0].GetContent()) + } + + require.NoError(t, testEvaluationErr) +} + +func TestStreamUserServiceLogs_WithFiltersAndWithFollowLogs(t *testing.T) { + + expectedServiceAmountLogLinesByServiceUuid := map[service.ServiceUUID]int{ + testUserService1Uuid: 1, + testUserService2Uuid: 1, + testUserService3Uuid: 1, + } + + firstTextFilter := logline.NewDoesContainTextLogLineFilter(secondFilterText) + regexFilter := logline.NewDoesContainMatchRegexLogLineFilter(secondMatchRegexFilterStr) + + logLinesFilters := []logline.LogLineFilter{ + *firstTextFilter, + *regexFilter, + } + + expectedFirstLogLine := logLine5 + + underlyingFs := createUnderlyingMapFilesystem() + + receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( + t, + logLinesFilters, + expectedServiceAmountLogLinesByServiceUuid, + followLogs, + underlyingFs, + ) + + for serviceUuid, serviceLogLines := range receivedUserServiceLogsByUuid { + expectedAmountLogLines, found := expectedServiceAmountLogLinesByServiceUuid[serviceUuid] + require.True(t, found) + require.Equal(t, expectedAmountLogLines, len(serviceLogLines)) + require.Equal(t, expectedFirstLogLine, serviceLogLines[0].GetContent()) + } + + require.NoError(t, testEvaluationErr) +} + +func TestStreamUserServiceLogs_FilteredAllLogLines(t *testing.T) { + + expectedServiceAmountLogLinesByServiceUuid := map[service.ServiceUUID]int{ + testUserService1Uuid: 0, + testUserService2Uuid: 0, + testUserService3Uuid: 0, + } + + firstTextFilter := logline.NewDoesContainTextLogLineFilter(notFoundedFilterText) + + logLinesFilters := []logline.LogLineFilter{ + *firstTextFilter, + } + + underlyingFs := createUnderlyingMapFilesystem() + + receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( + t, + logLinesFilters, + expectedServiceAmountLogLinesByServiceUuid, + doNotFollowLogs, + underlyingFs, + ) + + for serviceUuid, serviceLogLines := range receivedUserServiceLogsByUuid { + expectedAmountLogLines, found := expectedServiceAmountLogLinesByServiceUuid[serviceUuid] + require.True(t, found) + require.Equal(t, expectedAmountLogLines, len(serviceLogLines)) + } + + require.NoError(t, testEvaluationErr) +} + +func TestStreamUserServiceLogs_FilteredAllLogLinesWithFollowLogs(t *testing.T) { + expectedServiceAmountLogLinesByServiceUuid := map[service.ServiceUUID]int{ + testUserService1Uuid: 0, + testUserService2Uuid: 0, + testUserService3Uuid: 0, + } + + firstTextFilter := logline.NewDoesContainTextLogLineFilter(notFoundedFilterText) + + logLinesFilters := []logline.LogLineFilter{ + *firstTextFilter, + } + + underlyingFs := createUnderlyingMapFilesystem() + + receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( + t, + logLinesFilters, + expectedServiceAmountLogLinesByServiceUuid, + followLogs, + underlyingFs, + ) + + for serviceUuid, serviceLogLines := range receivedUserServiceLogsByUuid { + expectedAmountLogLines, found := expectedServiceAmountLogLinesByServiceUuid[serviceUuid] + require.True(t, found) + require.Equal(t, expectedAmountLogLines, len(serviceLogLines)) + } + + require.NoError(t, testEvaluationErr) + +} + +func TestStreamUserServiceLogs_NoLogsFromPersistentVolume(t *testing.T) { + expectedServiceAmountLogLinesByServiceUuid := map[service.ServiceUUID]int{ + testUserService1Uuid: 0, + testUserService2Uuid: 0, + testUserService3Uuid: 0, + } + + firstTextFilter := logline.NewDoesContainTextLogLineFilter(notFoundedFilterText) + + logLinesFilters := []logline.LogLineFilter{ + *firstTextFilter, + } + + underlyingFs := fstest.MapFS{} + + receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( + t, + logLinesFilters, + expectedServiceAmountLogLinesByServiceUuid, + followLogs, + underlyingFs, + ) + + for serviceUuid, serviceLogLines := range receivedUserServiceLogsByUuid { + expectedAmountLogLines, found := expectedServiceAmountLogLinesByServiceUuid[serviceUuid] + require.True(t, found) + require.Equal(t, expectedAmountLogLines, len(serviceLogLines)) + } + + require.NoError(t, testEvaluationErr) +} + +func TestStreamUserServiceLogs_ThousandsOfLogLinesSuccessfulExecution(t *testing.T) { + expectedAmountLogLines := 100000 + + expectedServiceAmountLogLinesByServiceUuid := map[service.ServiceUUID]int{ + testUserService1Uuid: expectedAmountLogLines, + } + + emptyFilters := []logline.LogLineFilter{} + + expectedFirstLogLine := logLine1 + + logLines := []string{} + + for i := 0; i <= expectedAmountLogLines; i++ { + logLines = append(logLines, logLine1) + } + + logLinesStr := strings.Join(logLines, "\n") + + file1 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpath, string(enclaveUuid), testUserService1Uuid, filetype) + + underlyingFs := fstest.MapFS{ + file1: { + Data: []byte(logLinesStr), + }, + } + + receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( + t, + emptyFilters, + expectedServiceAmountLogLinesByServiceUuid, + doNotFollowLogs, + underlyingFs, + ) + + for serviceUuid, serviceLogLines := range receivedUserServiceLogsByUuid { + expectedAmountLogLines, found := expectedServiceAmountLogLinesByServiceUuid[serviceUuid] + require.True(t, found) + require.Equal(t, expectedAmountLogLines, len(serviceLogLines)) + require.Equal(t, expectedFirstLogLine, serviceLogLines[0].GetContent()) + } + + require.NoError(t, testEvaluationErr) +} + +func TestStreamUserServiceLogs_EmptyLogLines(t *testing.T) { + expectedAmountLogLines := 0 + + expectedServiceAmountLogLinesByServiceUuid := map[service.ServiceUUID]int{ + testUserService1Uuid: expectedAmountLogLines, + } + + emptyFilters := []logline.LogLineFilter{} + + logLinesStr := "" + + file1 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpath, string(enclaveUuid), testUserService1Uuid, filetype) + + underlyingFs := fstest.MapFS{ + file1: { + Data: []byte(logLinesStr), + }, + } + + receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( + t, + emptyFilters, + expectedServiceAmountLogLinesByServiceUuid, + doNotFollowLogs, + underlyingFs, + ) + + for serviceUuid, serviceLogLines := range receivedUserServiceLogsByUuid { + expectedAmountLogLines, found := expectedServiceAmountLogLinesByServiceUuid[serviceUuid] + require.True(t, found) + require.Equal(t, expectedAmountLogLines, len(serviceLogLines)) + } + + require.NoError(t, testEvaluationErr) +} + +// ==================================================================================================== +// +// Private helper functions +// +// ==================================================================================================== +func executeStreamCallAndGetReceivedServiceLogLines( + t *testing.T, + logLinesFilters []logline.LogLineFilter, + expectedServiceAmountLogLinesByServiceUuid map[service.ServiceUUID]int, + shouldFollowLogs bool, + underlyingFs fstest.MapFS, +) (map[service.ServiceUUID][]logline.LogLine, error) { + t.Log("EXECUTING STREAM CALL AND GETTING RECEIVED SERVICE LOG LINE") + ctx := context.Background() + + userServiceUuids := map[service.ServiceUUID]bool{ + testUserService1Uuid: true, + testUserService2Uuid: true, + testUserService3Uuid: true, + } + + receivedServiceLogsByUuid := map[service.ServiceUUID][]logline.LogLine{} + + for serviceUuid := range expectedServiceAmountLogLinesByServiceUuid { + receivedServiceLogsByUuid[serviceUuid] = []logline.LogLine{} + } + + kurtosisBackend := backend_interface.NewMockKurtosisBackend(t) + mockedFs := NewMockedVolumeFilesystem(underlyingFs) + + logsDatabaseClient := NewPersistentVolumeLogsDatabaseClient(kurtosisBackend, mockedFs) + t.Log("CALLING STREAM USER SERVICE LOGS") + userServiceLogsByUuidChan, errChan, receivedCancelCtxFunc, err := logsDatabaseClient.StreamUserServiceLogs(ctx, enclaveUuid, userServiceUuids, logLinesFilters, shouldFollowLogs) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting user service logs for UUIDs '%+v' using log line filters '%v' in enclave '%v'", userServiceUuids, logLinesFilters, enclaveUuid) + } + defer func() { + if receivedCancelCtxFunc != nil { + receivedCancelCtxFunc() + } + }() + + require.NotNil(t, userServiceLogsByUuidChan, "Received a nil user service logs channel, but a non-nil value was expected") + require.NotNil(t, errChan, "Received a nil error logs channel, but a non-nil value was expected") + t.Log("DID NOT RECEIVE NIL SERVICE LOGS CHANNEL") + + shouldReceiveStream := true + for shouldReceiveStream { + t.Log("RECEIVING STREAM") + select { + case <-time.Tick(testTimeOut): + t.Log("TIMEOUT ERROR") + return nil, stacktrace.NewError("Receiving stream logs in the test has reached the '%v' time out", testTimeOut) + case streamErr, isChanOpen := <-errChan: + t.Log("RECEIVING STREAM ERROR") + if !isChanOpen { + shouldReceiveStream = false + break + } + + t.Logf("RECEIVING STREAM ERR: %v", streamErr) + case userServiceLogsByUuid, isChanOpen := <-userServiceLogsByUuidChan: + t.Log("RECEIVING LOGS") + if !isChanOpen { + shouldReceiveStream = false + break + } + + for serviceUuid, serviceLogLines := range userServiceLogsByUuid { + _, found := userServiceUuids[serviceUuid] + require.True(t, found) + + currentServiceLogLines := receivedServiceLogsByUuid[serviceUuid] + allServiceLogLines := append(currentServiceLogLines, serviceLogLines...) + receivedServiceLogsByUuid[serviceUuid] = allServiceLogLines + } + + for serviceUuid, expectedAmountLogLines := range expectedServiceAmountLogLinesByServiceUuid { + if len(receivedServiceLogsByUuid[serviceUuid]) == expectedAmountLogLines { + shouldReceiveStream = false + } else { + shouldReceiveStream = true + break + } + } + } + } + + return receivedServiceLogsByUuid, nil +} + +func createUnderlyingMapFilesystem() fstest.MapFS { + logLines := []string{logLine1, logLine2, logLine3, logLine4, logLine5, logLine6, logLine7, logLine8} + + logLinesStr := strings.Join(logLines, "\n") + + file1 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpath, string(enclaveUuid), testUserService1Uuid, filetype) + file2 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpath, string(enclaveUuid), testUserService2Uuid, filetype) + file3 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpath, string(enclaveUuid), testUserService3Uuid, filetype) + + mapFs := fstest.MapFS{ + file1: { + Data: []byte(logLinesStr), + }, + file2: { + Data: []byte(logLinesStr), + }, + file3: { + Data: []byte(logLinesStr), + }, + } + + return mapFs +} From d1f56a6160c7e5abc119a6d050aaf63af133a44d Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 18 Aug 2023 15:57:44 -0400 Subject: [PATCH 32/40] add unit tests for persistent volume client --- ...istent_volume_logs_database_client_test.go | 127 +++++++++++++----- .../persistent_volume/volume_filesystem.go | 22 ++- 2 files changed, 108 insertions(+), 41 deletions(-) diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go index b085acc393..50df3674d0 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go @@ -16,6 +16,9 @@ import ( ) const ( + // We use this storage path for tests because fstest.MapFS doesn't like forward slashes + logsStorageDirpathForTests = "var/log/kurtosis/" + testEnclaveUuid = "test-enclave" enclaveUuid = enclave.EnclaveUUID(testEnclaveUuid) testUserService1Uuid = "test-user-service-1" @@ -59,19 +62,25 @@ func TestStreamUserServiceLogs_WithFilters(t *testing.T) { *regexFilter, } - expectedFirstLogLine := logLine2 + expectedFirstLogLine := "Starting feature 'network partitioning'" + + underlyingFs := createFullUnderlyingMapFilesystem() - underlyingFs := createUnderlyingMapFilesystem() + userServiceUuids := map[service.ServiceUUID]bool{ + testUserService1Uuid: true, + testUserService2Uuid: true, + testUserService3Uuid: true, + } receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( t, logLinesFilters, + userServiceUuids, expectedServiceAmountLogLinesByServiceUuid, doNotFollowLogs, underlyingFs, ) - t.Logf("RECEIVER USER SERVICE LOGS BY UUID: %v", receivedUserServiceLogsByUuid) for serviceUuid, serviceLogLines := range receivedUserServiceLogsByUuid { expectedAmountLogLines, found := expectedServiceAmountLogLinesByServiceUuid[serviceUuid] require.True(t, found) @@ -98,13 +107,20 @@ func TestStreamUserServiceLogs_WithFiltersAndWithFollowLogs(t *testing.T) { *regexFilter, } - expectedFirstLogLine := logLine5 + expectedFirstLogLine := "Starting feature 'files manager'" + + underlyingFs := createFullUnderlyingMapFilesystem() - underlyingFs := createUnderlyingMapFilesystem() + userServiceUuids := map[service.ServiceUUID]bool{ + testUserService1Uuid: true, + testUserService2Uuid: true, + testUserService3Uuid: true, + } receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( t, logLinesFilters, + userServiceUuids, expectedServiceAmountLogLinesByServiceUuid, followLogs, underlyingFs, @@ -134,11 +150,18 @@ func TestStreamUserServiceLogs_FilteredAllLogLines(t *testing.T) { *firstTextFilter, } - underlyingFs := createUnderlyingMapFilesystem() + underlyingFs := createFullUnderlyingMapFilesystem() + + userServiceUuids := map[service.ServiceUUID]bool{ + testUserService1Uuid: true, + testUserService2Uuid: true, + testUserService3Uuid: true, + } receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( t, logLinesFilters, + userServiceUuids, expectedServiceAmountLogLinesByServiceUuid, doNotFollowLogs, underlyingFs, @@ -166,11 +189,18 @@ func TestStreamUserServiceLogs_FilteredAllLogLinesWithFollowLogs(t *testing.T) { *firstTextFilter, } - underlyingFs := createUnderlyingMapFilesystem() + underlyingFs := createFullUnderlyingMapFilesystem() + + userServiceUuids := map[service.ServiceUUID]bool{ + testUserService1Uuid: true, + testUserService2Uuid: true, + testUserService3Uuid: true, + } receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( t, logLinesFilters, + userServiceUuids, expectedServiceAmountLogLinesByServiceUuid, followLogs, underlyingFs, @@ -199,11 +229,18 @@ func TestStreamUserServiceLogs_NoLogsFromPersistentVolume(t *testing.T) { *firstTextFilter, } - underlyingFs := fstest.MapFS{} + underlyingFs := createEmptyUnderlyingMapFilesystem() + + userServiceUuids := map[service.ServiceUUID]bool{ + testUserService1Uuid: true, + testUserService2Uuid: true, + testUserService3Uuid: true, + } receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( t, logLinesFilters, + userServiceUuids, expectedServiceAmountLogLinesByServiceUuid, followLogs, underlyingFs, @@ -219,7 +256,7 @@ func TestStreamUserServiceLogs_NoLogsFromPersistentVolume(t *testing.T) { } func TestStreamUserServiceLogs_ThousandsOfLogLinesSuccessfulExecution(t *testing.T) { - expectedAmountLogLines := 100000 + expectedAmountLogLines := maxNumLogsToReturn expectedServiceAmountLogLinesByServiceUuid := map[service.ServiceUUID]int{ testUserService1Uuid: expectedAmountLogLines, @@ -227,7 +264,7 @@ func TestStreamUserServiceLogs_ThousandsOfLogLinesSuccessfulExecution(t *testing emptyFilters := []logline.LogLineFilter{} - expectedFirstLogLine := logLine1 + expectedFirstLogLine := "Starting feature 'centralized logs'" logLines := []string{} @@ -237,17 +274,22 @@ func TestStreamUserServiceLogs_ThousandsOfLogLinesSuccessfulExecution(t *testing logLinesStr := strings.Join(logLines, "\n") - file1 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpath, string(enclaveUuid), testUserService1Uuid, filetype) + file1 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpathForTests, string(enclaveUuid), testUserService1Uuid, filetype) - underlyingFs := fstest.MapFS{ + underlyingFs := &fstest.MapFS{ file1: { Data: []byte(logLinesStr), }, } + userServiceUuids := map[service.ServiceUUID]bool{ + testUserService1Uuid: true, + } + receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( t, emptyFilters, + userServiceUuids, expectedServiceAmountLogLinesByServiceUuid, doNotFollowLogs, underlyingFs, @@ -274,17 +316,22 @@ func TestStreamUserServiceLogs_EmptyLogLines(t *testing.T) { logLinesStr := "" - file1 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpath, string(enclaveUuid), testUserService1Uuid, filetype) + file1 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpathForTests, string(enclaveUuid), testUserService1Uuid, filetype) - underlyingFs := fstest.MapFS{ + underlyingFs := &fstest.MapFS{ file1: { Data: []byte(logLinesStr), }, } + userServiceUuids := map[service.ServiceUUID]bool{ + testUserService1Uuid: true, + } + receivedUserServiceLogsByUuid, testEvaluationErr := executeStreamCallAndGetReceivedServiceLogLines( t, emptyFilters, + userServiceUuids, expectedServiceAmountLogLinesByServiceUuid, doNotFollowLogs, underlyingFs, @@ -307,19 +354,13 @@ func TestStreamUserServiceLogs_EmptyLogLines(t *testing.T) { func executeStreamCallAndGetReceivedServiceLogLines( t *testing.T, logLinesFilters []logline.LogLineFilter, + userServiceUuids map[service.ServiceUUID]bool, expectedServiceAmountLogLinesByServiceUuid map[service.ServiceUUID]int, shouldFollowLogs bool, - underlyingFs fstest.MapFS, + underlyingFs *fstest.MapFS, ) (map[service.ServiceUUID][]logline.LogLine, error) { - t.Log("EXECUTING STREAM CALL AND GETTING RECEIVED SERVICE LOG LINE") ctx := context.Background() - userServiceUuids := map[service.ServiceUUID]bool{ - testUserService1Uuid: true, - testUserService2Uuid: true, - testUserService3Uuid: true, - } - receivedServiceLogsByUuid := map[service.ServiceUUID][]logline.LogLine{} for serviceUuid := range expectedServiceAmountLogLinesByServiceUuid { @@ -327,10 +368,10 @@ func executeStreamCallAndGetReceivedServiceLogLines( } kurtosisBackend := backend_interface.NewMockKurtosisBackend(t) - mockedFs := NewMockedVolumeFilesystem(underlyingFs) + mockedFs := NewMockedVolumeFilesystem(underlyingFs) logsDatabaseClient := NewPersistentVolumeLogsDatabaseClient(kurtosisBackend, mockedFs) - t.Log("CALLING STREAM USER SERVICE LOGS") + userServiceLogsByUuidChan, errChan, receivedCancelCtxFunc, err := logsDatabaseClient.StreamUserServiceLogs(ctx, enclaveUuid, userServiceUuids, logLinesFilters, shouldFollowLogs) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting user service logs for UUIDs '%+v' using log line filters '%v' in enclave '%v'", userServiceUuids, logLinesFilters, enclaveUuid) @@ -343,25 +384,19 @@ func executeStreamCallAndGetReceivedServiceLogLines( require.NotNil(t, userServiceLogsByUuidChan, "Received a nil user service logs channel, but a non-nil value was expected") require.NotNil(t, errChan, "Received a nil error logs channel, but a non-nil value was expected") - t.Log("DID NOT RECEIVE NIL SERVICE LOGS CHANNEL") shouldReceiveStream := true for shouldReceiveStream { - t.Log("RECEIVING STREAM") select { case <-time.Tick(testTimeOut): - t.Log("TIMEOUT ERROR") return nil, stacktrace.NewError("Receiving stream logs in the test has reached the '%v' time out", testTimeOut) case streamErr, isChanOpen := <-errChan: - t.Log("RECEIVING STREAM ERROR") if !isChanOpen { shouldReceiveStream = false break } - - t.Logf("RECEIVING STREAM ERR: %v", streamErr) + return nil, stacktrace.Propagate(streamErr, "Receiving streaming error.") case userServiceLogsByUuid, isChanOpen := <-userServiceLogsByUuidChan: - t.Log("RECEIVING LOGS") if !isChanOpen { shouldReceiveStream = false break @@ -390,16 +425,16 @@ func executeStreamCallAndGetReceivedServiceLogLines( return receivedServiceLogsByUuid, nil } -func createUnderlyingMapFilesystem() fstest.MapFS { +func createFullUnderlyingMapFilesystem() *fstest.MapFS { logLines := []string{logLine1, logLine2, logLine3, logLine4, logLine5, logLine6, logLine7, logLine8} logLinesStr := strings.Join(logLines, "\n") - file1 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpath, string(enclaveUuid), testUserService1Uuid, filetype) - file2 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpath, string(enclaveUuid), testUserService2Uuid, filetype) - file3 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpath, string(enclaveUuid), testUserService3Uuid, filetype) + file1 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpathForTests, testEnclaveUuid, testUserService1Uuid, filetype) + file2 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpathForTests, testEnclaveUuid, testUserService2Uuid, filetype) + file3 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpathForTests, testEnclaveUuid, testUserService3Uuid, filetype) - mapFs := fstest.MapFS{ + mapFs := &fstest.MapFS{ file1: { Data: []byte(logLinesStr), }, @@ -413,3 +448,23 @@ func createUnderlyingMapFilesystem() fstest.MapFS { return mapFs } + +func createEmptyUnderlyingMapFilesystem() *fstest.MapFS { + file1 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpathForTests, testEnclaveUuid, testUserService1Uuid, filetype) + file2 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpathForTests, testEnclaveUuid, testUserService2Uuid, filetype) + file3 := fmt.Sprintf("%s%s/%s%s", logsStorageDirpathForTests, testEnclaveUuid, testUserService3Uuid, filetype) + + mapFs := &fstest.MapFS{ + file1: { + Data: []byte{}, + }, + file2: { + Data: []byte{}, + }, + file3: { + Data: []byte{}, + }, + } + + return mapFs +} diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem.go index 722450d97c..f10a4ff7be 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem.go @@ -3,11 +3,16 @@ package persistent_volume import ( "io" "os" + "strings" "testing/fstest" ) +const ( + forwardSlash = "/" +) + // VolumeFilesystem interface is an abstraction of the disk filesystem -// primarily for the purpose of enabling unit testing PersistentVolumeLogsDatabaseClient +// primarily for the purpose of enabling unit testing persistentVolumeLogsDatabaseClient type VolumeFilesystem interface { Open(name string) (VolumeFile, error) } @@ -23,16 +28,23 @@ func NewOsVolumeFilesystem() *OsVolumeFilesystem { return &OsVolumeFilesystem{} } -func (fs *OsVolumeFilesystem) Open(name string) (VolumeFile, error) { return os.Open(name) } +func (fs *OsVolumeFilesystem) Open(name string) (VolumeFile, error) { + return os.Open(name) +} // MockedVolumeFilesystem is an implementation used for unit testing type MockedVolumeFilesystem struct { // we use an underlying map filesystem that's easy to mock file data with - mapFS fstest.MapFS + mapFS *fstest.MapFS } -func NewMockedVolumeFilesystem(fs fstest.MapFS) *MockedVolumeFilesystem { +func NewMockedVolumeFilesystem(fs *fstest.MapFS) *MockedVolumeFilesystem { return &MockedVolumeFilesystem{mapFS: fs} } -func (fs *MockedVolumeFilesystem) Open(name string) (VolumeFile, error) { return fs.mapFS.Open(name) } +func (fs *MockedVolumeFilesystem) Open(name string) (VolumeFile, error) { + // Trim any forward slashes from this filepath + // fstest.MapFS doesn't like absolute paths!!! + nameWoForwardSlash := strings.TrimLeft(name, forwardSlash) + return fs.mapFS.Open(nameWoForwardSlash) +} From f3db26d8cfce4072510ad97ae9771f5d6b3faef6 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 18 Aug 2023 16:07:36 -0400 Subject: [PATCH 33/40] clean up --- .circleci/config.yml | 4 ++-- .../implementations/vector/consts.go | 6 +++++- .../golang/testsuite/search_logs_test/search_logs_test.go | 2 ++ .../golang/testsuite/stream_logs_test/stream_logs_test.go | 2 ++ .../src/testsuite/search_logs_test/search_logs.test.ts | 2 ++ .../src/testsuite/stream_log_test/stream_logs.test.ts | 2 ++ 6 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e745bd1c7d..86c9ed4635 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -669,7 +669,7 @@ jobs: false fi - run: - name: "Verify Kurtosis cleaned up all its volumes (except for the log storage volume, so we can persiste enclaves)" + name: "Verify Kurtosis cleaned up all its volumes (except for the log storage volume, so we can persist service logs)" command: | if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l ) -eq 1 ]; then docker volume ls @@ -828,7 +828,7 @@ jobs: false fi - run: - name: "Verify Kurtosis cleaned up all its volumes (except for the logs storage volume)" + name: "Verify Kurtosis cleaned up all its volumes (except for the log storage volume, so we can persist service logs)" command: | if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l) -eq 1 ]; then docker volume ls diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go index 454d23a2b9..6340d469a7 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector/consts.go @@ -27,7 +27,10 @@ const ( logsFilepath = "\"" + logsStorageDirpath + "{{ enclave_uuid }}/{{ service_uuid }}.json\"" configFileTemplateName = "vectorConfigFileTemplate" - configFileTemplate = ` + + // Note: we set buffer to block so that we don't drop any logs, however this could apply backpressure up the topology + // if we start noticing slowdown due to vector buffer blocking, we might want to revisit our architecture + configFileTemplate = ` [sources.{{ .Source.Id }}] type = {{ .Source.Type }} address = "{{ .Source.Address }}" @@ -39,5 +42,6 @@ path = {{ .Sink.Filepath }} encoding.codec = "json" buffer.when_full = "block" ` + ////////////////////////--FINISH--VECTOR CONFIGURATION SECTION--///////////////////////////// ) diff --git a/internal_testsuites/golang/testsuite/search_logs_test/search_logs_test.go b/internal_testsuites/golang/testsuite/search_logs_test/search_logs_test.go index a32a77b933..483f1c4b3c 100644 --- a/internal_testsuites/golang/testsuite/search_logs_test/search_logs_test.go +++ b/internal_testsuites/golang/testsuite/search_logs_test/search_logs_test.go @@ -120,6 +120,8 @@ func TestSearchLogs(t *testing.T) { require.NoError(t, err, "An error occurred adding services with log lines '%+v'", logLinesByService) require.Equal(t, len(logLinesByService), len(serviceList)) + // It takes some time for logs to persist so we sleep to ensure logs have persisted + // Otherwise the test is flaky time.Sleep(secondsToWaitForLogs) // ------------------------------------- TEST RUN ------------------------------------------------- enclaveUuid := enclaveCtx.GetEnclaveUuid() diff --git a/internal_testsuites/golang/testsuite/stream_logs_test/stream_logs_test.go b/internal_testsuites/golang/testsuite/stream_logs_test/stream_logs_test.go index 5a87e38930..597eac4a0f 100644 --- a/internal_testsuites/golang/testsuite/stream_logs_test/stream_logs_test.go +++ b/internal_testsuites/golang/testsuite/stream_logs_test/stream_logs_test.go @@ -78,6 +78,8 @@ func TestStreamLogs(t *testing.T) { serviceList, err := test_helpers.AddServicesWithLogLines(ctx, enclaveCtx, logLinesByService) require.NoError(t, err, "An error occurred adding the datastore service") + // It takes some time for logs to persist so we sleep to ensure logs have persisted + // Otherwise the test is flaky time.Sleep(secondsToWaitForLogs) // ------------------------------------- TEST RUN ---------------------------------------------- diff --git a/internal_testsuites/typescript/src/testsuite/search_logs_test/search_logs.test.ts b/internal_testsuites/typescript/src/testsuite/search_logs_test/search_logs.test.ts index 37f90ff6f8..a348a8ec97 100644 --- a/internal_testsuites/typescript/src/testsuite/search_logs_test/search_logs.test.ts +++ b/internal_testsuites/typescript/src/testsuite/search_logs_test/search_logs.test.ts @@ -116,6 +116,8 @@ async function TestSearchLogs() { throw new Error(`Expected number of added services '${LOG_LINES_BY_SERVICE.size}', but the actual number of added services is '${serviceList.size}'`); } + // It takes some time for logs to persist so we sleep to ensure logs have persisted + // Otherwise the test is flaky await sleep(MILLISECONDS_TO_WAIT_FOR_LOGS); // ------------------------------------- TEST RUN ---------------------------------------------- diff --git a/internal_testsuites/typescript/src/testsuite/stream_log_test/stream_logs.test.ts b/internal_testsuites/typescript/src/testsuite/stream_log_test/stream_logs.test.ts index 32ad32f878..9997ec2b97 100644 --- a/internal_testsuites/typescript/src/testsuite/stream_log_test/stream_logs.test.ts +++ b/internal_testsuites/typescript/src/testsuite/stream_log_test/stream_logs.test.ts @@ -103,6 +103,8 @@ async function TestStreamLogs() { throw new Error(`Expected number of added services '${LOG_LINES_BY_SERVICE.size}', but the actual number of added services is '${serviceList.size}'`); } + // It takes some time for logs to persist so we sleep to ensure logs have persisted + // Otherwise the test is flaky await sleep(MILLISECONDS_TO_WAIT_FOR_LOGS); // ------------------------------------- TEST RUN ---------------------------------------------- From 47617143ee969fec2a9d754030715dc6defeaf8f Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 18 Aug 2023 16:21:09 -0400 Subject: [PATCH 34/40] add persisted logs test to golang --- .../golang/test_helpers/test_helpers.go | 25 +++ .../persisted_logs_test.go | 168 ++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 internal_testsuites/golang/testsuite/persisted_logs_test/persisted_logs_test.go diff --git a/internal_testsuites/golang/test_helpers/test_helpers.go b/internal_testsuites/golang/test_helpers/test_helpers.go index 30f3ddaa5b..b1883ee4ad 100644 --- a/internal_testsuites/golang/test_helpers/test_helpers.go +++ b/internal_testsuites/golang/test_helpers/test_helpers.go @@ -180,6 +180,31 @@ def run(plan): return serviceContext, nil } +func RemoveService( + ctx context.Context, + enclaveCtx *enclaves.EnclaveContext, + serviceName services.ServiceName, +) error { + starlarkScript := fmt.Sprintf(` +def run(plan): + plan.remove_service(name = "%s") +`, serviceName) + starlarkRunResult, err := enclaveCtx.RunStarlarkScriptBlocking(ctx, useDefaultMainFile, starlarkScript, "{}", false, defaultParallelism, noExperimentalFeature) + if err != nil { + return stacktrace.Propagate(err, "An error has occurred when running Starlark to remove service") + } + if len(starlarkRunResult.ValidationErrors) > 0 { + return stacktrace.NewError("An error has occurred when validating Starlark to remove service: %s", starlarkRunResult.ValidationErrors) + } + if starlarkRunResult.InterpretationError != nil { + return stacktrace.NewError("An error has occurred when interpreting Starlark to remove service: %s", starlarkRunResult.InterpretationError) + } + if starlarkRunResult.ExecutionError != nil { + return stacktrace.NewError("An error has occurred when executing Starlark to remove service: %s", starlarkRunResult.ExecutionError) + } + return nil +} + func AddDatastoreService( ctx context.Context, serviceName services.ServiceName, diff --git a/internal_testsuites/golang/testsuite/persisted_logs_test/persisted_logs_test.go b/internal_testsuites/golang/testsuite/persisted_logs_test/persisted_logs_test.go new file mode 100644 index 0000000000..a991e71154 --- /dev/null +++ b/internal_testsuites/golang/testsuite/persisted_logs_test/persisted_logs_test.go @@ -0,0 +1,168 @@ +//go:build !kubernetes +// +build !kubernetes + +// We don't run this test in Kubernetes because, as of 2022-10-28, the centralized logs feature is not implemented in Kubernetes yet +//TODO remove this comments after Kubernetes implementation + +package persisted_logs_test + +import ( + "context" + "github.com/kurtosis-tech/kurtosis-cli/golang_internal_testsuite/test_helpers" + "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/services" + "github.com/kurtosis-tech/kurtosis/api/golang/engine/lib/kurtosis_context" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +const ( + // Tests that logs are still around after services are removed + testName = "persisted-logs" + + exampleServiceNamePrefix = "persisted-logs-" + + shouldFollowLogs = true + shouldNotFollowLogs = false + + service1ServiceName services.ServiceName = exampleServiceNamePrefix + "service-1" + + firstFilterText = "The data have being loaded" + secondFilterText = "Starting feature" + thirdFilterText = "pool" + matchRegexFilterStr = "Starting.*logs'" + + testTimeOut = 180 * time.Second + + logLine1 = "Starting feature 'centralized logs'" + logLine2 = "Starting feature 'enclave pool'" + logLine3 = "Starting feature 'enclave pool with size 2'" + logLine4 = "The data have being loaded" + + secondsToWaitForLogs = 1 * time.Second +) + +var ( + expectedNonExistenceServiceUuids = map[services.ServiceUUID]bool{} + + service1LogLines = []string{ + logLine1, + logLine2, + logLine3, + logLine4, + } + + logLinesByService = map[services.ServiceName][]string{ + service1ServiceName: service1LogLines, + } + + doesContainTextFilterForFirstRequest = kurtosis_context.NewDoesContainTextLogLineFilter(firstFilterText) + doesContainTextFilterForSecondRequest = kurtosis_context.NewDoesContainTextLogLineFilter(secondFilterText) + doesNotContainTextFilter = kurtosis_context.NewDoesNotContainTextLogLineFilter(thirdFilterText) + doesContainMatchRegexFilter = kurtosis_context.NewDoesContainMatchRegexLogLineFilter(matchRegexFilterStr) + doesNotContainMatchRegexFilter = kurtosis_context.NewDoesNotContainMatchRegexLogLineFilter(matchRegexFilterStr) + + filtersByRequest = []kurtosis_context.LogLineFilter{ + *doesContainTextFilterForFirstRequest, + *doesContainTextFilterForSecondRequest, + *doesNotContainTextFilter, + *doesContainMatchRegexFilter, + *doesNotContainMatchRegexFilter, + } + + expectedLogLinesByRequest = [][]string{ + { + logLine4, + }, + { + logLine1, + logLine2, + logLine3, + }, + { + logLine1, + logLine4, + }, + { + logLine1, + }, + { + logLine2, + logLine3, + logLine4, + }, + } + + shouldFollowLogsValueByRequest = []bool{ + shouldFollowLogs, + shouldNotFollowLogs, + shouldNotFollowLogs, + shouldNotFollowLogs, + shouldNotFollowLogs, + } +) + +func TestSearchLogs(t *testing.T) { + ctx := context.Background() + + // ------------------------------------- ENGINE SETUP ---------------------------------------------- + enclaveCtx, _, destroyEnclaveFunc, err := test_helpers.CreateEnclave(t, ctx, testName) + require.NoError(t, err, "An error occurred creating an enclave") + defer func() { + err = destroyEnclaveFunc() + require.NoError(t, err, "An error occurred destroying the enclave after the test finished") + }() + + // ------------------------------------- TEST SETUP ---------------------------------------------- + kurtosisCtx, err := kurtosis_context.NewKurtosisContextFromLocalEngine() + require.NoError(t, err) + + serviceList, err := test_helpers.AddServicesWithLogLines(ctx, enclaveCtx, logLinesByService) + require.NoError(t, err, "An error occurred adding services with log lines '%+v'", logLinesByService) + require.Equal(t, len(logLinesByService), len(serviceList)) + + // It takes some time for logs to persist so we sleep to ensure logs have persisted + // Otherwise the test is flaky + time.Sleep(secondsToWaitForLogs) + // ------------------------------------- TEST RUN ------------------------------------------------- + enclaveUuid := enclaveCtx.GetEnclaveUuid() + + userServiceUuids := map[services.ServiceUUID]bool{} + for serviceName, serviceCtx := range serviceList { + serviceUuid := serviceCtx.GetServiceUUID() + userServiceUuids[serviceUuid] = true + + // NOW REMOVE THE SERVICE + err = test_helpers.RemoveService(ctx, enclaveCtx, serviceName) + require.NoError(t, err, "An error occurred trying to remove service '%v' during test", serviceName) + } + + expectedLogLinesByService := map[services.ServiceUUID][]string{} + + for requestIndex, filter := range filtersByRequest { + + for serviceUuid := range userServiceUuids { + expectedLogLinesByService[serviceUuid] = expectedLogLinesByRequest[requestIndex] + } + + shouldFollowLogsOption := shouldFollowLogsValueByRequest[requestIndex] + + testEvaluationErr, receivedLogLinesByService, receivedNotFoundServiceUuids := test_helpers.GetLogsResponse( + t, + ctx, + testTimeOut, + kurtosisCtx, + string(enclaveUuid), + userServiceUuids, + expectedLogLinesByService, + shouldFollowLogsOption, + &filter, + ) + + require.NoError(t, testEvaluationErr) + for serviceUuid := range userServiceUuids { + require.Equal(t, expectedLogLinesByRequest[requestIndex], receivedLogLinesByService[serviceUuid]) + } + require.Equal(t, expectedNonExistenceServiceUuids, receivedNotFoundServiceUuids) + } +} From c3e9b3a98ea574793135ad38d6b7b497aa9f8d66 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 18 Aug 2023 16:27:13 -0400 Subject: [PATCH 35/40] add typescript persisted logs test --- .../src/test_helpers/test_helpers.ts | 28 +++ .../persisted_logs.test.ts | 214 ++++++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 internal_testsuites/typescript/src/testsuite/persisted_logs_test/persisted_logs.test.ts diff --git a/internal_testsuites/typescript/src/test_helpers/test_helpers.ts b/internal_testsuites/typescript/src/test_helpers/test_helpers.ts index 8ce3913637..88eafc8959 100644 --- a/internal_testsuites/typescript/src/test_helpers/test_helpers.ts +++ b/internal_testsuites/typescript/src/test_helpers/test_helpers.ts @@ -336,6 +336,34 @@ export async function addServiceViaStarlark(enclaveContext: EnclaveContext, serv return ok(serviceContext) } + +export async function removeServiceViaStarlark(enclaveContext: EnclaveContext, serviceName: string): Promise> { + const removeServiceScript = `def run(plan): + plan.remove_service(name="${serviceName}") +` + const starlarkScriptRunResultPromise = enclaveContext.runStarlarkScriptBlocking( + DEFAULT_RUN_FUNCTION_NAME, + removeServiceScript, + STARLARK_SCRIPT_NO_PARAM, + STARLARK_NO_DRY_RUN, + NO_EXPERIMENTAL_FEATURE, + ) + const starlarkScriptRunResult = await starlarkScriptRunResultPromise + + if (starlarkScriptRunResult.isErr()) { + return err(starlarkScriptRunResult.error); + } + const starlarkScriptRunResultValue = starlarkScriptRunResult.value + if (starlarkScriptRunResultValue.interpretationError != undefined) { + return err(new Error(starlarkScriptRunResultValue.interpretationError.getErrorMessage())); + } else if (starlarkScriptRunResultValue.validationErrors.length > 0) { + return err(new Error(`Starlark script did not pass validation. Errors were:\n${starlarkScriptRunResultValue.validationErrors.join("\n")}`)); + } else if (starlarkScriptRunResultValue.executionError != undefined) { + return err(new Error(starlarkScriptRunResultValue.executionError.getErrorMessage())); + } + return ok() +} + // ==================================================================================================== // Private Helper Methods // ==================================================================================================== diff --git a/internal_testsuites/typescript/src/testsuite/persisted_logs_test/persisted_logs.test.ts b/internal_testsuites/typescript/src/testsuite/persisted_logs_test/persisted_logs.test.ts new file mode 100644 index 0000000000..5ef91d414a --- /dev/null +++ b/internal_testsuites/typescript/src/testsuite/persisted_logs_test/persisted_logs.test.ts @@ -0,0 +1,214 @@ +import {createEnclave} from "../../test_helpers/enclave_setup"; +import { + EnclaveUUID, + KurtosisContext, + LogLineFilter, + ServiceContext, + ServiceUUID, + ServiceName, + ServiceLog, +} from "kurtosis-sdk"; +import {err, ok, Result} from "neverthrow"; +import log from "loglevel"; +import { + addServicesWithLogLines, + delay, + getLogsResponseAndEvaluateResponse, + removeServiceViaStarlark +} from "../../test_helpers/test_helpers"; + +const TEST_NAME = "search-logs"; + +const EXAMPLE_SERVICE_NAME_PREFIX = "search-logs-"; + +const SHOULD_FOLLOW_LOGS = true; +const SHOULD_NOT_FOLLOW_LOGS = false; + +const SERVICE_1_SERVICE_NAME = EXAMPLE_SERVICE_NAME_PREFIX + "service-1"; + +const FIRST_FILTER_TEXT = "The data have being loaded"; +const SECOND_FILTER_TEXT = "Starting feature"; +const THIRD_FILTER_TEXT = "pool"; +const MATCH_REGEX_FILTER_STR = "Starting.*logs'"; + +const LOG_LINE_1 = new ServiceLog("Starting feature 'centralized logs'"); +const LOG_LINE_2 = new ServiceLog("Starting feature 'enclave pool'"); +const LOG_LINE_3 = new ServiceLog("Starting feature 'enclave pool with size 2'"); +const LOG_LINE_4 = new ServiceLog("The data have being loaded"); + +const EXPECTED_NON_EXISTENCE_SERVICE_UUIDS = new Set(); + +const SERVICE_1_LOG_LINES = [LOG_LINE_1, LOG_LINE_2, LOG_LINE_3, LOG_LINE_4]; + +const LOG_LINES_BY_SERVICE = new Map([ + [SERVICE_1_SERVICE_NAME, SERVICE_1_LOG_LINES], +]) + +const DOES_CONTAIN_TEXT_FILTER_FOR_FIRST_REQUEST = LogLineFilter.NewDoesContainTextLogLineFilter(FIRST_FILTER_TEXT); +const DOES_CONTAIN_TEXT_FILTER_FOR_SECOND_REQUEST = LogLineFilter.NewDoesContainTextLogLineFilter(SECOND_FILTER_TEXT); +const DOES_NOT_CONTAIN_TEXT_FILTER = LogLineFilter.NewDoesNotContainTextLogLineFilter(THIRD_FILTER_TEXT); +const DOES_CONTAIN_MATCH_REGEX_FILTER = LogLineFilter.NewDoesContainMatchRegexLogLineFilter(MATCH_REGEX_FILTER_STR); +const DOES_NOT_CONTAIN_MATCH_REGEX_FILTER = LogLineFilter.NewDoesNotContainMatchRegexLogLineFilter(MATCH_REGEX_FILTER_STR); + +const FILTERS_BY_REQUEST = new Array( + DOES_CONTAIN_TEXT_FILTER_FOR_FIRST_REQUEST, + DOES_CONTAIN_TEXT_FILTER_FOR_SECOND_REQUEST, + DOES_NOT_CONTAIN_TEXT_FILTER, + DOES_CONTAIN_MATCH_REGEX_FILTER, + DOES_NOT_CONTAIN_MATCH_REGEX_FILTER, +) + +const EXPECTED_LOG_LINES_BY_REQUEST = Array( + [ + LOG_LINE_1, + ], + [ + LOG_LINE_1, + LOG_LINE_2, + LOG_LINE_3, + ], + [ + LOG_LINE_1, + LOG_LINE_4, + ], + [ + LOG_LINE_1, + ], + [ + LOG_LINE_2, + LOG_LINE_3, + LOG_LINE_4, + ], +) + +const SHOULD_FOLLOW_LOGS_VALUES_BY_REQUEST: boolean[] = [ + SHOULD_FOLLOW_LOGS, + SHOULD_NOT_FOLLOW_LOGS, + SHOULD_NOT_FOLLOW_LOGS, + SHOULD_NOT_FOLLOW_LOGS, + SHOULD_NOT_FOLLOW_LOGS, +] + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const MILLISECONDS_TO_WAIT_FOR_LOGS = 1000 + +jest.setTimeout(180000); + +test("Test Search Logs", TestSearchLogs); + +async function TestSearchLogs() { + // ------------------------------------- ENGINE SETUP ---------------------------------------------- + const createEnclaveResult = await createEnclave(TEST_NAME); + + if (createEnclaveResult.isErr()) { + throw createEnclaveResult.error; + } + + const {enclaveContext, stopEnclaveFunction} = createEnclaveResult.value; + + try { + // ------------------------------------- TEST SETUP ---------------------------------------------- + + const serviceListResult: Result, Error> = await addServicesWithLogLines(enclaveContext, LOG_LINES_BY_SERVICE); + + if (serviceListResult.isErr()) { + throw new Error(`An error occurred adding the services for the test. Error:\n${serviceListResult.error}`); + } + + const serviceList: Map = serviceListResult.value; + + if (LOG_LINES_BY_SERVICE.size != serviceList.size) { + throw new Error(`Expected number of added services '${LOG_LINES_BY_SERVICE.size}', but the actual number of added services is '${serviceList.size}'`); + } + + // It takes some time for logs to persist so we sleep to ensure logs have persisted + // Otherwise the test is flaky + await sleep(MILLISECONDS_TO_WAIT_FOR_LOGS); + // ------------------------------------- TEST RUN ---------------------------------------------- + + const newKurtosisContextResult = await KurtosisContext.newKurtosisContextFromLocalEngine(); + if (newKurtosisContextResult.isErr()) { + log.error(`An error occurred connecting to the Kurtosis engine for running test search logs`) + return err(newKurtosisContextResult.error) + } + const kurtosisContext = newKurtosisContextResult.value; + + const enclaveUuid: EnclaveUUID = enclaveContext.getEnclaveUuid(); + + const userServiceUuids: Set = new Set(); + + let serviceUuid: ServiceUUID = ""; + + for (let [serviceName, serviceCtx] of serviceList) { + serviceUuid = serviceCtx.getServiceUUID(); + userServiceUuids.add(serviceUuid); + + // NOW REMOVE SERVICE + const removeServiceResult: Result = await removeServiceViaStarlark(enclaveContext, serviceName); + if (removeServiceResult.isErr()){ + throw new Error(`An error occurred removing the services for the test. Error:\n${removeServiceResult.error}`); + } + } + + for (let i = 0; i < FILTERS_BY_REQUEST.length; i++) { + const filter: LogLineFilter = FILTERS_BY_REQUEST[i]; + const expectedLogLines: ServiceLog[] = EXPECTED_LOG_LINES_BY_REQUEST[i]; + const shouldFollowLogsOption: boolean = SHOULD_FOLLOW_LOGS_VALUES_BY_REQUEST[i]; + const executionResult = await executeGetLogsRequestAndEvaluateResult( + kurtosisContext, + enclaveUuid, + serviceUuid, + userServiceUuids, + filter, + expectedLogLines, + shouldFollowLogsOption, + ); + + if (executionResult.isErr()) { + throw executionResult.error; + } + } + } finally { + stopEnclaveFunction() + } + + jest.clearAllTimers() +} + +// ==================================================================================================== +// Private helper functions +// ==================================================================================================== +async function executeGetLogsRequestAndEvaluateResult( + kurtosisCtx: KurtosisContext, + enclaveUuid: EnclaveUUID, + serviceUuid: ServiceUUID, + userServiceUuids: Set, + logLineFilter: LogLineFilter, + expectedLogLines: ServiceLog[], + shouldFollowLogs: boolean, +): Promise> { + + const serviceUuids: Set = new Set([ + serviceUuid, + ]) + + const expectedLogLinesByService = new Map([ + [serviceUuid, expectedLogLines], + ]) + + const getLogsResponseResult = await getLogsResponseAndEvaluateResponse( + kurtosisCtx, + enclaveUuid, + serviceUuids, + expectedLogLinesByService, + EXPECTED_NON_EXISTENCE_SERVICE_UUIDS, + shouldFollowLogs, + logLineFilter, + ) + + if (getLogsResponseResult.isErr()) { + throw getLogsResponseResult.error; + } + + return ok(null); +} From 07da2a454c61a4a3703b4d662a3cac62cd2f2994 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 18 Aug 2023 16:29:43 -0400 Subject: [PATCH 36/40] remove test script --- .../version-0.81.7-sidebars 2.json | 80 +++++++++++++++++++ logs.sh | 8 -- 2 files changed, 80 insertions(+), 8 deletions(-) create mode 100644 docs/versioned_sidebars/version-0.81.7-sidebars 2.json delete mode 100755 logs.sh diff --git a/docs/versioned_sidebars/version-0.81.7-sidebars 2.json b/docs/versioned_sidebars/version-0.81.7-sidebars 2.json new file mode 100644 index 0000000000..892198172b --- /dev/null +++ b/docs/versioned_sidebars/version-0.81.7-sidebars 2.json @@ -0,0 +1,80 @@ +{ + "main": [ + "home", + { + "type": "link", + "label": "Kurtosis for Web3", + "href": "https://web3.kurtosis.com" + }, + "quickstart", + "code-examples", + { + "type": "category", + "label": "Guides", + "collapsed": true, + "items": [ + { + "type": "autogenerated", + "dirName": "guides" + } + ] + }, + { + "type": "category", + "label": "Explanations", + "collapsed": true, + "items": [ + { + "type": "autogenerated", + "dirName": "explanations" + } + ] + }, + { + "type": "category", + "label": "CLI Reference", + "collapsed": true, + "link": { + "type": "doc", + "id": "cli-reference/index" + }, + "items": [ + { + "type": "autogenerated", + "dirName": "cli-reference" + } + ] + }, + { + "type": "category", + "label": "Starlark Reference", + "collapsed": true, + "link": { + "type": "doc", + "id": "starlark-reference/index" + }, + "items": [ + { + "type": "autogenerated", + "dirName": "starlark-reference" + } + ] + }, + { + "type": "category", + "label": "Concepts Reference", + "collapsed": true, + "items": [ + { + "type": "autogenerated", + "dirName": "concepts-reference" + } + ] + }, + "sdk", + "faq", + "best-practices", + "roadmap", + "changelog" + ] +} diff --git a/logs.sh b/logs.sh deleted file mode 100755 index 932761eb2f..0000000000 --- a/logs.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/zsh - -cd /Users/tewodrosmitiku/Desktop/kurtosis -kurtosis clean -a -kurtosis engine stop -./cli/cli/scripts/launch-cli.sh engine start --version=$1 -./cli/cli/scripts/launch-cli.sh run github.com/kurtosis-tech/mongodb-package - From d8c85e773395b027141c3cedbe2b0fa46a64cb8a Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 18 Aug 2023 16:43:55 -0400 Subject: [PATCH 37/40] remove typescript test --- .../src/test_helpers/test_helpers.ts | 28 --- .../persisted_logs.test.ts | 214 ------------------ 2 files changed, 242 deletions(-) delete mode 100644 internal_testsuites/typescript/src/testsuite/persisted_logs_test/persisted_logs.test.ts diff --git a/internal_testsuites/typescript/src/test_helpers/test_helpers.ts b/internal_testsuites/typescript/src/test_helpers/test_helpers.ts index 88eafc8959..8ce3913637 100644 --- a/internal_testsuites/typescript/src/test_helpers/test_helpers.ts +++ b/internal_testsuites/typescript/src/test_helpers/test_helpers.ts @@ -336,34 +336,6 @@ export async function addServiceViaStarlark(enclaveContext: EnclaveContext, serv return ok(serviceContext) } - -export async function removeServiceViaStarlark(enclaveContext: EnclaveContext, serviceName: string): Promise> { - const removeServiceScript = `def run(plan): - plan.remove_service(name="${serviceName}") -` - const starlarkScriptRunResultPromise = enclaveContext.runStarlarkScriptBlocking( - DEFAULT_RUN_FUNCTION_NAME, - removeServiceScript, - STARLARK_SCRIPT_NO_PARAM, - STARLARK_NO_DRY_RUN, - NO_EXPERIMENTAL_FEATURE, - ) - const starlarkScriptRunResult = await starlarkScriptRunResultPromise - - if (starlarkScriptRunResult.isErr()) { - return err(starlarkScriptRunResult.error); - } - const starlarkScriptRunResultValue = starlarkScriptRunResult.value - if (starlarkScriptRunResultValue.interpretationError != undefined) { - return err(new Error(starlarkScriptRunResultValue.interpretationError.getErrorMessage())); - } else if (starlarkScriptRunResultValue.validationErrors.length > 0) { - return err(new Error(`Starlark script did not pass validation. Errors were:\n${starlarkScriptRunResultValue.validationErrors.join("\n")}`)); - } else if (starlarkScriptRunResultValue.executionError != undefined) { - return err(new Error(starlarkScriptRunResultValue.executionError.getErrorMessage())); - } - return ok() -} - // ==================================================================================================== // Private Helper Methods // ==================================================================================================== diff --git a/internal_testsuites/typescript/src/testsuite/persisted_logs_test/persisted_logs.test.ts b/internal_testsuites/typescript/src/testsuite/persisted_logs_test/persisted_logs.test.ts deleted file mode 100644 index 5ef91d414a..0000000000 --- a/internal_testsuites/typescript/src/testsuite/persisted_logs_test/persisted_logs.test.ts +++ /dev/null @@ -1,214 +0,0 @@ -import {createEnclave} from "../../test_helpers/enclave_setup"; -import { - EnclaveUUID, - KurtosisContext, - LogLineFilter, - ServiceContext, - ServiceUUID, - ServiceName, - ServiceLog, -} from "kurtosis-sdk"; -import {err, ok, Result} from "neverthrow"; -import log from "loglevel"; -import { - addServicesWithLogLines, - delay, - getLogsResponseAndEvaluateResponse, - removeServiceViaStarlark -} from "../../test_helpers/test_helpers"; - -const TEST_NAME = "search-logs"; - -const EXAMPLE_SERVICE_NAME_PREFIX = "search-logs-"; - -const SHOULD_FOLLOW_LOGS = true; -const SHOULD_NOT_FOLLOW_LOGS = false; - -const SERVICE_1_SERVICE_NAME = EXAMPLE_SERVICE_NAME_PREFIX + "service-1"; - -const FIRST_FILTER_TEXT = "The data have being loaded"; -const SECOND_FILTER_TEXT = "Starting feature"; -const THIRD_FILTER_TEXT = "pool"; -const MATCH_REGEX_FILTER_STR = "Starting.*logs'"; - -const LOG_LINE_1 = new ServiceLog("Starting feature 'centralized logs'"); -const LOG_LINE_2 = new ServiceLog("Starting feature 'enclave pool'"); -const LOG_LINE_3 = new ServiceLog("Starting feature 'enclave pool with size 2'"); -const LOG_LINE_4 = new ServiceLog("The data have being loaded"); - -const EXPECTED_NON_EXISTENCE_SERVICE_UUIDS = new Set(); - -const SERVICE_1_LOG_LINES = [LOG_LINE_1, LOG_LINE_2, LOG_LINE_3, LOG_LINE_4]; - -const LOG_LINES_BY_SERVICE = new Map([ - [SERVICE_1_SERVICE_NAME, SERVICE_1_LOG_LINES], -]) - -const DOES_CONTAIN_TEXT_FILTER_FOR_FIRST_REQUEST = LogLineFilter.NewDoesContainTextLogLineFilter(FIRST_FILTER_TEXT); -const DOES_CONTAIN_TEXT_FILTER_FOR_SECOND_REQUEST = LogLineFilter.NewDoesContainTextLogLineFilter(SECOND_FILTER_TEXT); -const DOES_NOT_CONTAIN_TEXT_FILTER = LogLineFilter.NewDoesNotContainTextLogLineFilter(THIRD_FILTER_TEXT); -const DOES_CONTAIN_MATCH_REGEX_FILTER = LogLineFilter.NewDoesContainMatchRegexLogLineFilter(MATCH_REGEX_FILTER_STR); -const DOES_NOT_CONTAIN_MATCH_REGEX_FILTER = LogLineFilter.NewDoesNotContainMatchRegexLogLineFilter(MATCH_REGEX_FILTER_STR); - -const FILTERS_BY_REQUEST = new Array( - DOES_CONTAIN_TEXT_FILTER_FOR_FIRST_REQUEST, - DOES_CONTAIN_TEXT_FILTER_FOR_SECOND_REQUEST, - DOES_NOT_CONTAIN_TEXT_FILTER, - DOES_CONTAIN_MATCH_REGEX_FILTER, - DOES_NOT_CONTAIN_MATCH_REGEX_FILTER, -) - -const EXPECTED_LOG_LINES_BY_REQUEST = Array( - [ - LOG_LINE_1, - ], - [ - LOG_LINE_1, - LOG_LINE_2, - LOG_LINE_3, - ], - [ - LOG_LINE_1, - LOG_LINE_4, - ], - [ - LOG_LINE_1, - ], - [ - LOG_LINE_2, - LOG_LINE_3, - LOG_LINE_4, - ], -) - -const SHOULD_FOLLOW_LOGS_VALUES_BY_REQUEST: boolean[] = [ - SHOULD_FOLLOW_LOGS, - SHOULD_NOT_FOLLOW_LOGS, - SHOULD_NOT_FOLLOW_LOGS, - SHOULD_NOT_FOLLOW_LOGS, - SHOULD_NOT_FOLLOW_LOGS, -] - -const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); -const MILLISECONDS_TO_WAIT_FOR_LOGS = 1000 - -jest.setTimeout(180000); - -test("Test Search Logs", TestSearchLogs); - -async function TestSearchLogs() { - // ------------------------------------- ENGINE SETUP ---------------------------------------------- - const createEnclaveResult = await createEnclave(TEST_NAME); - - if (createEnclaveResult.isErr()) { - throw createEnclaveResult.error; - } - - const {enclaveContext, stopEnclaveFunction} = createEnclaveResult.value; - - try { - // ------------------------------------- TEST SETUP ---------------------------------------------- - - const serviceListResult: Result, Error> = await addServicesWithLogLines(enclaveContext, LOG_LINES_BY_SERVICE); - - if (serviceListResult.isErr()) { - throw new Error(`An error occurred adding the services for the test. Error:\n${serviceListResult.error}`); - } - - const serviceList: Map = serviceListResult.value; - - if (LOG_LINES_BY_SERVICE.size != serviceList.size) { - throw new Error(`Expected number of added services '${LOG_LINES_BY_SERVICE.size}', but the actual number of added services is '${serviceList.size}'`); - } - - // It takes some time for logs to persist so we sleep to ensure logs have persisted - // Otherwise the test is flaky - await sleep(MILLISECONDS_TO_WAIT_FOR_LOGS); - // ------------------------------------- TEST RUN ---------------------------------------------- - - const newKurtosisContextResult = await KurtosisContext.newKurtosisContextFromLocalEngine(); - if (newKurtosisContextResult.isErr()) { - log.error(`An error occurred connecting to the Kurtosis engine for running test search logs`) - return err(newKurtosisContextResult.error) - } - const kurtosisContext = newKurtosisContextResult.value; - - const enclaveUuid: EnclaveUUID = enclaveContext.getEnclaveUuid(); - - const userServiceUuids: Set = new Set(); - - let serviceUuid: ServiceUUID = ""; - - for (let [serviceName, serviceCtx] of serviceList) { - serviceUuid = serviceCtx.getServiceUUID(); - userServiceUuids.add(serviceUuid); - - // NOW REMOVE SERVICE - const removeServiceResult: Result = await removeServiceViaStarlark(enclaveContext, serviceName); - if (removeServiceResult.isErr()){ - throw new Error(`An error occurred removing the services for the test. Error:\n${removeServiceResult.error}`); - } - } - - for (let i = 0; i < FILTERS_BY_REQUEST.length; i++) { - const filter: LogLineFilter = FILTERS_BY_REQUEST[i]; - const expectedLogLines: ServiceLog[] = EXPECTED_LOG_LINES_BY_REQUEST[i]; - const shouldFollowLogsOption: boolean = SHOULD_FOLLOW_LOGS_VALUES_BY_REQUEST[i]; - const executionResult = await executeGetLogsRequestAndEvaluateResult( - kurtosisContext, - enclaveUuid, - serviceUuid, - userServiceUuids, - filter, - expectedLogLines, - shouldFollowLogsOption, - ); - - if (executionResult.isErr()) { - throw executionResult.error; - } - } - } finally { - stopEnclaveFunction() - } - - jest.clearAllTimers() -} - -// ==================================================================================================== -// Private helper functions -// ==================================================================================================== -async function executeGetLogsRequestAndEvaluateResult( - kurtosisCtx: KurtosisContext, - enclaveUuid: EnclaveUUID, - serviceUuid: ServiceUUID, - userServiceUuids: Set, - logLineFilter: LogLineFilter, - expectedLogLines: ServiceLog[], - shouldFollowLogs: boolean, -): Promise> { - - const serviceUuids: Set = new Set([ - serviceUuid, - ]) - - const expectedLogLinesByService = new Map([ - [serviceUuid, expectedLogLines], - ]) - - const getLogsResponseResult = await getLogsResponseAndEvaluateResponse( - kurtosisCtx, - enclaveUuid, - serviceUuids, - expectedLogLinesByService, - EXPECTED_NON_EXISTENCE_SERVICE_UUIDS, - shouldFollowLogs, - logLineFilter, - ) - - if (getLogsResponseResult.isErr()) { - throw getLogsResponseResult.error; - } - - return ok(null); -} From 12cc325a8779b9a188eb704196c36bcef4a4bd32 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 18 Aug 2023 16:54:43 -0400 Subject: [PATCH 38/40] remove random file --- .../version-0.81.7-sidebars 2.json | 80 ------------------- 1 file changed, 80 deletions(-) delete mode 100644 docs/versioned_sidebars/version-0.81.7-sidebars 2.json diff --git a/docs/versioned_sidebars/version-0.81.7-sidebars 2.json b/docs/versioned_sidebars/version-0.81.7-sidebars 2.json deleted file mode 100644 index 892198172b..0000000000 --- a/docs/versioned_sidebars/version-0.81.7-sidebars 2.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "main": [ - "home", - { - "type": "link", - "label": "Kurtosis for Web3", - "href": "https://web3.kurtosis.com" - }, - "quickstart", - "code-examples", - { - "type": "category", - "label": "Guides", - "collapsed": true, - "items": [ - { - "type": "autogenerated", - "dirName": "guides" - } - ] - }, - { - "type": "category", - "label": "Explanations", - "collapsed": true, - "items": [ - { - "type": "autogenerated", - "dirName": "explanations" - } - ] - }, - { - "type": "category", - "label": "CLI Reference", - "collapsed": true, - "link": { - "type": "doc", - "id": "cli-reference/index" - }, - "items": [ - { - "type": "autogenerated", - "dirName": "cli-reference" - } - ] - }, - { - "type": "category", - "label": "Starlark Reference", - "collapsed": true, - "link": { - "type": "doc", - "id": "starlark-reference/index" - }, - "items": [ - { - "type": "autogenerated", - "dirName": "starlark-reference" - } - ] - }, - { - "type": "category", - "label": "Concepts Reference", - "collapsed": true, - "items": [ - { - "type": "autogenerated", - "dirName": "concepts-reference" - } - ] - }, - "sdk", - "faq", - "best-practices", - "roadmap", - "changelog" - ] -} From caf07a42c0bed63cab75807b8a7611ebf0687be9 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Mon, 21 Aug 2023 07:13:52 -0700 Subject: [PATCH 39/40] s/partitioning/idempotent --- .../persistent_volume_logs_database_client_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go index 50df3674d0..4be18b60dc 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go @@ -26,8 +26,8 @@ const ( testUserService3Uuid = "test-user-service-3" logLine1 = "{\"log\":\"Starting feature 'centralized logs'\"}" - logLine2 = "{\"log\":\"Starting feature 'network partitioning'\"}" - logLine3 = "{\"log\":\"Starting feature 'network soft partitioning'\"}" + logLine2 = "{\"log\":\"Starting feature 'idempotent runs'\"}" + logLine3 = "{\"log\":\"Starting feature 'idempotent apics'\"}" logLine4 = "{\"log\":\"Starting feature 'files storage'\"}" logLine5 = "{\"log\":\"Starting feature 'files manager'\"}" logLine6 = "{\"log\":\"The enclave was created\"}" @@ -37,7 +37,7 @@ const ( firstFilterText = "feature" secondFilterText = "Files" notFoundedFilterText = "it shouldn't be found in the log lines" - firstMatchRegexFilterStr = "Starting.*partitioning'" + firstMatchRegexFilterStr = "Starting.*idempotent'" secondMatchRegexFilterStr = "[S].*manager" testTimeOut = 2 * time.Second @@ -62,7 +62,7 @@ func TestStreamUserServiceLogs_WithFilters(t *testing.T) { *regexFilter, } - expectedFirstLogLine := "Starting feature 'network partitioning'" + expectedFirstLogLine := "Starting feature 'idempotent runs'" underlyingFs := createFullUnderlyingMapFilesystem() From bde64b032cb9e7a1a2bc08f996adf157605e90d0 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Mon, 21 Aug 2023 07:25:52 -0700 Subject: [PATCH 40/40] fix test --- .../persistent_volume_logs_database_client_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go index 4be18b60dc..07b5cbabe1 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go @@ -26,8 +26,8 @@ const ( testUserService3Uuid = "test-user-service-3" logLine1 = "{\"log\":\"Starting feature 'centralized logs'\"}" - logLine2 = "{\"log\":\"Starting feature 'idempotent runs'\"}" - logLine3 = "{\"log\":\"Starting feature 'idempotent apics'\"}" + logLine2 = "{\"log\":\"Starting feature 'runs idempotently'\"}" + logLine3 = "{\"log\":\"Starting feature 'apic idempotently'\"}" logLine4 = "{\"log\":\"Starting feature 'files storage'\"}" logLine5 = "{\"log\":\"Starting feature 'files manager'\"}" logLine6 = "{\"log\":\"The enclave was created\"}" @@ -37,7 +37,7 @@ const ( firstFilterText = "feature" secondFilterText = "Files" notFoundedFilterText = "it shouldn't be found in the log lines" - firstMatchRegexFilterStr = "Starting.*idempotent'" + firstMatchRegexFilterStr = "Starting.*idempotently'" secondMatchRegexFilterStr = "[S].*manager" testTimeOut = 2 * time.Second @@ -62,7 +62,7 @@ func TestStreamUserServiceLogs_WithFilters(t *testing.T) { *regexFilter, } - expectedFirstLogLine := "Starting feature 'idempotent runs'" + expectedFirstLogLine := "Starting feature 'runs idempotently'" underlyingFs := createFullUnderlyingMapFilesystem()