From 9ea344ededfaac909342df32dddf23757c8e873d Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Sat, 7 Oct 2023 17:03:20 -0400 Subject: [PATCH] feat: remove logs on enclave rm and clean -a (#1489) ## Description: Addresses https://github.com/kurtosis-tech/kurtosis/issues/1394 and https://github.com/kurtosis-tech/kurtosis/issues/1459 by removing an enclaves logs on `kurtosis enclave rm ...` and removing all logs on `kurtosis clean -a` ## Is this change user facing? YES ## References: https://github.com/kurtosis-tech/kurtosis/issues/1394 https://github.com/kurtosis-tech/kurtosis/issues/1459 --- cli/cli/commands/clean/clean.go | 4 +- .../engine_functions/create_engine.go | 18 -- .../mock_kurtosis_backend.go | 2 +- .../log_file_creator/log_file_creator.go | 142 ----------- .../log_file_manager/log_file_manager.go | 221 ++++++++++++++++++ .../log_file_manager/log_file_manager_test.go | 180 ++++++++++++++ .../log_remover/log_remover.go | 44 ---- .../log_remover/log_remover_test.go | 53 ----- .../per_week_stream_logs_strategy.go | 13 +- .../per_week_stream_logs_strategy_test.go | 2 +- .../persistent_volume/volume_consts/consts.go | 5 +- .../engine/enclave_manager/enclave_manager.go | 26 ++- engine/server/engine/main.go | 38 ++- .../server/engine_connect_server_service.go | 1 - 14 files changed, 447 insertions(+), 302 deletions(-) delete mode 100644 engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_creator/log_file_creator.go create mode 100644 engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager.go create mode 100644 engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager_test.go delete mode 100644 engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_remover/log_remover.go delete mode 100644 engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_remover/log_remover_test.go diff --git a/cli/cli/commands/clean/clean.go b/cli/cli/commands/clean/clean.go index cea56d441a..79238e84d5 100644 --- a/cli/cli/commands/clean/clean.go +++ b/cli/cli/commands/clean/clean.go @@ -195,6 +195,6 @@ func formattedUuidAndName(enclaveUuidWithName *kurtosis_engine_rpc_api_bindings. } func cleanUnusedImages(ctx context.Context, kurtosisBackend backend_interface.KurtosisBackend, shouldCleanAll bool) ([]string, []error, error) { - cleanedImages, cleanError := kurtosisBackend.PruneUnusedImages(ctx) - return cleanedImages, nil, cleanError + cleanedImages, cleanErr := kurtosisBackend.PruneUnusedImages(ctx) + return cleanedImages, nil, cleanErr } diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go index 3ac97edcc5..4cc40f92c7 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go @@ -15,10 +15,6 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/uuid_generator" "github.com/kurtosis-tech/kurtosis/engine/launcher/args" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_remover" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/logs_clock" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem" "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" "time" @@ -111,20 +107,6 @@ func CreateEngine( removeLogsAggregatorFunc() } }() - - // schedule log removal for log retention - go func() { - osFs := volume_filesystem.NewOsVolumeFilesystem() - realTime := logs_clock.NewRealClock() - logRemover := log_remover.NewLogRemover(osFs, realTime) - // do a first removal - logRemover.Run() - - logRemovalTicker := time.NewTicker(volume_consts.RemoveLogsWaitHours) - for range logRemovalTicker.C { - logRemover.Run() - } - }() logrus.Infof("Centralized logs components started.") enclaveManagerUIPortSpec, err := port_spec.NewPortSpec(uint16(enclaveManagerUIPort), consts.EngineTransportProtocol, consts.HttpApplicationProtocol, defaultWait) diff --git a/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go b/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go index 6b3e058bd4..8a22104f91 100644 --- a/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.29.0. DO NOT EDIT. +// Code generated by mockery v2.23.1. DO NOT EDIT. package backend_interface diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_creator/log_file_creator.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_creator/log_file_creator.go deleted file mode 100644 index 93168672ed..0000000000 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_creator/log_file_creator.go +++ /dev/null @@ -1,142 +0,0 @@ -package log_file_creator - -import ( - "context" - "fmt" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/uuid_generator" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/logs_clock" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem" - "github.com/kurtosis-tech/stacktrace" - "github.com/sirupsen/logrus" - "os" - "strconv" -) - -// LogFileCreator is responsible for creating the necessary file paths for service logs across all enclaves. -// Context: -// The LogsAggregator is configured to write logs to three different log file paths, one for uuid, service name, and shortened uuid. -// This is so that the logs are retrievable by each identifier even when enclaves are stopped. -// (More context on this here: https://github.com/kurtosis-tech/kurtosis/pull/1213) -// To prevent storing duplicate logs, the LogFileCreator will ensure that the service name and short uuid log files are just -// symlinks to the uuid log file path. -type LogFileCreator struct { - kurtosisBackend backend_interface.KurtosisBackend - - filesystem volume_filesystem.VolumeFilesystem - - time logs_clock.LogsClock -} - -func NewLogFileCreator( - kurtosisBackend backend_interface.KurtosisBackend, - filesystem volume_filesystem.VolumeFilesystem, - time logs_clock.LogsClock) *LogFileCreator { - return &LogFileCreator{ - kurtosisBackend: kurtosisBackend, - filesystem: filesystem, - time: time, - } -} - -// CreateLogFiles creates three log files for every service across all running enclaves. -// The first is a file with the name ending in the uuid of the service. -// The other two file paths are symlinks to the uuid file, ending with the shortened uuid and service name respectively. -// If files exist for the shortened uuid and service name files, but they are not symlinks, they are removed and symlink files -// are created to prevent duplicate log storage. -func (creator *LogFileCreator) CreateLogFiles(ctx context.Context) error { - var err error - - year, week := creator.time.Now().ISOWeek() - - enclaveToServicesMap, err := creator.getEnclaveAndServiceInfo(ctx) - if err != nil { - // already wrapped with propagate - return err - } - - for enclaveUuid, serviceRegistrations := range enclaveToServicesMap { - for _, serviceRegistration := range serviceRegistrations { - serviceUuidStr := string(serviceRegistration.GetUUID()) - serviceNameStr := string(serviceRegistration.GetName()) - serviceShortUuidStr := uuid_generator.ShortenedUUIDString(serviceUuidStr) - - serviceUuidFilePathStr := getFilepathStr(year, week, string(enclaveUuid), serviceUuidStr) - if err = creator.createLogFileIdempotently(serviceUuidFilePathStr); err != nil { - return err - } - - serviceNameFilePathStr := getFilepathStr(year, week, string(enclaveUuid), serviceNameStr) - if err = creator.createSymlinkLogFile(serviceUuidFilePathStr, serviceNameFilePathStr); err != nil { - return err - } - logrus.Tracef("Created symlinked log file: '%v'", serviceNameFilePathStr) - - serviceShortUuidFilePathStr := getFilepathStr(year, week, string(enclaveUuid), serviceShortUuidStr) - if err = creator.createSymlinkLogFile(serviceUuidFilePathStr, serviceShortUuidFilePathStr); err != nil { - return err - } - logrus.Tracef("Created symlinked log file: '%v'", serviceShortUuidFilePathStr) - } - } - - return nil -} - -func (creator *LogFileCreator) getEnclaveAndServiceInfo(ctx context.Context) (map[enclave.EnclaveUUID][]*service.ServiceRegistration, error) { - enclaveToServicesMap := map[enclave.EnclaveUUID][]*service.ServiceRegistration{} - - enclaves, err := creator.kurtosisBackend.GetEnclaves(ctx, &enclave.EnclaveFilters{UUIDs: nil, Statuses: nil}) - if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred while trying to get all enclaves from kurtosis backend.") - } - for enclaveUuid := range enclaves { - var serviceRegistrations []*service.ServiceRegistration - - enclaveServices, err := creator.kurtosisBackend.GetUserServices(ctx, enclaveUuid, &service.ServiceFilters{Names: nil, UUIDs: nil, Statuses: nil}) - if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred while trying to get user services for enclave '%v' from kurtosis backend.", enclaveUuid) - } - for _, serviceInfo := range enclaveServices { - serviceRegistrations = append(serviceRegistrations, serviceInfo.GetRegistration()) - } - - enclaveToServicesMap[enclaveUuid] = serviceRegistrations - } - return enclaveToServicesMap, nil -} - -func (creator *LogFileCreator) createLogFileIdempotently(logFilePath string) error { - var err error - if _, err = creator.filesystem.Stat(logFilePath); os.IsNotExist(err) { - if _, err = creator.filesystem.Create(logFilePath); err != nil { - return stacktrace.Propagate(err, "An error occurred creating a log file path at '%v'", logFilePath) - } - logrus.Tracef("Created log file: '%v'", logFilePath) - return nil - } - if err != nil { - return stacktrace.Propagate(err, "An error occurred checking if log file path at '%v' existed.", logFilePath) - } - return nil -} - -func (creator *LogFileCreator) createSymlinkLogFile(targetLogFilePath, symlinkLogFilePath string) error { - // remove existing log files that could be storing logs at this path - if err := creator.filesystem.Remove(symlinkLogFilePath); err != nil { - return stacktrace.Propagate(err, "An error occurred attempting to remove an existing log file at the symlink file path '%v'.", symlinkLogFilePath) - } - // replace with symlink - if err := creator.filesystem.Symlink(targetLogFilePath, symlinkLogFilePath); err != nil { - return stacktrace.Propagate(err, "An error occurred creating a symlink file path '%v' for target file path '%v'.", targetLogFilePath, targetLogFilePath) - } - return nil -} - -// creates a filepath of format //year/week//serviceIdentifier. -func getFilepathStr(year, week int, enclaveUuid, serviceIdentifier string) string { - return fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), strconv.Itoa(week), enclaveUuid, serviceIdentifier, volume_consts.Filetype) -} diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager.go new file mode 100644 index 0000000000..6c01115795 --- /dev/null +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager.go @@ -0,0 +1,221 @@ +package log_file_manager + +import ( + "context" + "fmt" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/uuid_generator" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/logs_clock" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem" + "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/api/errors" + "os" + "strconv" + "time" +) + +const ( + oneWeek = 7 * 24 * time.Hour +) + +// LogFileManager is responsible for creating and removing log files from filesystem. +type LogFileManager struct { + kurtosisBackend backend_interface.KurtosisBackend + + filesystem volume_filesystem.VolumeFilesystem + + time logs_clock.LogsClock +} + +func NewLogFileManager( + kurtosisBackend backend_interface.KurtosisBackend, + filesystem volume_filesystem.VolumeFilesystem, + time logs_clock.LogsClock) *LogFileManager { + return &LogFileManager{ + kurtosisBackend: kurtosisBackend, + filesystem: filesystem, + time: time, + } +} + +// StartLogFileManagement initiates logic for managing log files in the filesystem +func (manager *LogFileManager) StartLogFileManagement(ctx context.Context) { + // Schedule thread for removing log files beyond retention period + go func() { + logrus.Debugf("Scheduling log removal for log retention every '%v' hours...", volume_consts.RemoveLogsWaitHours) + manager.RemoveLogsBeyondRetentionPeriod() + + logRemovalTicker := time.NewTicker(volume_consts.RemoveLogsWaitHours) + for range logRemovalTicker.C { + logrus.Debug("Attempting to remove old log file paths...") + manager.RemoveLogsBeyondRetentionPeriod() + } + }() + + // Schedule thread for creating log files + go func() { + // TODO: Remove this when moving away from persistent volume logs db + // Creating log file paths on an interval is a hack to prevent duplicate logs from being stored by the log aggregator + // The LogsAggregator is configured to write logs to three different log file paths, one for uuid, service name, and shortened uuid + // This is so that the logs are retrievable by each identifier even when enclaves are stopped. More context on this here: https://github.com/kurtosis-tech/kurtosis/pull/1213 + // To prevent storing duplicate logs, the CreateLogFiles will ensure that the service name and short uuid log files are just symlinks to the uuid log file path + logFileCreatorTicker := time.NewTicker(volume_consts.CreateLogsWaitMinutes) + + logrus.Debugf("Scheduling log file path creation every '%v' minutes...", volume_consts.CreateLogsWaitMinutes) + for range logFileCreatorTicker.C { + logrus.Debug("Creating log file paths...") + err := manager.CreateLogFiles(ctx) + if err != nil { + logrus.Errorf("An error occurred attempting to create log file paths: %v", err) + } else { + logrus.Debug("Successfully created log file paths.") + } + } + }() +} + +// CreateLogFiles creates three log files for every service across all running enclaves. +// The first is a file with the name ending in the uuid of the service. +// The other two file paths are symlinks to the uuid file, ending with the shortened uuid and service name respectively. +// If files exist for the shortened uuid and service name files, but they are not symlinks, they are removed and symlink files +// are created to prevent duplicate log storage. +func (manager *LogFileManager) CreateLogFiles(ctx context.Context) error { + var err error + + year, week := manager.time.Now().ISOWeek() + + enclaveToServicesMap, err := manager.getEnclaveAndServiceInfo(ctx) + if err != nil { + // already wrapped with propagate + return err + } + + for enclaveUuid, serviceRegistrations := range enclaveToServicesMap { + for _, serviceRegistration := range serviceRegistrations { + serviceUuidStr := string(serviceRegistration.GetUUID()) + serviceNameStr := string(serviceRegistration.GetName()) + serviceShortUuidStr := uuid_generator.ShortenedUUIDString(serviceUuidStr) + + serviceUuidFilePathStr := getFilepathStr(year, week, string(enclaveUuid), serviceUuidStr) + if err = manager.createLogFileIdempotently(serviceUuidFilePathStr); err != nil { + return err + } + + serviceNameFilePathStr := getFilepathStr(year, week, string(enclaveUuid), serviceNameStr) + if err = manager.createSymlinkLogFile(serviceUuidFilePathStr, serviceNameFilePathStr); err != nil { + return err + } + logrus.Tracef("Created symlinked log file: '%v'", serviceNameFilePathStr) + + serviceShortUuidFilePathStr := getFilepathStr(year, week, string(enclaveUuid), serviceShortUuidStr) + if err = manager.createSymlinkLogFile(serviceUuidFilePathStr, serviceShortUuidFilePathStr); err != nil { + return err + } + logrus.Tracef("Created symlinked log file: '%v'", serviceShortUuidFilePathStr) + } + } + + return nil +} + +// RemoveLogsBeyondRetentionPeriod implements the Job cron interface. It removes logs a week older than the log retention period. +func (manager *LogFileManager) RemoveLogsBeyondRetentionPeriod() { + // compute the next oldest week + year, weekToRemove := manager.time.Now().Add(time.Duration(-volume_consts.LogRetentionPeriodInWeeks) * oneWeek).ISOWeek() + + // remove directory for that week + oldLogsDirPath := getLogsDirPathForWeek(year, weekToRemove) + if err := manager.filesystem.RemoveAll(oldLogsDirPath); err != nil { + logrus.Warnf("An error occurred removing old logs at the following path '%v': %v\n", oldLogsDirPath, err) + } + logrus.Debugf("Removed logs beyond retention period at the following path: '%v'", oldLogsDirPath) +} + +func (manager *LogFileManager) RemoveAllLogs() error { + if err := manager.filesystem.RemoveAll(volume_consts.LogsStorageDirpath); err != nil { + return stacktrace.Propagate(err, "An error occurred attempting to remove all logs.") + } + return nil +} + +func (manager *LogFileManager) RemoveEnclaveLogs(enclaveUuid string) error { + currentTime := manager.time.Now() + for i := 0; i < volume_consts.LogRetentionPeriodInWeeks; i++ { + year, week := currentTime.Add(time.Duration(-i) * oneWeek).ISOWeek() + enclaveLogsDirPathForWeek := getEnclaveLogsDirPath(year, week, enclaveUuid) + if err := manager.filesystem.RemoveAll(enclaveLogsDirPathForWeek); err != nil { + return stacktrace.Propagate(err, "An error occurred attempting to remove logs for enclave '%v' logs at the following path: %v", enclaveUuid, enclaveLogsDirPathForWeek) + } + } + return nil +} + +func (manager *LogFileManager) getEnclaveAndServiceInfo(ctx context.Context) (map[enclave.EnclaveUUID][]*service.ServiceRegistration, error) { + enclaveToServicesMap := map[enclave.EnclaveUUID][]*service.ServiceRegistration{} + + enclaves, err := manager.kurtosisBackend.GetEnclaves(ctx, &enclave.EnclaveFilters{UUIDs: nil, Statuses: nil}) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while trying to get all enclaves from kurtosis backend.") + } + for enclaveUuid := range enclaves { + var serviceRegistrations []*service.ServiceRegistration + + enclaveServices, err := manager.kurtosisBackend.GetUserServices(ctx, enclaveUuid, &service.ServiceFilters{Names: nil, UUIDs: nil, Statuses: nil}) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while trying to get user services for enclave '%v' from kurtosis backend.", enclaveUuid) + } + for _, serviceInfo := range enclaveServices { + serviceRegistrations = append(serviceRegistrations, serviceInfo.GetRegistration()) + } + + enclaveToServicesMap[enclaveUuid] = serviceRegistrations + } + return enclaveToServicesMap, nil +} + +func (manager *LogFileManager) createLogFileIdempotently(logFilePath string) error { + var err error + if _, err = manager.filesystem.Stat(logFilePath); os.IsNotExist(err) { + if _, err = manager.filesystem.Create(logFilePath); err != nil { + return stacktrace.Propagate(err, "An error occurred creating a log file path at '%v'", logFilePath) + } + logrus.Tracef("Created log file: '%v'", logFilePath) + return nil + } + if err != nil { + return stacktrace.Propagate(err, "An error occurred checking if log file path at '%v' existed.", logFilePath) + } + return nil +} + +func (manager *LogFileManager) createSymlinkLogFile(targetLogFilePath, symlinkLogFilePath string) error { + // remove existing log files that could be storing logs at this path + if err := manager.filesystem.RemoveAll(symlinkLogFilePath); err != nil && !errors.IsNotFound(err) { + return stacktrace.Propagate(err, "An error occurred attempting to remove an existing log file at the symlink file path '%v'.", symlinkLogFilePath) + } + // replace with symlink + if err := manager.filesystem.Symlink(targetLogFilePath, symlinkLogFilePath); err != nil { + return stacktrace.Propagate(err, "An error occurred creating a symlink file path '%v' for target file path '%v'.", targetLogFilePath, targetLogFilePath) + } + return nil +} + +// creates a filepath of format //year/week//serviceIdentifier. +func getFilepathStr(year, week int, enclaveUuid, serviceIdentifier string) string { + return fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), strconv.Itoa(week), enclaveUuid, serviceIdentifier, volume_consts.Filetype) +} + +// creates a directory path of format //year/week// +func getEnclaveLogsDirPath(year, week int, enclaveUuid string) string { + logsDirPathForYearAndWeek := getLogsDirPathForWeek(year, week) + return fmt.Sprintf("%s/%s/", logsDirPathForYearAndWeek, enclaveUuid) +} + +// creates a directory path of format //year/week/ +func getLogsDirPathForWeek(year, week int) string { + return fmt.Sprintf(volume_consts.PerWeekDirPathStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), strconv.Itoa(week)) +} diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager_test.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager_test.go new file mode 100644 index 0000000000..18ceb43e90 --- /dev/null +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager_test.go @@ -0,0 +1,180 @@ +package log_file_manager + +import ( + "context" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/container" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/uuid_generator" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/logs_clock" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem" + "github.com/stretchr/testify/require" + "net" + "os" + "testing" +) + +const ( + testEnclaveUuid = "test-enclave" + testUserService1Name = "test-user-service-1" + testUserService1Uuid = "0e10c199bb1a4094839c3ebd432b2c49" + + defaultDay = 0 +) + +func TestRemoveLogsBeyondRetentionPeriod(t *testing.T) { + mockKurtosisBackend := backend_interface.NewMockKurtosisBackend(t) + mockTime := logs_clock.NewMockLogsClock(2023, 2, defaultDay) + + // setup filesystem + mockFs := volume_filesystem.NewMockedVolumeFilesystem() + week49filepath := getFilepathStr(2022, 49, testEnclaveUuid, testUserService1Uuid) + week50filepath := getFilepathStr(2022, 50, testEnclaveUuid, testUserService1Uuid) + week51filepath := getFilepathStr(2022, 51, testEnclaveUuid, testUserService1Uuid) + week52filepath := getFilepathStr(2022, 52, testEnclaveUuid, testUserService1Uuid) + week1filepath := getFilepathStr(2023, 1, testEnclaveUuid, testUserService1Uuid) + week2filepath := getFilepathStr(2023, 2, testEnclaveUuid, testUserService1Uuid) + + _, _ = mockFs.Create(week49filepath) + _, _ = mockFs.Create(week50filepath) + _, _ = mockFs.Create(week51filepath) + _, _ = mockFs.Create(week52filepath) + _, _ = mockFs.Create(week1filepath) + _, _ = mockFs.Create(week2filepath) + + logFileManager := NewLogFileManager(mockKurtosisBackend, mockFs, mockTime) + logFileManager.RemoveLogsBeyondRetentionPeriod() // should remove week 49 logs + + _, err := mockFs.Stat(week49filepath) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) +} + +func TestRemoveEnclaveLogs(t *testing.T) { + mockKurtosisBackend := backend_interface.NewMockKurtosisBackend(t) + mockTime := logs_clock.NewMockLogsClock(2022, 52, defaultDay) + + // setup filesystem + mockFs := volume_filesystem.NewMockedVolumeFilesystem() + + week51filepath := getFilepathStr(2022, 51, testEnclaveUuid, testUserService1Uuid) + week52filepathDiffEnclave := getFilepathStr(2022, 52, "enclaveOne", "serviceTwo") + week52filepath := getFilepathStr(2022, 52, testEnclaveUuid, testUserService1Uuid) + week52filepathDiffService := getFilepathStr(2022, 52, testEnclaveUuid, "serviceThree") + + _, _ = mockFs.Create(week51filepath) + _, _ = mockFs.Create(week52filepathDiffEnclave) + _, _ = mockFs.Create(week52filepath) + _, _ = mockFs.Create(week52filepathDiffService) + + logFileManager := NewLogFileManager(mockKurtosisBackend, mockFs, mockTime) + err := logFileManager.RemoveEnclaveLogs(testEnclaveUuid) // should remove only all log files for enclave one + + require.NoError(t, err) + + _, err = mockFs.Stat(week52filepathDiffEnclave) + require.NoError(t, err) // logs should still exist for different enclave + + _, err = mockFs.Stat(week52filepath) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + + _, err = mockFs.Stat(week51filepath) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + + _, err = mockFs.Stat(week52filepathDiffService) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) +} + +func TestRemoveAllLogs(t *testing.T) { + mockKurtosisBackend := backend_interface.NewMockKurtosisBackend(t) + mockTime := logs_clock.NewMockLogsClock(2022, 52, defaultDay) + + // setup filesystem + mockFs := volume_filesystem.NewMockedVolumeFilesystem() + + week51filepath := getFilepathStr(2022, 51, testEnclaveUuid, testUserService1Uuid) + week52filepathDiffEnclave := getFilepathStr(2022, 52, "enclaveOne", "serviceTwo") + week52filepath := getFilepathStr(2022, 52, testEnclaveUuid, testUserService1Uuid) + week52filepathDiffService := getFilepathStr(2022, 52, testEnclaveUuid, "serviceThree") + + _, _ = mockFs.Create(week51filepath) + _, _ = mockFs.Create(week52filepathDiffEnclave) + _, _ = mockFs.Create(week52filepath) + _, _ = mockFs.Create(week52filepathDiffService) + + logFileManager := NewLogFileManager(mockKurtosisBackend, mockFs, mockTime) + err := logFileManager.RemoveAllLogs() + + require.NoError(t, err) + + _, err = mockFs.Stat(week52filepathDiffEnclave) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + + _, err = mockFs.Stat(week52filepath) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + + _, err = mockFs.Stat(week51filepath) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + + _, err = mockFs.Stat(week52filepathDiffService) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) +} + +func TestCreateLogFiles(t *testing.T) { + mockTime := logs_clock.NewMockLogsClock(2022, 52, defaultDay) + mockFs := volume_filesystem.NewMockedVolumeFilesystem() + + // setup kurtosis backend + ctx := context.Background() + mockKurtosisBackend := backend_interface.NewMockKurtosisBackend(t) + + // mock enclave + enclaveUuid := enclave.EnclaveUUID(testEnclaveUuid) + enclaveStatus := enclave.EnclaveStatus_Running + enclaveCreationTime := mockTime.Now() // time doesn't matter + enclaveMap := map[enclave.EnclaveUUID]*enclave.Enclave{ + enclaveUuid: enclave.NewEnclave(enclaveUuid, testEnclaveUuid, enclaveStatus, &enclaveCreationTime, false), + } + + mockKurtosisBackend. + EXPECT(). + GetEnclaves(ctx, &enclave.EnclaveFilters{UUIDs: nil, Statuses: nil}). + Return(enclaveMap, nil) + + serviceUuid := service.ServiceUUID(testUserService1Uuid) + serviceRegistration := service.NewServiceRegistration(testUserService1Name, serviceUuid, enclaveUuid, net.IP{}, "") + serviceContainer := container.NewContainer(container.ContainerStatus_Running, "", []string{}, []string{}, map[string]string{}) + servicesMap := map[service.ServiceUUID]*service.Service{ + serviceUuid: service.NewService(serviceRegistration, map[string]*port_spec.PortSpec{}, net.IP{}, map[string]*port_spec.PortSpec{}, serviceContainer), + } + mockKurtosisBackend. + EXPECT(). + GetUserServices(ctx, enclaveUuid, &service.ServiceFilters{Names: nil, UUIDs: nil, Statuses: nil}). + Return(servicesMap, nil) + + expectedServiceUuidFilePath := getFilepathStr(2022, 52, testEnclaveUuid, testUserService1Uuid) + expectedServiceNameFilePath := getFilepathStr(2022, 52, testEnclaveUuid, testUserService1Name) + expectedServiceShortUuidFilePath := getFilepathStr(2022, 52, testEnclaveUuid, uuid_generator.ShortenedUUIDString(testUserService1Uuid)) + + logFileManager := NewLogFileManager(mockKurtosisBackend, mockFs, mockTime) + err := logFileManager.CreateLogFiles(ctx) + require.NoError(t, err) + + _, err = mockFs.Stat(expectedServiceUuidFilePath) + require.NoError(t, err) + + _, err = mockFs.Stat(expectedServiceNameFilePath) + require.NoError(t, err) + + _, err = mockFs.Stat(expectedServiceShortUuidFilePath) + require.NoError(t, err) +} diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_remover/log_remover.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_remover/log_remover.go deleted file mode 100644 index 9ecf1e6ea5..0000000000 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_remover/log_remover.go +++ /dev/null @@ -1,44 +0,0 @@ -package log_remover - -import ( - "fmt" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/logs_clock" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem" - "github.com/sirupsen/logrus" - "strconv" - "time" -) - -const ( - oneWeek = 7 * 24 * time.Hour -) - -// LogRemover removes logs one week older than the log retention period. -type LogRemover struct { - filesystem volume_filesystem.VolumeFilesystem - - time logs_clock.LogsClock -} - -func NewLogRemover(filesystem volume_filesystem.VolumeFilesystem, time logs_clock.LogsClock) *LogRemover { - return &LogRemover{ - filesystem: filesystem, - time: time, - } -} - -// Run implements the Job cron interface. It removes logs a week older than the log retention period. -func (remover LogRemover) Run() { - // [LogRetentionPeriodInWeeks] weeks plus an extra week of logs are retained so remove logs a week past that, hence +1 - numWeeksBack := volume_consts.LogRetentionPeriodInWeeks + 1 - - // compute the next oldest week - year, weekToRemove := remover.time.Now().Add(time.Duration(-numWeeksBack) * oneWeek).ISOWeek() - - // remove directory for that week - oldLogsDirPath := fmt.Sprintf(volume_consts.PerWeekDirPathStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), strconv.Itoa(weekToRemove)) - if err := remover.filesystem.RemoveAll(oldLogsDirPath); err != nil { - logrus.Warnf("An error occurred removing old logs at the following path '%v': %v\n", oldLogsDirPath, err) - } -} diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_remover/log_remover_test.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_remover/log_remover_test.go deleted file mode 100644 index ce4cbced91..0000000000 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_remover/log_remover_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package log_remover - -import ( - "fmt" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/logs_clock" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem" - "github.com/stretchr/testify/require" - "os" - "strconv" - "testing" -) - -const ( - testEnclaveUuid = "test-enclave" - testUserService1Uuid = "test-user-service-1" - - defaultDay = 0 -) - -func TestLogRemover_Run(t *testing.T) { - mockFs := volume_filesystem.NewMockedVolumeFilesystem() - - week49filepath := getWeekFilepathStr(2022, 49) - week50filepath := getWeekFilepathStr(2022, 50) - week51filepath := getWeekFilepathStr(2022, 51) - week52filepath := getWeekFilepathStr(2022, 52) - week1filepath := getWeekFilepathStr(2023, 1) - week2filepath := getWeekFilepathStr(2023, 2) - - _, _ = mockFs.Create(week49filepath) - _, _ = mockFs.Create(week50filepath) - _, _ = mockFs.Create(week51filepath) - _, _ = mockFs.Create(week52filepath) - _, _ = mockFs.Create(week1filepath) - _, _ = mockFs.Create(week2filepath) - - currentWeek := 2 - - mockTime := logs_clock.NewMockLogsClock(2023, currentWeek, defaultDay) - logRemover := NewLogRemover(mockFs, mockTime) - - // log remover should remove week 49 logs - logRemover.Run() - - _, err := mockFs.Stat(week49filepath) - require.Error(t, err) - require.True(t, os.IsNotExist(err)) -} - -func getWeekFilepathStr(year, week int) string { - return fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), strconv.Itoa(week), testEnclaveUuid, testUserService1Uuid, volume_consts.Filetype) -} diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy.go index 43cf059118..6d06700b6f 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy.go @@ -67,11 +67,11 @@ func (strategy *PerWeekStreamLogsStrategy) StreamLogs( serviceUuid, enclaveUuid) return } - if len(paths) > volume_consts.LogRetentionPeriodInWeeks+1 { + if len(paths) > volume_consts.LogRetentionPeriodInWeeks { logrus.Warnf( `We expected to retrieve logs going back '%v' weeks, but instead retrieved logs going back '%v' weeks. This means logs past the retention period are being returned, likely a bug in Kurtosis.`, - volume_consts.LogRetentionPeriodInWeeks+1, len(paths)) + volume_consts.LogRetentionPeriodInWeeks, len(paths)) } logsReader, files, err := getLogsReader(fs, paths) @@ -108,10 +108,9 @@ func (strategy *PerWeekStreamLogsStrategy) StreamLogs( } // [getLogFilePaths] returns a list of log file paths containing logs for [serviceUuid] in [enclaveUuid] -// going back ([retentionPeriodInWeeks] + 1) week back from the [currentWeek]. +// going [retentionPeriodInWeeks] back from the [currentWeek]. // Notes: // - File paths are of the format '/week/enclave uuid/service uuid.json' where 'week' is %W strftime specifier -// - The +1 is because we retain an extra week of logs compared to what we promise to retain for safety. // - The list of file paths is returned in order of oldest logs to most recent logs e.g. [ 3/80124/1234.json, /4/801234/1234.json, ...] // - If a file path does not exist, the function with exits and returns whatever file paths were found func (strategy *PerWeekStreamLogsStrategy) getLogFilePaths(filesystem volume_filesystem.VolumeFilesystem, retentionPeriodInWeeks int, enclaveUuid, serviceUuid string) ([]string, error) { @@ -120,7 +119,7 @@ func (strategy *PerWeekStreamLogsStrategy) getLogFilePaths(filesystem volume_fil // scan for first existing log file firstWeekWithLogs := 0 - for i := 0; i < (retentionPeriodInWeeks + 1); i++ { + for i := 0; i < retentionPeriodInWeeks; i++ { year, week := currentTime.Add(time.Duration(-i) * oneWeek).ISOWeek() filePathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), strconv.Itoa(week), enclaveUuid, serviceUuid, volume_consts.Filetype) if _, err := filesystem.Stat(filePathStr); err == nil { @@ -136,7 +135,7 @@ func (strategy *PerWeekStreamLogsStrategy) getLogFilePaths(filesystem volume_fil } // scan for remaining files as far back as they exist - for i := firstWeekWithLogs + 1; i < (retentionPeriodInWeeks + 1); i++ { + for i := firstWeekWithLogs + 1; i < retentionPeriodInWeeks; i++ { year, week := currentTime.Add(time.Duration(-i) * oneWeek).ISOWeek() filePathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), strconv.Itoa(week), enclaveUuid, serviceUuid, volume_consts.Filetype) if _, err := filesystem.Stat(filePathStr); err != nil { @@ -337,7 +336,7 @@ func (strategy *PerWeekStreamLogsStrategy) sendJsonLogLine( // Returns true if [logLine] has no timestamp func (strategy *PerWeekStreamLogsStrategy) isWithinRetentionPeriod(logLine JsonLog) (bool, error) { - retentionPeriod := strategy.time.Now().Add(time.Duration(-volume_consts.LogRetentionPeriodInWeeks-1) * oneWeek) + retentionPeriod := strategy.time.Now().Add(time.Duration(-volume_consts.LogRetentionPeriodInWeeks) * oneWeek) timestampStr, found := logLine[volume_consts.TimestampLabel] if !found { return true, nil diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy_test.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy_test.go index c46d49942c..a81d090c19 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy_test.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy_test.go @@ -149,7 +149,7 @@ func TestGetLogFilePathsWithDiffRetentionPeriod(t *testing.T) { _, _ = filesystem.Create(week2filepath) currentWeek := 2 - retentionPeriod := 2 + retentionPeriod := 3 expectedLogFilePaths := []string{ week52filepath, diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts/consts.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts/consts.go index 253e44a6c0..5f0467f999 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts/consts.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts/consts.go @@ -17,11 +17,12 @@ const ( EndOfJsonLine = "}" - LogRetentionPeriodInWeeks = 4 + // promise to keep 4 weeks of logs for users, but store an additional week for safety + LogRetentionPeriodInWeeks = 5 RemoveLogsWaitHours = 6 * time.Hour - CreateLogFilesInterval = 3 * time.Minute + CreateLogsWaitMinutes = 3 * time.Minute // basepath/enclave uuid/service uuid PerFileFmtStr = "%s%s/%s%s" diff --git a/engine/server/engine/enclave_manager/enclave_manager.go b/engine/server/engine/enclave_manager/enclave_manager.go index f0d010f965..0c68dc8716 100644 --- a/engine/server/engine/enclave_manager/enclave_manager.go +++ b/engine/server/engine/enclave_manager/enclave_manager.go @@ -3,6 +3,7 @@ package enclave_manager import ( "context" "fmt" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager" "sort" "strings" "sync" @@ -60,9 +61,10 @@ type EnclaveManager struct { // this is an append only list allExistingAndHistoricalIdentifiers []*kurtosis_engine_rpc_api_bindings.EnclaveIdentifiers - enclaveCreator *EnclaveCreator - enclavePool *EnclavePool - enclaveEnvVars string + enclaveCreator *EnclaveCreator + enclavePool *EnclavePool + enclaveEnvVars string + enclaveLogFileManager *log_file_manager.LogFileManager metricsUserID string didUserAcceptSendingMetrics bool @@ -75,6 +77,7 @@ func CreateEnclaveManager( engineVersion string, poolSize uint8, enclaveEnvVars string, + enclaveLogFileManager *log_file_manager.LogFileManager, metricsUserID string, didUserAcceptSendingMetrics bool, ) (*EnclaveManager, error) { @@ -101,6 +104,7 @@ func CreateEnclaveManager( enclaveCreator: enclaveCreator, enclavePool: enclavePool, enclaveEnvVars: enclaveEnvVars, + enclaveLogFileManager: enclaveLogFileManager, metricsUserID: metricsUserID, didUserAcceptSendingMetrics: didUserAcceptSendingMetrics, } @@ -253,6 +257,9 @@ func (manager *EnclaveManager) DestroyEnclave(ctx context.Context, enclaveIdenti return stacktrace.Propagate(err, "An error occurred destroying the enclave") } if _, found := successfullyDestroyedEnclaves[enclaveUuid]; found { + if err = manager.enclaveLogFileManager.RemoveEnclaveLogs(string(enclaveUuid)); err != nil { + return stacktrace.Propagate(err, "An error occurred attempting to remove enclave '%v' logs after it was destroyed.", enclaveIdentifier) + } return nil } destructionErr, found := erroredEnclaves[enclaveUuid] @@ -479,14 +486,19 @@ func (manager *EnclaveManager) cleanEnclaves( return nil, nil, stacktrace.Propagate(err, "An error occurred destroying enclaves during cleaning") } + enclaveDestructionErrors := []error{} + for _, destructionError := range erroredEnclaves { + enclaveDestructionErrors = append(enclaveDestructionErrors, destructionError) + } + successfullyDestroyedEnclaveIdStrs := []string{} for enclaveId := range successfullyDestroyedEnclaves { successfullyDestroyedEnclaveIdStrs = append(successfullyDestroyedEnclaveIdStrs, string(enclaveId)) - } - enclaveDestructionErrors := []error{} - for _, destructionError := range erroredEnclaves { - enclaveDestructionErrors = append(enclaveDestructionErrors, destructionError) + if err := manager.enclaveLogFileManager.RemoveEnclaveLogs(string(enclaveId)); err != nil { + logRemovalErr := stacktrace.Propagate(err, "An error occurred removing enclave '%v' logs.", enclaveId) + enclaveDestructionErrors = append(enclaveDestructionErrors, logRemovalErr) + } } return successfullyDestroyedEnclaveIdStrs, enclaveDestructionErrors, nil diff --git a/engine/server/engine/main.go b/engine/server/engine/main.go index c11d558c25..ca6763bce1 100644 --- a/engine/server/engine/main.go +++ b/engine/server/engine/main.go @@ -20,10 +20,9 @@ import ( "github.com/kurtosis-tech/kurtosis/engine/launcher/args" "github.com/kurtosis-tech/kurtosis/engine/launcher/args/kurtosis_backend_config" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_creator" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/logs_clock" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem" "github.com/kurtosis-tech/kurtosis/engine/server/engine/enclave_manager" "github.com/kurtosis-tech/kurtosis/engine/server/engine/server" @@ -136,37 +135,26 @@ func runMain() error { return stacktrace.Propagate(err, "An error occurred getting the Kurtosis backend for backend type '%v' and config '%+v'", serverArgs.KurtosisBackendType, backendConfig) } - enclaveManager, err := getEnclaveManager(kurtosisBackend, serverArgs.KurtosisBackendType, serverArgs.ImageVersionTag, serverArgs.PoolSize, serverArgs.EnclaveEnvVars, serverArgs.MetricsUserID, serverArgs.DidUserAcceptSendingMetrics) - if err != nil { - return stacktrace.Propagate(err, "Failed to create an enclave manager for backend type '%v' and config '%+v'", serverArgs.KurtosisBackendType, backendConfig) - } - - // osFs is a wrapper around disk osFs := volume_filesystem.NewOsVolumeFilesystem() + realTime := logs_clock.NewRealClock() + + // TODO: remove once users are fully migrated to log retention/new log schema // pulls logs per enclave/per service id perFileStreamStrategy := stream_logs_strategy.NewPerFileStreamLogsStrategy() perFileLogsDatabaseClient := persistent_volume.NewPersistentVolumeLogsDatabaseClient(kurtosisBackend, osFs, perFileStreamStrategy) // pulls logs /per week/per enclave/per service - realTime := logs_clock.NewRealClock() perWeekStreamStrategy := stream_logs_strategy.NewPerWeekStreamLogsStrategy(realTime) perWeekLogsDatabaseClient := persistent_volume.NewPersistentVolumeLogsDatabaseClient(kurtosisBackend, osFs, perWeekStreamStrategy) - go func() { - // TODO: Remove this when moving away from persistent volume logs db - // creating log file paths on an interval is a hack to prevent duplicate logs from being stored by the log aggregator - fileCreator := log_file_creator.NewLogFileCreator(kurtosisBackend, osFs, realTime) - logFileCreatorTicker := time.NewTicker(volume_consts.CreateLogFilesInterval) - for range logFileCreatorTicker.C { - logrus.Debug("Creating log file paths...") - err = fileCreator.CreateLogFiles(ctx) - if err != nil { - logrus.Errorf("An error occurred attempting to create log file paths: %v", err) - } else { - logrus.Debug("Successfully created log file paths.") - } - } - }() + // TODO: Move logFileManager into LogsDatabaseClient + logFileManager := log_file_manager.NewLogFileManager(kurtosisBackend, osFs, realTime) + logFileManager.StartLogFileManagement(ctx) + + enclaveManager, err := getEnclaveManager(kurtosisBackend, serverArgs.KurtosisBackendType, serverArgs.ImageVersionTag, serverArgs.PoolSize, serverArgs.EnclaveEnvVars, logFileManager, serverArgs.MetricsUserID, serverArgs.DidUserAcceptSendingMetrics) + if err != nil { + return stacktrace.Propagate(err, "Failed to create an enclave manager for backend type '%v' and config '%+v'", serverArgs.KurtosisBackendType, backendConfig) + } go func() { fileServer := http.FileServer(http.Dir(pathToStaticFolder)) @@ -237,6 +225,7 @@ func getEnclaveManager( engineVersion string, poolSize uint8, enclaveEnvVars string, + enclaveLogFileManager *log_file_manager.LogFileManager, metricsUserID string, didUserAcceptSendingMetrics bool, ) (*enclave_manager.EnclaveManager, error) { @@ -257,6 +246,7 @@ func getEnclaveManager( engineVersion, poolSize, enclaveEnvVars, + enclaveLogFileManager, metricsUserID, didUserAcceptSendingMetrics, ) diff --git a/engine/server/engine/server/engine_connect_server_service.go b/engine/server/engine/server/engine_connect_server_service.go index 9d95ac7f07..ee8bbd7ecd 100644 --- a/engine/server/engine/server/engine_connect_server_service.go +++ b/engine/server/engine/server/engine_connect_server_service.go @@ -143,7 +143,6 @@ func (service *EngineConnectServerService) Clean(ctx context.Context, connectArg if err != nil { return nil, stacktrace.Propagate(err, "An error occurred while cleaning enclaves") } - response := &kurtosis_engine_rpc_api_bindings.CleanResponse{RemovedEnclaveNameAndUuids: removedEnclaveUuidsAndNames} return connect.NewResponse(response), nil }