-
Notifications
You must be signed in to change notification settings - Fork 0
feat(daemon): add daily cleanup timer service for large log files #161
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
723e1f8
6cbbcb3
8055c17
af725dd
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -13,8 +13,6 @@ import ( | |||||||||||
| "go.opentelemetry.io/otel/trace" | ||||||||||||
| ) | ||||||||||||
|
|
||||||||||||
| const logFileSizeThreshold int64 = 50 * 1024 * 1024 // 50 MB | ||||||||||||
|
|
||||||||||||
| var GCCommand *cli.Command = &cli.Command{ | ||||||||||||
| Name: "gc", | ||||||||||||
| Usage: "clean internal storage", | ||||||||||||
|
|
@@ -34,52 +32,6 @@ var GCCommand *cli.Command = &cli.Command{ | |||||||||||
| Action: commandGC, | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| // cleanLogFile removes a log file if it exceeds the threshold or if force is true. | ||||||||||||
| // Returns the size of the deleted file (0 if not deleted or file doesn't exist). | ||||||||||||
| func cleanLogFile(filePath string, threshold int64, force bool) (int64, error) { | ||||||||||||
| info, err := os.Stat(filePath) | ||||||||||||
| if os.IsNotExist(err) { | ||||||||||||
| return 0, nil | ||||||||||||
| } | ||||||||||||
| if err != nil { | ||||||||||||
| return 0, fmt.Errorf("failed to stat file %s: %w", filePath, err) | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| fileSize := info.Size() | ||||||||||||
| if !force && fileSize < threshold { | ||||||||||||
| return 0, nil | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| if err := os.Remove(filePath); err != nil { | ||||||||||||
| return 0, fmt.Errorf("failed to remove file %s: %w", filePath, err) | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| slog.Info("cleaned log file", slog.String("file", filePath), slog.Int64("size_bytes", fileSize)) | ||||||||||||
| return fileSize, nil | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| // cleanLargeLogFiles checks all log files and removes those exceeding the size threshold. | ||||||||||||
| // If force is true, removes all log files regardless of size. | ||||||||||||
| func cleanLargeLogFiles(force bool) (int64, error) { | ||||||||||||
| logFiles := []string{ | ||||||||||||
| model.GetLogFilePath(), | ||||||||||||
| model.GetHeartbeatLogFilePath(), | ||||||||||||
| model.GetSyncPendingFilePath(), | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| var totalFreed int64 | ||||||||||||
| for _, filePath := range logFiles { | ||||||||||||
| freed, err := cleanLogFile(filePath, logFileSizeThreshold, force) | ||||||||||||
| if err != nil { | ||||||||||||
| slog.Warn("failed to clean log file", slog.String("file", filePath), slog.Any("err", err)) | ||||||||||||
| continue | ||||||||||||
| } | ||||||||||||
| totalFreed += freed | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| return totalFreed, nil | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| // backupAndWriteFile backs up the existing file and writes new content. | ||||||||||||
| func backupAndWriteFile(filePath string, content []byte) error { | ||||||||||||
| backupFile := filePath + ".bak" | ||||||||||||
|
|
@@ -231,9 +183,17 @@ func commandGC(c *cli.Context) error { | |||||||||||
| return nil | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| // Get config for threshold | ||||||||||||
| cfg, err := configService.ReadConfigFile(ctx) | ||||||||||||
| if err != nil { | ||||||||||||
| slog.Warn("failed to read config, using default threshold", slog.Any("err", err)) | ||||||||||||
| cfg.LogCleanup = &model.LogCleanup{ThresholdMB: 100} | ||||||||||||
| } | ||||||||||||
| thresholdBytes := cfg.LogCleanup.ThresholdMB * 1024 * 1024 | ||||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The magic number multiplication |
||||||||||||
|
|
||||||||||||
| // Clean log files: force clean if --withLog, otherwise only clean large files | ||||||||||||
| forceCleanLogs := c.Bool("withLog") | ||||||||||||
| freedBytes, err := cleanLargeLogFiles(forceCleanLogs) | ||||||||||||
| freedBytes, err := model.CleanLargeLogFiles(thresholdBytes, forceCleanLogs) | ||||||||||||
| if err != nil { | ||||||||||||
| slog.Warn("error during log cleanup", slog.Any("err", err)) | ||||||||||||
| } | ||||||||||||
|
Comment on lines
+196
to
199
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Following the proposed change in
Suggested change
|
||||||||||||
|
|
||||||||||||
| Original file line number | Diff line number | Diff line change | ||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1,103 @@ | ||||||||||||
| package daemon | ||||||||||||
|
|
||||||||||||
| import ( | ||||||||||||
| "context" | ||||||||||||
| "log/slog" | ||||||||||||
| "sync" | ||||||||||||
| "time" | ||||||||||||
|
|
||||||||||||
| "github.com/malamtime/cli/model" | ||||||||||||
| ) | ||||||||||||
|
|
||||||||||||
| const ( | ||||||||||||
| // CleanupInterval is the interval for log cleanup (24 hours) | ||||||||||||
| CleanupInterval = 24 * time.Hour | ||||||||||||
| ) | ||||||||||||
|
|
||||||||||||
| // CleanupTimerService handles periodic cleanup of large log files | ||||||||||||
| type CleanupTimerService struct { | ||||||||||||
| config model.ShellTimeConfig | ||||||||||||
| ticker *time.Ticker | ||||||||||||
| stopChan chan struct{} | ||||||||||||
| wg sync.WaitGroup | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| // NewCleanupTimerService creates a new cleanup timer service | ||||||||||||
| func NewCleanupTimerService(config model.ShellTimeConfig) *CleanupTimerService { | ||||||||||||
| return &CleanupTimerService{ | ||||||||||||
| config: config, | ||||||||||||
| stopChan: make(chan struct{}), | ||||||||||||
| } | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| // Start begins the periodic cleanup job | ||||||||||||
| func (s *CleanupTimerService) Start(ctx context.Context) error { | ||||||||||||
| s.ticker = time.NewTicker(CleanupInterval) | ||||||||||||
| s.wg.Add(1) | ||||||||||||
|
|
||||||||||||
| go func() { | ||||||||||||
| defer s.wg.Done() | ||||||||||||
|
|
||||||||||||
| // NOTE: Do not run at startup, only on timer | ||||||||||||
| // This avoids slowing daemon startup and prevents cleanup on restart loops | ||||||||||||
|
|
||||||||||||
| for { | ||||||||||||
| select { | ||||||||||||
| case <-s.ticker.C: | ||||||||||||
| s.cleanup(ctx) | ||||||||||||
| case <-s.stopChan: | ||||||||||||
| return | ||||||||||||
| case <-ctx.Done(): | ||||||||||||
| return | ||||||||||||
| } | ||||||||||||
| } | ||||||||||||
| }() | ||||||||||||
|
|
||||||||||||
| slog.Info("Cleanup timer service started", | ||||||||||||
| slog.Duration("interval", CleanupInterval), | ||||||||||||
| slog.Int64("thresholdMB", s.config.LogCleanup.ThresholdMB)) | ||||||||||||
| return nil | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| // Stop stops the cleanup service | ||||||||||||
| func (s *CleanupTimerService) Stop() { | ||||||||||||
| if s.ticker != nil { | ||||||||||||
| s.ticker.Stop() | ||||||||||||
| } | ||||||||||||
| close(s.stopChan) | ||||||||||||
| s.wg.Wait() | ||||||||||||
| slog.Info("Cleanup timer service stopped") | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| // cleanup performs the log cleanup | ||||||||||||
| func (s *CleanupTimerService) cleanup(ctx context.Context) { | ||||||||||||
| thresholdBytes := s.config.LogCleanup.ThresholdMB * 1024 * 1024 | ||||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The magic number multiplication |
||||||||||||
|
|
||||||||||||
| slog.Debug("Starting scheduled log cleanup", | ||||||||||||
| slog.Int64("thresholdMB", s.config.LogCleanup.ThresholdMB)) | ||||||||||||
|
|
||||||||||||
| var totalFreed int64 | ||||||||||||
|
|
||||||||||||
| // Clean CLI log files | ||||||||||||
| freedCLI, err := model.CleanLargeLogFiles(thresholdBytes, false) | ||||||||||||
| if err != nil { | ||||||||||||
| slog.Warn("error during CLI log cleanup", slog.Any("err", err)) | ||||||||||||
| } | ||||||||||||
|
Comment on lines
+82
to
+85
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Following the proposed change in
Suggested change
|
||||||||||||
| totalFreed += freedCLI | ||||||||||||
|
|
||||||||||||
| // Clean daemon log files (macOS only) | ||||||||||||
| freedDaemon, err := model.CleanDaemonLogFiles(thresholdBytes, false) | ||||||||||||
| if err != nil { | ||||||||||||
| slog.Warn("error during daemon log cleanup", slog.Any("err", err)) | ||||||||||||
| } | ||||||||||||
|
Comment on lines
+89
to
+92
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Following the proposed change in
Suggested change
|
||||||||||||
| totalFreed += freedDaemon | ||||||||||||
|
|
||||||||||||
| if totalFreed > 0 { | ||||||||||||
| slog.Info("scheduled log cleanup completed", | ||||||||||||
| slog.Int64("totalFreedBytes", totalFreed), | ||||||||||||
| slog.Int64("cliFreedBytes", freedCLI), | ||||||||||||
| slog.Int64("daemonFreedBytes", freedDaemon)) | ||||||||||||
| } else { | ||||||||||||
| slog.Debug("scheduled log cleanup completed, no files exceeded threshold") | ||||||||||||
| } | ||||||||||||
| } | ||||||||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The
ReadConfigFilefunction ensures thatcfg.LogCleanupandcfg.LogCleanup.Enabledare initialized with default values and are nevernil. Therefore, this check can be simplified to just check the boolean value.