Skip to content

Commit

Permalink
logging: added --log-dir-max-total-size-mb flag
Browse files Browse the repository at this point in the history 
This limits the total size of all logs in a directory to 1 GB.
  • Loading branch information
@jkowalski
jkowalski committed Dec 3, 2021
1 parent 1f9c40d commit 0ce3ec0
Show file tree
Hide file tree
Showing 2 changed files with 101 additions and 24 deletions.
57 changes: 33 additions & 24 deletions internal/logfile/logfile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,23 +31,25 @@ const logsDirMode = 0o700
var logLevels = []string{"debug", "info", "warning", "error"}

type loggingFlags struct {
logFile string
contentLogFile string
logDir string
logDirMaxFiles int
logDirMaxAge time.Duration
contentLogDirMaxFiles int
contentLogDirMaxAge time.Duration
logFileMaxSegmentSize int
logLevel string
fileLogLevel string
fileLogLocalTimezone bool
jsonLogFile bool
jsonLogConsole bool
forceColor bool
disableColor bool
consoleLogTimestamps bool
waitForLogSweep bool
logFile string
contentLogFile string
logDir string
logDirMaxFiles int
logDirMaxAge time.Duration
logDirMaxTotalSizeMB float64
contentLogDirMaxFiles int
contentLogDirMaxAge time.Duration
contentLogDirMaxTotalSizeMB float64
logFileMaxSegmentSize int
logLevel string
fileLogLevel string
fileLogLocalTimezone bool
jsonLogFile bool
jsonLogConsole bool
forceColor bool
disableColor bool
consoleLogTimestamps bool
waitForLogSweep bool

cliApp *cli.App
}
Expand All @@ -59,10 +61,12 @@ func (c *loggingFlags) setup(cliApp *cli.App, app *kingpin.Application) {
app.Flag("log-dir", "Directory where log files should be written.").Envar("KOPIA_LOG_DIR").Default(ospath.LogsDir()).StringVar(&c.logDir)
app.Flag("log-dir-max-files", "Maximum number of log files to retain").Envar("KOPIA_LOG_DIR_MAX_FILES").Default("1000").Hidden().IntVar(&c.logDirMaxFiles)
app.Flag("log-dir-max-age", "Maximum age of log files to retain").Envar("KOPIA_LOG_DIR_MAX_AGE").Hidden().Default("720h").DurationVar(&c.logDirMaxAge)
app.Flag("max-log-file-segment-size", "Maximum size of log segment").Envar("KOPIA_LOG_FILE_MAX_SEGMENT_SIZE").Default("50000000").Hidden().IntVar(&c.logFileMaxSegmentSize)
app.Flag("log-dir-max-total-size-mb", "Maximum total size of log files to retain").Envar("KOPIA_LOG_DIR_MAX_SIZE_MB").Hidden().Default("1000").Float64Var(&c.logDirMaxTotalSizeMB)
app.Flag("max-log-file-segment-size", "Maximum size of a single log file segment").Envar("KOPIA_LOG_FILE_MAX_SEGMENT_SIZE").Default("50000000").Hidden().IntVar(&c.logFileMaxSegmentSize)
app.Flag("wait-for-log-sweep", "Wait for log sweep before program exit").Default("true").Hidden().BoolVar(&c.waitForLogSweep)
app.Flag("content-log-dir-max-files", "Maximum number of content log files to retain").Envar("KOPIA_CONTENT_LOG_DIR_MAX_FILES").Default("5000").Hidden().IntVar(&c.contentLogDirMaxFiles)
app.Flag("content-log-dir-max-age", "Maximum age of content log files to retain").Envar("KOPIA_CONTENT_LOG_DIR_MAX_AGE").Default("720h").Hidden().DurationVar(&c.contentLogDirMaxAge)
app.Flag("content-log-dir-max-total-size-mb", "Maximum total size of log files to retain").Envar("KOPIA_CONTENT_LOG_DIR_MAX_SIZE_MB").Hidden().Default("1000").Float64Var(&c.contentLogDirMaxTotalSizeMB)
app.Flag("log-level", "Console log level").Default("info").EnumVar(&c.logLevel, logLevels...)
app.Flag("json-log-console", "JSON log file").Hidden().BoolVar(&c.jsonLogConsole)
app.Flag("json-log-file", "JSON log file").Hidden().BoolVar(&c.jsonLogFile)
Expand Down Expand Up @@ -182,7 +186,7 @@ func (c *loggingFlags) setupConsoleCore() zapcore.Core {
)
}

func (c *loggingFlags) setupLogFileBasedLogger(now time.Time, subdir, suffix, logFileOverride string, maxFiles int, maxAge time.Duration) zapcore.WriteSyncer {
func (c *loggingFlags) setupLogFileBasedLogger(now time.Time, subdir, suffix, logFileOverride string, maxFiles int, maxSizeMB float64, maxAge time.Duration) zapcore.WriteSyncer {
var logFileName, symlinkName string

if logFileOverride != "" {
Expand Down Expand Up @@ -213,7 +217,7 @@ func (c *loggingFlags) setupLogFileBasedLogger(now time.Time, subdir, suffix, lo
// do not scrub directory if custom log file has been provided.
if logFileOverride == "" && shouldSweepLog(maxFiles, maxAge) {
doSweep = func() {
sweepLogDir(context.TODO(), logDir, maxFiles, maxAge)
sweepLogDir(context.TODO(), logDir, maxFiles, maxSizeMB, maxAge)
}
}

Expand Down Expand Up @@ -258,7 +262,7 @@ func (c *loggingFlags) setupLogFileCore(now time.Time, suffix string) zapcore.Co
EncodeDuration: zapcore.StringDurationEncoder,
ConsoleSeparator: " ",
}, c.jsonLogFile),
c.setupLogFileBasedLogger(now, "cli-logs", suffix, c.logFile, c.logDirMaxFiles, c.logDirMaxAge),
c.setupLogFileBasedLogger(now, "cli-logs", suffix, c.logFile, c.logDirMaxFiles, c.logDirMaxTotalSizeMB, c.logDirMaxAge),
logLevelFromFlag(c.fileLogLevel),
)
}
Expand All @@ -281,15 +285,15 @@ func (c *loggingFlags) setupContentLogFileBackend(now time.Time, suffix string)
EncodeDuration: zapcore.StringDurationEncoder,
ConsoleSeparator: " ",
}),
c.setupLogFileBasedLogger(now, "content-logs", suffix, c.contentLogFile, c.contentLogDirMaxFiles, c.contentLogDirMaxAge),
c.setupLogFileBasedLogger(now, "content-logs", suffix, c.contentLogFile, c.contentLogDirMaxFiles, c.contentLogDirMaxTotalSizeMB, c.contentLogDirMaxAge),
zap.DebugLevel)
}

func shouldSweepLog(maxFiles int, maxAge time.Duration) bool {
return maxFiles > 0 || maxAge > 0
}

func sweepLogDir(ctx context.Context, dirname string, maxCount int, maxAge time.Duration) {
func sweepLogDir(ctx context.Context, dirname string, maxCount int, maxSizeMB float64, maxAge time.Duration) {
var timeCutoff time.Time
if maxAge > 0 {
timeCutoff = clock.Now().Add(-maxAge)
Expand All @@ -299,6 +303,8 @@ func sweepLogDir(ctx context.Context, dirname string, maxCount int, maxAge time.
maxCount = math.MaxInt32
}

maxTotalSizeBytes := int64(maxSizeMB * 1e6)

entries, err := os.ReadDir(dirname)
if err != nil {
log(ctx).Errorf("unable to read log directory: %v", err)
Expand Down Expand Up @@ -327,6 +333,7 @@ func sweepLogDir(ctx context.Context, dirname string, maxCount int, maxAge time.
})

cnt := 0
totalSize := int64(0)

for _, fi := range fileInfos {
if !strings.HasPrefix(fi.Name(), logFileNamePrefix) {
Expand All @@ -339,7 +346,9 @@ func sweepLogDir(ctx context.Context, dirname string, maxCount int, maxAge time.

cnt++

if cnt > maxCount || fi.ModTime().Before(timeCutoff) {
totalSize += fi.Size()

if cnt > maxCount || totalSize > maxTotalSizeBytes || fi.ModTime().Before(timeCutoff) {
if err = os.Remove(filepath.Join(dirname, fi.Name())); err != nil && !os.IsNotExist(err) {
log(ctx).Errorf("unable to remove log file: %v", err)
}
Expand Down
68 changes: 68 additions & 0 deletions internal/logfile/logfile_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package logfile_test

import (
"bufio"
"fmt"
"os"
"path/filepath"
"regexp"
Expand All @@ -14,6 +15,7 @@ import (
"github.com/kopia/kopia/internal/clock"
"github.com/kopia/kopia/internal/logfile"
"github.com/kopia/kopia/internal/testutil"
"github.com/kopia/kopia/tests/testdirtree"
"github.com/kopia/kopia/tests/testenv"
)

Expand Down Expand Up @@ -143,6 +145,50 @@ func TestLogFileRotation(t *testing.T) {
}
}

func TestLogFileMaxTotalSize(t *testing.T) {
runner := testenv.NewInProcRunner(t)
runner.CustomizeApp = logfile.Attach

env := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, runner)
env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir)

srcDir := testutil.TempDirectory(t)
tmpLogDir := testutil.TempDirectory(t)

// 5-level directory with <=10 files and <=10 subdirectories at each level
testdirtree.CreateDirectoryTree(srcDir, testdirtree.MaybeSimplifyFilesystem(testdirtree.DirectoryTreeOptions{
Depth: 3,
MaxSubdirsPerDirectory: 10,
MaxFilesPerDirectory: 100,
MaxFileSize: 10,
}), &testdirtree.DirectoryTreeCounters{})

env.RunAndExpectSuccess(t, "snap", "create", srcDir,
"--file-log-local-tz", "--log-level=error", "--file-log-level=debug",
"--max-log-file-segment-size=1000", "--log-dir", tmpLogDir)

subdirFlags := map[string]string{
"cli-logs": "--log-dir-max-total-size-mb",
"content-logs": "--content-log-dir-max-total-size-mb",
}

for subdir, flag := range subdirFlags {
logSubdir := filepath.Join(tmpLogDir, subdir)
flag := flag

t.Run(subdir, func(t *testing.T) {
env.RunAndExpectSuccess(t, "snap", "ls", "--file-log-level=debug", "--log-dir", tmpLogDir, flag+"=40000")
size1 := getTotalDirSize(t, logSubdir)
size1MB := float64(size1) / 1e6

env.RunAndExpectSuccess(t, "snap", "ls", "--file-log-level=debug", "--log-dir", tmpLogDir, fmt.Sprintf("%s=%v", flag, size1MB/2))
size2 := getTotalDirSize(t, logSubdir)
require.Less(t, size2, size1/2)
require.Greater(t, size2, size1/4)
})
}
}

func verifyFileLogFormat(t *testing.T, fname string, re *regexp.Regexp) {
t.Helper()

Expand All @@ -163,3 +209,25 @@ func isUTC() bool {

return offset == 0
}

func getTotalDirSize(t *testing.T, dir string) int {
t.Helper()

entries, err := os.ReadDir(dir)
require.NoError(t, err)

var totalSize int

for _, ent := range entries {
info, err := ent.Info()
require.NoError(t, err)

t.Logf("%v %v", info.Name(), info.Size())

if info.Mode().IsRegular() {
totalSize += int(info.Size())
}
}

return totalSize
}

0 comments on commit 0ce3ec0

Please sign in to comment.