Permalink
Browse files

Add ability to optionally compress logfiles just prior to rotation in…

… logwrite package.

Add note about new compression option in logging documentation.

Signed-off-by: Paul Richter <prichter828@gmail.com>
  • Loading branch information...
teeg82 committed Jan 7, 2019
1 parent cf20f3d commit 817f7173163e01f57255eb6f7357c28ba3e2297e
Showing with 74 additions and 6 deletions.
  1. +6 −1 docs/logging.md
  2. +68 −5 pkg/logwrite/logwrite.go
@@ -60,8 +60,9 @@ a maximum size of 1 MiB and up to 10 files are kept per log. The arguments
`-max-log-files` and `-max-log-size` can be used to override these defaults.

Here is an example log file:

```
# cat /var/log/onboot.001-dhcpcd.out
# cat /var/log/onboot.001-dhcpcd.out
2018-07-08T09:16:53Z onboot.001-dhcpcd.out eth0: waiting for carrier
2018-07-08T09:16:53Z onboot.001-dhcpcd.out eth0: carrier acquired
2018-07-08T09:16:53Z onboot.001-dhcpcd.out DUID 00:01:00:01:22:d4:93:05:02:50:00
@@ -80,6 +81,10 @@ conds
2018-07-08T09:16:53Z onboot.001-dhcpcd.out dhcpcd exited
```

### Compression

Rotated logs can be automatically compressed (gzip) if needed. By default, this feature is disabled, however this may be overwritten using the argument `-compress true`.

## Current issues and limitations:

- No docker logger plugin support yet - it could be nice to add support to
@@ -4,10 +4,13 @@ package main

import (
"bufio"
"bytes"
"compress/gzip"
"errors"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"net"
"os"
@@ -25,6 +28,12 @@ const (

const mb = 1024 * 1024

// Default value whether to compress rotated log files.
const compressFiles = false

// String appended to files when compression occurs
const gzipFileExtension = ".gz"

// LogMessage is a message received from memlogd.
type LogMessage struct {
Time time.Time // time message was received by memlogd
@@ -100,19 +109,71 @@ func (l *LogFile) Close() error {
return l.File.Close()
}

// Rotate closes the current log file, rotates the files and creates an empty log file.
func (l *LogFile) Rotate(maxLogFiles int) error {
// Compress the given LogFile
func (l *LogFile) Compress() error {
originalLogFile, err := os.OpenFile(l.Path, os.O_RDONLY, 0644)

if err != nil {
return err
}

// calculate the buffer size for rawfile
info, _ := originalLogFile.Stat()

var logFileSize = info.Size()
rawBytes := make([]byte, logFileSize)

// Read the log file into memory
inputBuffer := bufio.NewReader(originalLogFile)
_, err = inputBuffer.Read(rawBytes)

originalLogFile.Close()

if err != nil {
return err
}

// Create a GZIP writer
var outputBuffer bytes.Buffer
writer := gzip.NewWriter(&outputBuffer)
writer.Write(rawBytes)
writer.Close()

// Write the gzipped data to the same file
err = ioutil.WriteFile(l.Path, outputBuffer.Bytes(), info.Mode())

if err != nil {
return err
}

return nil
}

// Rotate and optionally compress the currently open log file. Close the current log file, and create a new empty file.
func (l *LogFile) Rotate(maxLogFiles int, compress bool) error {
if err := l.File.Close(); err != nil {
return err
}

// This specifies the expected file extensions on the rotated files if compression is enabled.
fileExtension := ""

// Optionally compress the log file before rotation
if compress {
fileExtension = gzipFileExtension
if err := l.Compress(); err != nil {
return err
}
}

for i := maxLogFiles - 1; i >= 0; i-- {
newerFile := fmt.Sprintf("%s.%d", l.Path, i-1)
newerFile := fmt.Sprintf("%s.%d%s", l.Path, i-1, fileExtension)
// special case: if index is 0 we omit the suffix i.e. we expect
// foo foo.1 foo.2 up to foo.<maxLogFiles-1>
if i == 0 {
newerFile = l.Path
}
olderFile := fmt.Sprintf("%s.%d", l.Path, i)
olderFile := fmt.Sprintf("%s.%d%s", l.Path, i, fileExtension)
// overwrite the olderFile with the newerFile
err := os.Rename(newerFile, olderFile)
if os.IsNotExist(err) {
@@ -137,6 +198,8 @@ func main() {
logDir := flag.String("log-dir", "/var/log", "Directory containing log files")
maxLogFiles := flag.Int("max-log-files", 10, "Maximum number of rotated log files before deletion")
maxLogSize := flag.Int("max-log-size", mb, "Maximum size of a log file before rotation")
compress := flag.Bool("compress", compressFiles, fmt.Sprintf("Enables compression of rotated log files (default: %t)", compressFiles))

flag.Parse()

addr := net.UnixAddr{
@@ -195,7 +258,7 @@ func main() {
continue
}
if logF.BytesWritten > *maxLogSize {
logF.Rotate(*maxLogFiles)
logF.Rotate(*maxLogFiles, *compress)
}
}
}

0 comments on commit 817f717

Please sign in to comment.