Skip to content
This repository has been archived by the owner on Jan 20, 2021. It is now read-only.

Commit

Permalink
add compress option for 'jsonfiles' log driver
Browse files Browse the repository at this point in the history 
This PR adds support for compressibility of log file.
I added a new option conpression for the jsonfile log driver,
this option allows the user to specify compression algorithm to
compress the log files. By default, the log files will be
not compressed. At present, only support 'gzip'.

Signed-off-by: Yanqiang Miao <miao.yanqiang@zte.com.cn>

'docker logs' can read from compressed files

Signed-off-by: Yanqiang Miao <miao.yanqiang@zte.com.cn>

Add Metadata to the gzip header, optmize 'readlog'

Signed-off-by: Yanqiang Miao <miao.yanqiang@zte.com.cn>
  • Loading branch information
Yanqiang Miao committed Mar 15, 2018
1 parent 241c904 commit f69f09f
Show file tree
Hide file tree
Showing 3 changed files with 322 additions and 49 deletions.
18 changes: 17 additions & 1 deletion daemon/logger/jsonfilelog/jsonfilelog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ func New(info logger.Info) (logger.Logger, error) {
if err != nil {
return nil, err
}
if capval <= 0 {
return nil, fmt.Errorf("max-size should be a positive numbler")
}
}
var maxFiles = 1
if maxFileString, ok := info.Config["max-file"]; ok {
Expand All @@ -62,6 +65,18 @@ func New(info logger.Info) (logger.Logger, error) {
}
}

var compress bool
if compressString, ok := info.Config["compress"]; ok {
var err error
compress, err = strconv.ParseBool(compressString)
if err != nil {
return nil, err
}
if compress && (maxFiles == 1 || capval == -1) {
return nil, fmt.Errorf("compress cannot be true when max-file is less than 2 or max-size is not set")
}
}

attrs, err := info.ExtraAttributes(nil)
if err != nil {
return nil, err
Expand Down Expand Up @@ -95,7 +110,7 @@ func New(info logger.Info) (logger.Logger, error) {
return b, nil
}

writer, err := loggerutils.NewLogFile(info.LogPath, capval, maxFiles, marshalFunc, decodeFunc, 0640)
writer, err := loggerutils.NewLogFile(info.LogPath, capval, maxFiles, compress, marshalFunc, decodeFunc, 0640)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -139,6 +154,7 @@ func ValidateLogOpt(cfg map[string]string) error {
switch key {
case "max-file":
case "max-size":
case "compress":
case "labels":
case "env":
case "env-regex":
Expand Down
64 changes: 59 additions & 5 deletions daemon/logger/jsonfilelog/jsonfilelog_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package jsonfilelog // import "github.com/docker/docker/daemon/logger/jsonfilelo

import (
"bytes"
"compress/gzip"
"encoding/json"
"io/ioutil"
"os"
Expand Down Expand Up @@ -142,7 +143,7 @@ func TestJSONFileLoggerWithOpts(t *testing.T) {
}
defer os.RemoveAll(tmp)
filename := filepath.Join(tmp, "container.log")
config := map[string]string{"max-file": "2", "max-size": "1k"}
config := map[string]string{"max-file": "3", "max-size": "1k", "compress": "true"}
l, err := New(logger.Info{
ContainerID: cid,
LogPath: filename,
Expand All @@ -152,21 +153,55 @@ func TestJSONFileLoggerWithOpts(t *testing.T) {
t.Fatal(err)
}
defer l.Close()
for i := 0; i < 20; i++ {
for i := 0; i < 36; i++ {
if err := l.Log(&logger.Message{Line: []byte("line" + strconv.Itoa(i)), Source: "src1"}); err != nil {
t.Fatal(err)
}
}

res, err := ioutil.ReadFile(filename)
if err != nil {
t.Fatal(err)
}

penUlt, err := ioutil.ReadFile(filename + ".1")
if err != nil {
if !os.IsNotExist(err) {
t.Fatal(err)
}

file, err := os.Open(filename + ".1.gz")
defer file.Close()
if err != nil {
t.Fatal(err)
}
zipReader, err := gzip.NewReader(file)
defer zipReader.Close()
if err != nil {
t.Fatal(err)
}
penUlt, err = ioutil.ReadAll(zipReader)
if err != nil {
t.Fatal(err)
}
}

file, err := os.Open(filename + ".2.gz")
defer file.Close()
if err != nil {
t.Fatal(err)
}
zipReader, err := gzip.NewReader(file)
defer zipReader.Close()
if err != nil {
t.Fatal(err)
}
antepenult, err := ioutil.ReadAll(zipReader)
if err != nil {
t.Fatal(err)
}

expectedPenultimate := `{"log":"line0\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
expectedAntepenultimate := `{"log":"line0\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line1\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line2\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line3\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
Expand All @@ -183,10 +218,27 @@ func TestJSONFileLoggerWithOpts(t *testing.T) {
{"log":"line14\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line15\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
`
expected := `{"log":"line16\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
expectedPenultimate := `{"log":"line16\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line17\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line18\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line19\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line20\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line21\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line22\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line23\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line24\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line25\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line26\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line27\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line28\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line29\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line30\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line31\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
`
expected := `{"log":"line32\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line33\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line34\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
{"log":"line35\n","stream":"src1","time":"0001-01-01T00:00:00Z"}
`

if string(res) != expected {
Expand All @@ -195,7 +247,9 @@ func TestJSONFileLoggerWithOpts(t *testing.T) {
if string(penUlt) != expectedPenultimate {
t.Fatalf("Wrong log content: %q, expected %q", penUlt, expectedPenultimate)
}

if string(antepenult) != expectedAntepenultimate {
t.Fatalf("Wrong log content: %q, expected %q", antepenult, expectedAntepenultimate)
}
}

func TestJSONFileLoggerWithLabelsEnv(t *testing.T) {
Expand Down
Loading

0 comments on commit f69f09f

Please sign in to comment.