Merge branch 'check-clean-expired' into 'master'

adjust log

See merge request paas/zanredisdb!22

Makefile

@@ -2,7 +2,7 @@ PREFIX=/usr/local
 DESTDIR=
 BINDIR=${PREFIX}/bin
 PROJECT?=github.com/youzan/ZanRedisDB
-VERBINARY?= 0.9.2
+VERBINARY?= 0.9.3
 COMMIT?=$(shell git rev-parse --short HEAD)
 BUILD_TIME?=$(shell date '+%Y-%m-%d_%H:%M:%S-%Z')
 GOFLAGS=-ldflags "-X ${PROJECT}/common.VerBinary=${VERBINARY} -X ${PROJECT}/common.Commit=${COMMIT} -X ${PROJECT}/common.BuildTime=${BUILD_TIME}"

apps/placedriver/main_test.go

@@ -1,11 +1,16 @@
 package main
 
 import (
+	"os"
+	"path"
+	"strconv"
 	"testing"
+	"time"
 
 	"github.com/BurntSushi/toml"
 	"github.com/mreiferson/go-options"
 	"github.com/stretchr/testify/assert"
+	"github.com/youzan/ZanRedisDB/common"
 	"github.com/youzan/ZanRedisDB/pdserver"
 )
 
@@ -20,6 +25,16 @@ func TestAppConfigParse(t *testing.T) {
 	}
 	opts := pdserver.NewServerConfig()
 	options.Resolve(opts, flagSet, cfg)
-	pdserver.NewServer(opts)
+	opts.LogDir = path.Join(os.TempDir(), strconv.Itoa(int(time.Now().UnixNano())))
+	os.MkdirAll(opts.LogDir, 0755)
+	common.SetZapRotateOptions(false, true, path.Join(opts.LogDir, "test.log"), 0, 0, 0)
+	s, err := pdserver.NewServer(opts)
+	t.Log(err)
 	assert.Equal(t, "v2", opts.BalanceVer)
+
+	s.Start()
+	t.Log(opts.LogDir)
+
+	time.Sleep(time.Second)
+	s.Stop()
 }

apps/zankv/main_test.go

@@ -3,7 +3,11 @@ package main
 import (
 	"encoding/json"
 	"io/ioutil"
+	"os"
+	"path"
+	"strconv"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/youzan/ZanRedisDB/server"
@@ -19,5 +23,6 @@ func TestAppConfigParse(t *testing.T) {
 	assert.Nil(t, err)
 
 	serverConf := configFile.ServerConf
+	serverConf.LogDir = path.Join(os.TempDir(), strconv.Itoa(int(time.Now().UnixNano())))
 	server.NewServer(serverConf)
 }

common/logger.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -17,10 +16,6 @@ import (
 var inTestLog bool
 
 func init() {
-	if len(os.Args) > 1 && strings.HasPrefix(os.Args[1], "-test") {
-		inTestLog = true
-		log.Printf("using log for test, %s\n", os.Args[1])
-	}
 	conf := zap.NewProductionConfig()
 	conf.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
 	conf.DisableStacktrace = true
@@ -390,6 +385,7 @@ func SetZapRotateOptions(alsoLogToStdout bool, alsoLogErrToStdErr bool, logfile
 	encConf := zap.NewProductionEncoderConfig()
 	encConf.EncodeTime = zapcore.ISO8601TimeEncoder
 	enc := zapcore.NewJSONEncoder(encConf)
+	fmt.Printf("zap logger option: %v, %v, %v, %v\n", alsoLogToStdout, alsoLogErrToStdErr, logfile, maxAgeDay)
 	if logfile == "" {
 		wrap := zap.WrapCore(func(c zapcore.Core) zapcore.Core {
 			return zapcore.NewTee(
@@ -440,49 +436,56 @@ func SetZapRotateOptions(alsoLogToStdout bool, alsoLogErrToStdErr bool, logfile
 }
 
 type zapLogger struct {
-	logger *zap.SugaredLogger
+	module string
 }
 
 // note: currently, the zap logger do not support buffer writer by default, so we need merge new code after the zap new
 // release published.
 func newZapLogger(module string) *zapLogger {
-	if module == "" {
-		return &zapLogger{
-			logger: zapLog.Sugar(),
-		}
-	}
 	return &zapLogger{
-		logger: zapLog.Named(module).Sugar(),
+		module: module,
 	}
 }
 
 func (zl *zapLogger) Output(maxdepth int, s string) error {
+	if zapLog == nil {
+		return nil
+	}
 	if maxdepth <= 2 {
-		zl.logger.Info(s)
+		zapLog.Named(zl.module).Info(s)
 	} else {
-		zl.logger.Desugar().WithOptions(zap.AddCallerSkip(maxdepth - 2)).Info(s)
+		zapLog.Named(zl.module).WithOptions(zap.AddCallerSkip(maxdepth - 2)).Info(s)
 	}
 	return nil
 }
 
 func (zl *zapLogger) OutputWarning(maxdepth int, s string) error {
+	if zapLog == nil {
+		return nil
+	}
 	if maxdepth == 2 {
-		zl.logger.Warn(s)
+		zapLog.Named(zl.module).Warn(s)
 	} else {
-		zl.logger.Desugar().WithOptions(zap.AddCallerSkip(maxdepth - 2)).Warn(s)
+		zapLog.Named(zl.module).WithOptions(zap.AddCallerSkip(maxdepth - 2)).Warn(s)
 	}
 	return nil
 }
 
 func (zl *zapLogger) OutputErr(maxdepth int, s string) error {
+	if zapLog == nil {
+		return nil
+	}
 	if maxdepth == 2 {
-		zl.logger.Error(s)
+		zapLog.Named(zl.module).Error(s)
 	} else {
-		zl.logger.Desugar().WithOptions(zap.AddCallerSkip(maxdepth - 2)).Error(s)
+		zapLog.Named(zl.module).WithOptions(zap.AddCallerSkip(maxdepth - 2)).Error(s)
 	}
 	return nil
 }
 
 func (zl *zapLogger) Flush() {
-	zl.logger.Sync()
+	if zapLog == nil {
+		return
+	}
+	zapLog.Sync()
 }

engine/pebble_eng_test.go

@@ -115,6 +115,7 @@ func TestPebbleReopenAndCheck(t *testing.T) {
 	wb.Put([]byte("test"), []byte("test"))
 	err = pe.Write(wb)
 	assert.Nil(t, err)
+	wb.Clear()
 	ck, _ := pe.NewCheckpoint(false)
 	err = ck.Save(path.Join(tmpDir, "cktmp"), make(chan struct{}))
 	assert.Nil(t, err)
@@ -153,6 +154,7 @@ func TestPebbleSharedCacheForMulti(t *testing.T) {
 	wb.Put([]byte("test"), []byte("test"))
 	err = pe.Write(wb)
 	assert.Nil(t, err)
+	wb.Clear()
 
 	pe.eng.Flush()
 
@@ -169,6 +171,7 @@ func TestPebbleSharedCacheForMulti(t *testing.T) {
 	wb2.Put([]byte("test"), []byte("test2"))
 	err = pe2.Write(wb2)
 	assert.Nil(t, err)
+	wb2.Clear()
 	pe2.eng.Flush()
 
 	v1, err := pe.GetBytes([]byte("test"))
@@ -182,6 +185,7 @@ func TestPebbleSharedCacheForMulti(t *testing.T) {
 	wb.Put([]byte("test"), []byte("test"))
 	err = pe.Write(wb)
 	assert.Nil(t, err)
+	wb.Clear()
 	pe.eng.Flush()
 
 	v1, err = pe.GetBytes([]byte("test"))

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/absolute8511/go-hll v0.0.0-20190228064837-043118556d83
 	github.com/absolute8511/hyperloglog v0.0.0-20171127080255-5259284545fc
 	github.com/absolute8511/hyperloglog2 v0.1.1
-	github.com/absolute8511/redcon v0.9.2
+	github.com/absolute8511/redcon v0.9.3
 	github.com/absolute8511/redigo v1.4.6
 	github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 // indirect
 	github.com/alicebob/miniredis v2.5.0+incompatible // indirect

go.sum

@@ -22,6 +22,8 @@ github.com/absolute8511/hyperloglog2 v0.1.1 h1:ZFAXGBcLsbXTtfMVqOJkaJe0Z1GNaeNFE
 github.com/absolute8511/hyperloglog2 v0.1.1/go.mod h1:4ZpNQu/xhWhgXqUKLPaqJlbcyBvKsqbVNsxYcv/cqVo=
 github.com/absolute8511/redcon v0.9.2 h1:engVvU2x7lGgK78y2lh2OqAPu7Vml8jPjtol7NiSFUQ=
 github.com/absolute8511/redcon v0.9.2/go.mod h1:pcezn7cyNCl6Bbics6NRddlZw1GwLqpNFapwKI14N4A=
+github.com/absolute8511/redcon v0.9.3 h1:na5mBQ1OcLxhIP9omakpeWH1/d52UdJmPlTksQEjI0E=
+github.com/absolute8511/redcon v0.9.3/go.mod h1:pcezn7cyNCl6Bbics6NRddlZw1GwLqpNFapwKI14N4A=
 github.com/absolute8511/redigo v1.4.6 h1:22Zgx7rKi/veL2xs7PEOvQS5fy2E5BBW6yRhId9WsTo=
 github.com/absolute8511/redigo v1.4.6/go.mod h1:ncnpDtZTl7oMnWRkyfo8hfwC8+u3HlzK3XEzJ7W/M08=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=

pdserver/pdconf.example.conf

@@ -2,7 +2,7 @@
 http_address = "0.0.0.0:18001"
 
 ## the network interface for broadcast, the ip will be detected automatically.
-broadcast_interface = "eth0"
+broadcast_interface = "lo0"
 
 ## local reverse proxy port, basically used for collecting the stats 
 # reverse_proxy_port = "18003"
@@ -11,7 +11,7 @@ profile_port = "6667"
 
 cluster_id = "test-cluster-dev-1"
 ## the etcd cluster ip list
-cluster_leadership_addresses = "127.0.0.1:2379"
+cluster_leadership_addresses = "http://127.0.0.1:2379"
 
 ## data dir for some cluster data
 data_dir = ""

slow/slowlog.go

@@ -71,7 +71,7 @@ func getIndex(si SlowLogInfo) int {
 func checkLastLogCollTime(si SlowLogInfo) bool {
 	index := getIndex(si)
 	t := logCollTimes[index%len(logCollTimes)]
-	return atomic.LoadInt64(t)+time.Second.Nanoseconds() <= time.Now().UnixNano()
+	return atomic.LoadInt64(t)+3*time.Second.Nanoseconds() <= time.Now().UnixNano()
 }
 
 func updateLastLogCollTime(si SlowLogInfo) {

slow/slowlog_test.go

@@ -63,7 +63,7 @@ func TestSlowLogLevel(t *testing.T) {
 	str, logged = LogLargeCollection(collectionLargeLen, NewSlowLogInfo("scope_test", "test", ""))
 	t.Log(str)
 	assert.Equal(t, false, logged)
-	time.Sleep(time.Second)
+	time.Sleep(3 * time.Second)
 	str, logged = LogLargeCollection(collectionLargeLen, NewSlowLogInfo("scope_test", "test", ""))
 	t.Log(str)
 	assert.Equal(t, true, logged)