Merge 4d48714 into 488c46a

check-log/lib/check-log.go

@@ -222,13 +222,19 @@ func run(args []string) *checkers.Checker {
 
 func (opts *logOpts) searchLog(logFile string) (int64, int64, string, error) {
 	stateFile := getStateFile(opts.StateDir, logFile, opts.origArgs)
-	skipBytes := int64(0)
+	skipBytes, inode := int64(0), uint(0)
 	if !opts.NoState {
 		s, err := getBytesToSkip(stateFile)
 		if err != nil {
 			return 0, 0, "", err
 		}
 		skipBytes = s
+
+		i, err := getInode(stateFile)
+		if err != nil {
+			return 0, 0, "", err
+		}
+		inode = i
 	}
 
 	f, err := os.Open(logFile)
@@ -237,6 +243,15 @@ func (opts *logOpts) searchLog(logFile string) (int64, int64, string, error) {
 	}
 	defer f.Close()
 
+	var oldf *os.File
+	if !opts.NoState {
+		oldf, err = openOldFile(logFile, &state{SkipBytes: skipBytes, Inode: inode})
+		if err != nil {
+			return 0, 0, "", err
+		}
+		defer oldf.Close()
+	}
+
 	stat, err := f.Stat()
 	if err != nil {
 		return 0, 0, "", err
@@ -256,6 +271,7 @@ func (opts *logOpts) searchLog(logFile string) (int64, int64, string, error) {
 	}
 
 	var r io.Reader = f
+	var oldr io.Reader = oldf
 	if opts.Encoding != "" {
 		e := encoding.GetEncoding(opts.Encoding)
 		if e == nil {
@@ -269,14 +285,30 @@ func (opts *logOpts) searchLog(logFile string) (int64, int64, string, error) {
 		return warnNum, critNum, errLines, err
 	}
 
+	if oldf != nil {
+		// search old file
+		var (
+			oldWarnNum, oldCritNum int64
+			oldErrLines            string
+		)
+		// ignore readBytes under the premise that the old file will never be updated.
+		oldWarnNum, oldCritNum, _, oldErrLines, err := opts.searchReader(oldr)
+		if err != nil {
+			return oldWarnNum, critNum, errLines, err
+		}
+		warnNum += oldWarnNum
+		critNum += oldCritNum
+		errLines += oldErrLines
+	}
+
 	if rotated {
 		skipBytes = readBytes
 	} else {
 		skipBytes += readBytes
 	}
 
 	if !opts.NoState {
-		err = saveState(stateFile, &state{SkipBytes: skipBytes})
+		err = saveState(stateFile, &state{SkipBytes: skipBytes, Inode: detectInode(stat)})
 		if err != nil {
 			log.Printf("writeByteToSkip failed: %s\n", err.Error())
 		}
@@ -349,6 +381,7 @@ func (opts *logOpts) match(line string) (bool, []string) {
 
 type state struct {
 	SkipBytes int64 `json:"skip_bytes"`
+	Inode     uint  `json:"inode"`
 }
 
 func loadState(fname string) (*state, error) {
@@ -410,6 +443,18 @@ func getBytesToSkipOld(f string) (int64, error) {
 	return i, nil
 }
 
+func getInode(f string) (uint, error) {
+	state, err := loadState(f)
+	if err != nil {
+		return 0, err
+	}
+	if state != nil {
+		// json file exists
+		return state.Inode, nil
+	}
+	return 0, nil
+}
+
 func saveState(f string, state *state) error {
 	b, _ := json.Marshal(state)
 	if err := os.MkdirAll(filepath.Dir(f), 0755); err != nil {
@@ -418,6 +463,46 @@ func saveState(f string, state *state) error {
 	return writeFileAtomically(f, b)
 }
 
+var errFileNotFoundByInode = fmt.Errorf("old file not found")
+
+func findFileByInode(inode uint, dir string) (string, error) {
+	fis, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return "", err
+	}
+	for _, fi := range fis {
+		if detectInode(fi) == inode {
+			return filepath.Join(dir, fi.Name()), nil
+		}
+	}
+	return "", errFileNotFoundByInode
+}
+
+func openOldFile(f string, state *state) (*os.File, error) {
+	fi, err := os.Stat(f)
+	if err != nil {
+		return nil, err
+	}
+	inode := detectInode(fi)
+	if state.Inode > 0 && state.Inode != inode {
+		if oldFile, err := findFileByInode(state.Inode, filepath.Dir(f)); err == nil {
+			oldf, err := os.Open(oldFile)
+			if err != nil {
+				return nil, err
+			}
+			oldfi, _ := oldf.Stat()
+			if oldfi.Size() > state.SkipBytes {
+				oldf.Seek(state.SkipBytes, io.SeekStart)
+				return oldf, nil
+			}
+		} else if err != errFileNotFoundByInode {
+			return nil, err
+		}
+		// just ignore the process of searching old file if errFileNotFoundByInode
+	}
+	return nil, nil
+}
+
 func writeFileAtomically(f string, contents []byte) error {
 	// MUST be located on same disk partition
 	tmpf, err := ioutil.TempFile(filepath.Dir(f), "tmp")

check-log/lib/check-log_test.go

@@ -26,19 +26,38 @@ func TestGetStateFile(t *testing.T) {
 
 func TestSaveState(t *testing.T) {
 	f := ".tmp/fuga/piyo.json"
-	err := saveState(f, &state{SkipBytes: 15})
+	err := saveState(f, &state{SkipBytes: 15, Inode: 150})
+	defer os.RemoveAll(".tmp")
 	assert.Equal(t, err, nil, "err should be nil")
 
 	state, err := loadState(f)
 	assert.Equal(t, err, nil, "err should be nil")
 	assert.Equal(t, state.SkipBytes, int64(15))
+	assert.Equal(t, state.Inode, uint(150))
+}
+
+func TestGetInode(t *testing.T) {
+	f := ".tmp/hoge/piyo.json"
+	state := &state{SkipBytes: 15, Inode: 150}
+	defer os.RemoveAll(".tmp")
+
+	i, err := getInode(f)
+	assert.Equal(t, err, nil, "err should be nil")
+	assert.Equal(t, i, uint(0), "inode should be empty")
+
+	saveState(f, state)
+
+	i, err = getInode(f)
+	assert.Equal(t, err, nil, "err should be nil")
+	assert.Equal(t, state.Inode, uint(150))
 }
 
 func TestGetBytesToSkip(t *testing.T) {
 	// fallback testing for backward compatibility
 	oldf := ".tmp/fuga/piyo"
 	newf := ".tmp/fuga/piyo.json"
 	state := &state{SkipBytes: 15}
+	os.MkdirAll(filepath.Dir(oldf), 0755)
 	writeFileAtomically(oldf, []byte(fmt.Sprintf("%d", state.SkipBytes)))
 	defer os.RemoveAll(".tmp")
 

check-log/lib/check-log_unix.go

@@ -0,0 +1,15 @@
+// +build !windows
+
+package checklog
+
+import (
+	"os"
+	"syscall"
+)
+
+func detectInode(fi os.FileInfo) uint {
+	if stat, ok := fi.Sys().(*syscall.Stat_t); ok {
+		return uint(stat.Ino)
+	}
+	return 0
+}

check-log/lib/check-log_unix_test.go

@@ -0,0 +1,189 @@
+// +build !windows
+
+package checklog
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFindFileByInode(t *testing.T) {
+	dir, err := ioutil.TempDir("", "check-log-test")
+	if err != nil {
+		t.Errorf("something went wrong")
+	}
+	defer os.RemoveAll(dir)
+
+	logf := filepath.Join(dir, "dummy")
+	fh, _ := os.Create(logf)
+	defer fh.Close()
+
+	testFileExist := func() {
+		logfi, err := os.Stat(logf)
+		inode := detectInode(logfi)
+		f, err := findFileByInode(inode, dir)
+		assert.Equal(t, err, nil, "err should be nil")
+		assert.Equal(t, logf, f)
+	}
+	testFileExist()
+
+	testFileNotExist := func() {
+		f, err := findFileByInode(100, dir)
+		assert.Equal(t, err, errFileNotFoundByInode, "err should be errFileNotFoundByInode")
+		assert.Equal(t, "", f)
+	}
+	testFileNotExist()
+}
+
+func TestOpenOldFile(t *testing.T) {
+	dir, err := ioutil.TempDir("", "check-log-test")
+	if err != nil {
+		t.Fatalf("TempDir failed: %s", err)
+	}
+	defer os.RemoveAll(dir)
+
+	logf := filepath.Join(dir, "dummy")
+	fh, _ := os.Create(logf)
+	defer fh.Close()
+
+	testFoundOldFile := func() {
+		ologf := filepath.Join(dir, "dummy.1")
+		ofh, _ := os.Create(ologf)
+		ofi, _ := ofh.Stat()
+		defer ofh.Close()
+
+		l1 := "FATAL\n"
+		ofh.WriteString(l1 + l1)
+
+		state := &state{SkipBytes: int64(len(l1)), Inode: detectInode(ofi)}
+		f, err := openOldFile(logf, state)
+		defer f.Close()
+		pos, _ := f.Seek(-1, io.SeekCurrent) // get current offset
+
+		assert.Equal(t, err, nil, "err should be nil")
+		assert.Equal(t, ofh.Name(), f.Name(), "openOldFile should be return old file")
+		assert.Equal(t, len(l1)-1, int(pos))
+	}
+	testFoundOldFile()
+
+	testNotFoundOldFile := func() {
+		state := &state{SkipBytes: 0, Inode: 0}
+		f, err := openOldFile(logf, state)
+
+		assert.Equal(t, err, nil, "err should be nil")
+		assert.Equal(t, f, (*os.File)(nil), "file should be nil")
+	}
+	testNotFoundOldFile()
+
+	testFoundOldFileByInode := func() {
+		// dir different from logf
+		dir, err := ioutil.TempDir("", "check-log-test")
+		if err != nil {
+			t.Fatalf("TempDir failed: %s", err)
+		}
+		defer os.RemoveAll(dir)
+		ologf := filepath.Join(dir, "dummy.1")
+
+		ofh, _ := os.Create(ologf)
+		ofi, _ := ofh.Stat()
+		defer ofh.Close()
+
+		state := &state{SkipBytes: 0, Inode: detectInode(ofi)}
+		f, err := openOldFile(logf, state)
+		defer f.Close()
+
+		assert.Equal(t, err, nil, "err should be nil")
+	}
+	testFoundOldFileByInode()
+}
+
+func TestRunTraceInode(t *testing.T) {
+	dir, err := ioutil.TempDir("", "check-log-test")
+	if err != nil {
+		t.Errorf("something went wrong")
+	}
+	defer os.RemoveAll(dir)
+
+	logf := filepath.Join(dir, "dummy")
+	fh, _ := os.Create(logf)
+	defer fh.Close()
+
+	ptn := `FATAL`
+	opts, _ := parseArgs([]string{"-s", dir, "-f", logf, "-p", ptn})
+	opts.prepare()
+
+	stateFile := getStateFile(opts.StateDir, logf, opts.origArgs)
+
+	bytes, _ := getBytesToSkip(stateFile)
+	assert.Equal(t, int64(0), bytes, "something went wrong")
+
+	l1 := "SUCCESS\n"
+	l2 := "FATAL\n"
+	l3 := "SUCCESS\n"
+	testRotate := func() {
+		// first check
+		fh.WriteString(l1)
+		opts.searchLog(logf)
+
+		// write FATAL
+		fh.WriteString(l2)
+		fh.Close()
+
+		// logrotate
+		rotatedLogf := filepath.Join(dir, "dummy.1")
+		os.Rename(logf, rotatedLogf)
+		fh, _ = os.Create(logf)
+
+		fh.WriteString(l3)
+		// second check
+		w, c, errLines, err := opts.searchLog(logf)
+		assert.Equal(t, err, nil, "err should be nil")
+		assert.Equal(t, int64(1), w, "something went wrong")
+		assert.Equal(t, int64(1), c, "something went wrong")
+		assert.Equal(t, "FATAL\n", errLines, "something went wrong")
+
+		bytes, _ = getBytesToSkip(stateFile)
+		assert.Equal(t, int64(len(l3)), bytes, "should not include oldfile skip bytes")
+
+		os.Remove(rotatedLogf)
+	}
+	testRotate()
+
+	// case of renaming to different directory
+	// from the directory of the target logfile
+	testRotateDifferentDir := func() {
+		// first check
+		fh.WriteString(l1)
+		opts.searchLog(logf)
+
+		// write FATAL
+		fh.WriteString(l2)
+		fh.Close()
+
+		// logrotate to <dir>/dummy/dummy.1
+		newDir := filepath.Join(dir, "dummy")
+		os.MkdirAll(newDir, 0755)
+		rotatedLogf := filepath.Join(newDir, "dummy.1")
+		os.Rename(logf, rotatedLogf)
+		fh, _ = os.Create(logf)
+
+		fh.WriteString(l3)
+		// second check
+		w, c, errLines, err := opts.searchLog(logf)
+		assert.Equal(t, err, nil, "err should be nil when the old file is not found")
+		assert.Equal(t, int64(0), w, "something went wrong")
+		assert.Equal(t, int64(0), c, "something went wrong")
+		assert.Equal(t, "", errLines, "something went wrong")
+
+		bytes, _ = getBytesToSkip(stateFile)
+		assert.Equal(t, int64(len(l3)), bytes, "should not include oldfile skip bytes")
+
+		os.Remove(rotatedLogf)
+	}
+	testRotateDifferentDir()
+}