Skip to content
Permalink
Browse files

Move libbeat filebeat dependency to libbeat

Libbeat had a dependency on filebeat because of file rotation. This PR removes the dependency by moving the file rotation code to
  • Loading branch information...
ruflin committed Jun 6, 2017
1 parent c2fb1b9 commit 8ca06132c7f7ff4230959ac1a1ecd54eaabb3ca4
@@ -6,8 +6,6 @@ import (
"fmt" "fmt"
"os" "os"
"syscall" "syscall"

"github.com/elastic/beats/libbeat/logp"
) )


type StateOS struct { type StateOS struct {
@@ -38,15 +36,6 @@ func (fs StateOS) String() string {
return fmt.Sprintf("%d-%d", fs.Inode, fs.Device) return fmt.Sprintf("%d-%d", fs.Inode, fs.Device)
} }


// SafeFileRotate safely rotates an existing file under path and replaces it with the tempfile
func SafeFileRotate(path, tempfile string) error {
if e := os.Rename(tempfile, path); e != nil {
logp.Err("Rotate error: %s", e)
return e
}
return nil
}

// ReadOpen opens a file for reading only // ReadOpen opens a file for reading only
func ReadOpen(path string) (*os.File, error) { func ReadOpen(path string) (*os.File, error) {
flag := os.O_RDONLY flag := os.O_RDONLY
@@ -46,58 +46,4 @@ func TestIsSameFile(t *testing.T) {
assert.True(t, file2.IsSameFile(file3)) assert.True(t, file2.IsSameFile(file3))
} }


func TestSafeFileRotateExistingFile(t *testing.T) {


tempdir, err := ioutil.TempDir("", "")
assert.NoError(t, err)
defer func() {
assert.NoError(t, os.RemoveAll(tempdir))
}()

// create an existing registry file
err = ioutil.WriteFile(filepath.Join(tempdir, "registry"),
[]byte("existing filebeat"), 0x777)
assert.NoError(t, err)

// create a new registry.new file
err = ioutil.WriteFile(filepath.Join(tempdir, "registry.new"),
[]byte("new filebeat"), 0x777)
assert.NoError(t, err)

// rotate registry.new into registry
err = SafeFileRotate(filepath.Join(tempdir, "registry"),
filepath.Join(tempdir, "registry.new"))
assert.NoError(t, err)

contents, err := ioutil.ReadFile(filepath.Join(tempdir, "registry"))
assert.NoError(t, err)
assert.Equal(t, []byte("new filebeat"), contents)

// do it again to make sure we deal with deleting the old file

err = ioutil.WriteFile(filepath.Join(tempdir, "registry.new"),
[]byte("new filebeat 1"), 0x777)
assert.NoError(t, err)

err = SafeFileRotate(filepath.Join(tempdir, "registry"),
filepath.Join(tempdir, "registry.new"))
assert.NoError(t, err)

contents, err = ioutil.ReadFile(filepath.Join(tempdir, "registry"))
assert.NoError(t, err)
assert.Equal(t, []byte("new filebeat 1"), contents)

// and again for good measure

err = ioutil.WriteFile(filepath.Join(tempdir, "registry.new"),
[]byte("new filebeat 2"), 0x777)
assert.NoError(t, err)

err = SafeFileRotate(filepath.Join(tempdir, "registry"),
filepath.Join(tempdir, "registry.new"))
assert.NoError(t, err)

contents, err = ioutil.ReadFile(filepath.Join(tempdir, "registry"))
assert.NoError(t, err)
assert.Equal(t, []byte("new filebeat 2"), contents)
}
@@ -5,8 +5,6 @@ import (
"os" "os"
"reflect" "reflect"
"syscall" "syscall"

"github.com/elastic/beats/libbeat/logp"
) )


type StateOS struct { type StateOS struct {
@@ -49,31 +47,6 @@ func (fs StateOS) String() string {
return fmt.Sprintf("%d-%d-%d", fs.IdxHi, fs.IdxLo, fs.Vol) return fmt.Sprintf("%d-%d-%d", fs.IdxHi, fs.IdxLo, fs.Vol)
} }


// SafeFileRotate safely rotates an existing file under path and replaces it with the tempfile
func SafeFileRotate(path, tempfile string) error {
old := path + ".old"
var e error

// In Windows, one cannot rename a file if the destination already exists, at least
// not with using the os.Rename function that Golang offers.
// This tries to move the existing file into an old file first and only do the
// move after that.
if e = os.Remove(old); e != nil {
logp.Debug("filecompare", "delete old: %v", e)
// ignore error in case old doesn't exit yet
}
if e = os.Rename(path, old); e != nil {
logp.Debug("filecompare", "rotate to old: %v", e)
// ignore error in case path doesn't exist
}

if e = os.Rename(tempfile, path); e != nil {
logp.Err("rotate: %v", e)
return e
}
return nil
}

// ReadOpen opens a file for reading only // ReadOpen opens a file for reading only
// As Windows blocks deleting a file when its open, some special params are passed here. // As Windows blocks deleting a file when its open, some special params are passed here.
func ReadOpen(path string) (*os.File, error) { func ReadOpen(path string) (*os.File, error) {
@@ -10,6 +10,7 @@ import (
"github.com/elastic/beats/filebeat/input/file" "github.com/elastic/beats/filebeat/input/file"
"github.com/elastic/beats/filebeat/publisher" "github.com/elastic/beats/filebeat/publisher"
"github.com/elastic/beats/filebeat/util" "github.com/elastic/beats/filebeat/util"
helper "github.com/elastic/beats/libbeat/common/file"
"github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/logp"
"github.com/elastic/beats/libbeat/monitoring" "github.com/elastic/beats/libbeat/monitoring"
"github.com/elastic/beats/libbeat/paths" "github.com/elastic/beats/libbeat/paths"
@@ -228,7 +229,7 @@ func (r *Registrar) writeRegistry() error {
// Directly close file because of windows // Directly close file because of windows
f.Close() f.Close()


err = file.SafeFileRotate(r.registryFile, tempfile) err = helper.SafeFileRotate(r.registryFile, tempfile)


logp.Debug("registrar", "Registry file updated. %d states written.", len(states)) logp.Debug("registrar", "Registry file updated. %d states written.", len(states))
registryWrites.Add(1) registryWrites.Add(1)
@@ -45,10 +45,10 @@ import (


"github.com/satori/go.uuid" "github.com/satori/go.uuid"


"github.com/elastic/beats/filebeat/input/file"
"github.com/elastic/beats/libbeat/api" "github.com/elastic/beats/libbeat/api"
"github.com/elastic/beats/libbeat/cfgfile" "github.com/elastic/beats/libbeat/cfgfile"
"github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/common/file"
"github.com/elastic/beats/libbeat/dashboards" "github.com/elastic/beats/libbeat/dashboards"
"github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/logp"
"github.com/elastic/beats/libbeat/monitoring/report" "github.com/elastic/beats/libbeat/monitoring/report"
@@ -0,0 +1,16 @@
package file

import (
"os"

"github.com/elastic/beats/libbeat/logp"
)

// SafeFileRotate safely rotates an existing file under path and replaces it with the tempfile
func SafeFileRotate(path, tempfile string) error {
if e := os.Rename(tempfile, path); e != nil {
logp.Err("Rotate error: %s", e)
return e
}
return nil
}
@@ -0,0 +1,68 @@
// +build !integration

package file

import (
"io/ioutil"
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
)

func TestSafeFileRotateExistingFile(t *testing.T) {

tempdir, err := ioutil.TempDir("", "")
assert.NoError(t, err)
defer func() {
assert.NoError(t, os.RemoveAll(tempdir))
}()

// create an existing registry file
err = ioutil.WriteFile(filepath.Join(tempdir, "registry"),
[]byte("existing filebeat"), 0x777)
assert.NoError(t, err)

// create a new registry.new file
err = ioutil.WriteFile(filepath.Join(tempdir, "registry.new"),
[]byte("new filebeat"), 0x777)
assert.NoError(t, err)

// rotate registry.new into registry
err = SafeFileRotate(filepath.Join(tempdir, "registry"),
filepath.Join(tempdir, "registry.new"))
assert.NoError(t, err)

contents, err := ioutil.ReadFile(filepath.Join(tempdir, "registry"))
assert.NoError(t, err)
assert.Equal(t, []byte("new filebeat"), contents)

// do it again to make sure we deal with deleting the old file

err = ioutil.WriteFile(filepath.Join(tempdir, "registry.new"),
[]byte("new filebeat 1"), 0x777)
assert.NoError(t, err)

err = SafeFileRotate(filepath.Join(tempdir, "registry"),
filepath.Join(tempdir, "registry.new"))
assert.NoError(t, err)

contents, err = ioutil.ReadFile(filepath.Join(tempdir, "registry"))
assert.NoError(t, err)
assert.Equal(t, []byte("new filebeat 1"), contents)

// and again for good measure

err = ioutil.WriteFile(filepath.Join(tempdir, "registry.new"),
[]byte("new filebeat 2"), 0x777)
assert.NoError(t, err)

err = SafeFileRotate(filepath.Join(tempdir, "registry"),
filepath.Join(tempdir, "registry.new"))
assert.NoError(t, err)

contents, err = ioutil.ReadFile(filepath.Join(tempdir, "registry"))
assert.NoError(t, err)
assert.Equal(t, []byte("new filebeat 2"), contents)
}
@@ -0,0 +1,32 @@
package file

import (
"os"

"github.com/elastic/beats/libbeat/logp"
)

// SafeFileRotate safely rotates an existing file under path and replaces it with the tempfile
func SafeFileRotate(path, tempfile string) error {
old := path + ".old"
var e error

// In Windows, one cannot rename a file if the destination already exists, at least
// not with using the os.Rename function that Golang offers.
// This tries to move the existing file into an old file first and only do the
// move after that.
if e = os.Remove(old); e != nil {
logp.Debug("filecompare", "delete old: %v", e)
// ignore error in case old doesn't exit yet
}
if e = os.Rename(path, old); e != nil {
logp.Debug("filecompare", "rotate to old: %v", e)
// ignore error in case path doesn't exist
}

if e = os.Rename(tempfile, path); e != nil {
logp.Err("rotate: %v", e)
return e
}
return nil
}

0 comments on commit 8ca0613

Please sign in to comment.
You can’t perform that action at this time.