Skip to content
This repository has been archived by the owner on Mar 10, 2021. It is now read-only.

Commit

Permalink
Merge 7a1f0fd into 9884a7d
Browse files Browse the repository at this point in the history
  • Loading branch information
@obalunenko
obalunenko committed Dec 8, 2018
2 parents 9884a7d + 7a1f0fd commit a54c6dd
Show file tree
Hide file tree
Showing 40 changed files with 6,886 additions and 193 deletions.
44 changes: 23 additions & 21 deletions config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,33 +6,36 @@ import (
"path/filepath"
"reflect"
"testing"

log "github.com/sirupsen/logrus"
)

func testConfigCreate(configPath string, logsFilesListJSON string, logLevel string, mongoURL string, mongoDB string, mongoCollection string, dropDB bool) error {
func testConfigCreate(t *testing.T, configPath string, logsFilesListJSON string, logLevel string, mongoURL string, mongoDB string,
mongoCollection string, dropDB bool) error {

fmt.Println("Helper func in action")
if err := os.MkdirAll(filepath.Dir(configPath), 0700); err != nil {
log.Fatalf("failed creating all dirs for config file [%s]: %v", filepath.Dir(configPath), err)
t.Fatalf("failed creating all dirs for config file [%s]: %v", filepath.Dir(configPath), err)

}

configFile, err := os.OpenFile(configPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0777)

if err != nil {
log.Fatalf("error opening config file: %v", err)
t.Fatalf("error opening config file: %v", err)
}

if err != nil {
return fmt.Errorf("Failed to create test config: %v", err)
return fmt.Errorf("failed to create test config: %v", err)
}

_, err = configFile.WriteString(fmt.Sprintf(`LogLevel="%s"
LogsFilesListJSON='%s'
MongoURL="%s"
MongoDB="%s"
MongoCollection="%s"
DropDB=%t`, logLevel, logsFilesListJSON, mongoURL, mongoDB, mongoCollection, dropDB))

if err != nil {
return fmt.Errorf("Failed to write test config: %v", err)
return fmt.Errorf("failed to write test config: %v", err)
}
if err := configFile.Close(); err != nil {
return fmt.Errorf("failed to close test config file descriptor: %v", err)
Expand All @@ -41,10 +44,10 @@ func testConfigCreate(configPath string, logsFilesListJSON string, logLevel stri
return nil
}

func testConfigDelete(configPath string) {
func testConfigDelete(t *testing.T, configPath string) {
err := os.Remove(configPath)
if err != nil {
log.Fatalf("Failed to delete old config file")
t.Fatalf("Failed to delete old config file")
}
}

Expand All @@ -63,15 +66,13 @@ func TestLoadConfig(t *testing.T) {
wantConfig *Config
wantErr bool
}
type testCase struct {
var tests = []struct {
id int
description string
input input
expectedResult expectedResult
}
type testSuite []testCase
var ts = testSuite{
testCase{
}{
{
id: 1,
description: `Check configuration loading from cofig file`,
input: input{
Expand Down Expand Up @@ -99,7 +100,7 @@ func TestLoadConfig(t *testing.T) {
wantErr: false,
},
},
testCase{
{
id: 2,
description: `Broken config: incorrect json with files`,
input: input{
Expand All @@ -117,7 +118,7 @@ func TestLoadConfig(t *testing.T) {
wantErr: true,
},
},
testCase{
{
id: 3,
description: `Broken config: empty json with files`,
input: input{
Expand All @@ -136,23 +137,24 @@ func TestLoadConfig(t *testing.T) {
},
},
}

currentDir, err := filepath.Abs(filepath.Dir(os.Args[0]))
if err != nil {
log.Fatal(err)
t.Fatal(err)
}
configPath := filepath.Join(currentDir, "config.toml")
configPath := filepath.Join(currentDir, "testdata", "config.toml")

for _, tc := range ts {
for _, tc := range tests {

t.Run(fmt.Sprintf("Test%d:%s", tc.id, tc.description), func(t *testing.T) {

err = testConfigCreate(configPath, tc.input.logsFilesListJSON, tc.input.logLevel, tc.input.mongoURL, tc.input.mongoDB, tc.input.mongoCollection, tc.input.dropDB)
err = testConfigCreate(t, configPath, tc.input.logsFilesListJSON, tc.input.logLevel, tc.input.mongoURL, tc.input.mongoDB, tc.input.mongoCollection, tc.input.dropDB)
if err != nil {
t.Fatalf("Error while creating test config: %v", err)
}

gotModel, err := LoadConfig(configPath)
testConfigDelete(configPath) // Delete old config file after it was loaded
testConfigDelete(t, configPath) // Delete old config file after it was loaded

if (err != nil) != tc.expectedResult.wantErr {
t.Errorf("processLine() error = %+v, \nwantErr %+v", err, tc.expectedResult.wantErr)
Expand Down
23 changes: 17 additions & 6 deletions converter/converter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,32 @@ package converter
import (
"fmt"
"strings"
"sync"
"time"

"github.com/hpcloud/tail"
logModel "github.com/oleg-balunenko/logs-converter/model"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)

// Start starts converting of logfile
func Start(logName string, format string, resultChan chan *LogModel) {
func Start(logName string, format string, mustExist bool, follow bool, resultChan chan *logModel.LogModel,
errorsChan chan error, wg *sync.WaitGroup) {

log.Infof("Starting tailing and converting file [%s] with logs format [%s]", logName, format)

defer wg.Done()

t, err := tail.TailFile(logName, tail.Config{
Follow: true,
Follow: follow,
MustExist: mustExist,
})
if err != nil {
log.Fatalf("failed to tail file [%s]: %v", logName, err)
msg := fmt.Sprintf("failed to tail file [%s]", logName)
errorsChan <- errors.Wrap(err, msg)

return
}

var cnt uint64
Expand All @@ -26,7 +37,7 @@ func Start(logName string, format string, resultChan chan *LogModel) {
log.Debugf("File:[%s] Line tailed: [%v]", logName, line)
model, err := processLine(logName, line.Text, format, cnt)
if err != nil {
log.Errorf("Failed to process line [%s]: %v", line.Text, err)
errorsChan <- errors.Wrap(err, fmt.Sprintf("Failed to process line [%s]", line.Text))
}

log.Debugf("Go routine for file [%s] sending model to chanel", logName)
Expand All @@ -37,7 +48,7 @@ func Start(logName string, format string, resultChan chan *LogModel) {

}

func processLine(logName string, line string, format string, lineNumber uint64) (model *LogModel, err error) {
func processLine(logName string, line string, format string, lineNumber uint64) (model *logModel.LogModel, err error) {

lineElements := strings.Split(line, " | ")

Expand All @@ -58,7 +69,7 @@ func processLine(logName string, line string, format string, lineNumber uint64)
msg = lineElements[1]
}

model = &LogModel{
model = &logModel.LogModel{
LogTime: logTime,
LogMsg: msg,
FileName: logName,
Expand Down
57 changes: 28 additions & 29 deletions converter/converter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@ package converter

import (
"fmt"
"reflect"
"testing"
"time"

"github.com/oleg-balunenko/logs-converter/model"
"github.com/stretchr/testify/assert"
)

func Test_processLine(t *testing.T) {
Expand All @@ -15,20 +17,16 @@ func Test_processLine(t *testing.T) {
lineNumber uint64
}
type expectedResult struct {
wantModel *LogModel
wantModel *model.LogModel
wantErr bool
}
type testCase struct {
var tests = []struct {
id int
description string
input input
expectedResult expectedResult
}

type testSuite []testCase

var ts = testSuite{
testCase{
}{
{
id: 1,
description: `Invalid format of line. Separator "|" not found`,
input: input{
Expand All @@ -42,7 +40,7 @@ func Test_processLine(t *testing.T) {
wantErr: true,
},
},
testCase{
{
id: 2,
description: `Positive case. First format - one "|" separator`,
input: input{
Expand All @@ -52,7 +50,7 @@ func Test_processLine(t *testing.T) {
lineNumber: 1,
},
expectedResult: expectedResult{
wantModel: &LogModel{
wantModel: &model.LogModel{
LogTime: time.Date(2018, 02, 01, 15, 04, 05, 0, time.UTC),
LogMsg: `This is log message`,
LogFormat: `first_format`,
Expand All @@ -61,7 +59,7 @@ func Test_processLine(t *testing.T) {
wantErr: false,
},
},
testCase{
{
id: 3,
description: `Positive case. First format - more that one "|" separator`,
input: input{
Expand All @@ -71,7 +69,7 @@ func Test_processLine(t *testing.T) {
lineNumber: 1,
},
expectedResult: expectedResult{
wantModel: &LogModel{
wantModel: &model.LogModel{
LogTime: time.Date(2018, 02, 01, 15, 04, 05, 0, time.UTC),
LogMsg: `This is log message | that has|more than one separator`,
LogFormat: `first_format`,
Expand All @@ -80,7 +78,7 @@ func Test_processLine(t *testing.T) {
wantErr: false,
},
},
testCase{
{
id: 4,
description: `Positive case. Second format`,
input: input{
Expand All @@ -90,7 +88,7 @@ func Test_processLine(t *testing.T) {
lineNumber: 1,
},
expectedResult: expectedResult{
wantModel: &LogModel{
wantModel: &model.LogModel{
LogTime: time.Date(2018, 02, 01, 15, 04, 05, 0, time.UTC),
LogMsg: `This is log message`,
LogFormat: `second_format`,
Expand All @@ -99,7 +97,7 @@ func Test_processLine(t *testing.T) {
wantErr: false,
},
},
testCase{
{
id: 5,
description: `Negative case. Format missmatch - time format is second, but in config specified that file has first`,
input: input{
Expand All @@ -113,7 +111,7 @@ func Test_processLine(t *testing.T) {
wantErr: true,
},
},
testCase{
{
id: 6,
description: `Negative case. Not supported format specified in config`,
input: input{
Expand All @@ -127,7 +125,7 @@ func Test_processLine(t *testing.T) {
wantErr: true,
},
},
testCase{
{
id: 7,
description: `Negative case. Format missmatch - time format is first, but in config specified that file has second`,
input: input{
Expand All @@ -141,7 +139,7 @@ func Test_processLine(t *testing.T) {
wantErr: true,
},
},
testCase{
{
id: 8,
description: `Negative case. Empty line received`,
input: input{
Expand All @@ -155,7 +153,7 @@ func Test_processLine(t *testing.T) {
wantErr: true,
},
},
testCase{
{
id: 9,
description: `Negative case. Failed to parse time`,
input: input{
Expand All @@ -171,18 +169,19 @@ func Test_processLine(t *testing.T) {
},
}

for _, tc := range ts {
//if tc.id == 8 {
for _, tc := range tests {
t.Run(fmt.Sprintf("Test%d:%s", tc.id, tc.description), func(t *testing.T) {
gotModel, err := processLine(tc.input.logName, tc.input.line, tc.input.format, tc.input.lineNumber)
if (err != nil) != tc.expectedResult.wantErr {
t.Errorf("processLine() error = %+v, \nwantErr %+v", err, tc.expectedResult.wantErr)
return
}
if !reflect.DeepEqual(gotModel, tc.expectedResult.wantModel) {
t.Errorf("processLine() = %+v, \nwant %+v", gotModel, tc.expectedResult.wantModel)

switch tc.expectedResult.wantErr {
case true:
assert.Error(t, err, "Expected to receive error")
case false:
assert.NoError(t, err, "Unexpected error")
}

assert.Equal(t, tc.expectedResult.wantModel, gotModel)

})
}
//}
}
35 changes: 35 additions & 0 deletions db/interface.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package db

import (
"fmt"

"github.com/oleg-balunenko/logs-converter/db/mongo"
"github.com/oleg-balunenko/logs-converter/model"
)

const ( // database types

// Mongo - mongo db type
Mongo = "mongo"
)

// Repository is a contract for databases
type Repository interface {
Store(logModel *model.LogModel) (string, error)
Update(id string, logModel model.LogModel) error
Delete(id string) error
Drop(bool) error
Close()
}

// Connect establish connection to passed database type
func Connect(dbType string, url string, dbName string, colletionName string, username string, password string) (Repository, error) {
switch dbType {
case Mongo:
return mongo.NewMongoDBConnection(url, dbName, colletionName, username, password), nil
default:
return Repository(nil), fmt.Errorf("not supported database type [%s]", dbType)

}

}
Loading

0 comments on commit a54c6dd

Please sign in to comment.