/
rlogs.go
106 lines (89 loc) · 2.15 KB
/
rlogs.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
package rlogs
import (
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
// Version of rlogs pacakge
const Version = "v0.1.1"
// LogQueue is message queue between Reader and main procedure.
// It includes both of LogRecord and Error but should be set either one.
type LogQueue struct {
Log *LogRecord
Error error
}
// LogRecord has not only log message (original log) but also parsed meta data.
type LogRecord struct {
// Tag indicates log type (log schema)
Tag string
// Timestamp comes from log data
Timestamp time.Time
// Raw is raw log data
Raw []byte
// Value is parsed log data
Values interface{}
// Sequence number in log object
Seq int
// Log source location
Src LogSource
}
// MessageQueue is a queue bring raw log message and sequence between Loader and Parser
type MessageQueue struct {
Error error
Raw []byte
Seq int
Src LogSource
}
// Parser converts raw log message to LogRecord(s)
type Parser interface {
Parse(msg *MessageQueue) ([]*LogRecord, error)
}
// Loader downloads object from cloud object storage and create MessageQueue(s)
type Loader interface {
Load(src LogSource) chan *MessageQueue
}
// Pipeline is a pair of Parser and Loader.
type Pipeline struct {
Ldr Loader
Psr Parser
QueueSize int
}
// Run of Pipeline downloads object and parse it.
func (x *Pipeline) Run(src LogSource, ch chan *LogQueue) {
defer close(ch)
msgch := x.Ldr.Load(src)
if msgch == nil {
return // ignore
}
for msg := range msgch {
if msg.Error != nil {
ch <- &LogQueue{
Error: errors.Wrap(msg.Error, "Fail to load log message"),
Log: &LogRecord{
Raw: msg.Raw,
},
}
return
}
logs, err := x.Psr.Parse(msg)
if err != nil {
ch <- &LogQueue{
Error: errors.Wrap(err, "Fail to parse log message"),
Log: &LogRecord{
Raw: msg.Raw,
},
}
return
}
for i := range logs {
ch <- &LogQueue{Log: logs[i]}
}
}
}
// Logger is logrus based logger and exposed to be controlled from outside also.
var Logger = logrus.New()
func init() {
Logger.SetLevel(logrus.InfoLevel)
}
// String function just converts string to string pointer
func String(s string) *string { return &s }