-
-
Notifications
You must be signed in to change notification settings - Fork 3
/
LogManager.go
107 lines (98 loc) · 2.83 KB
/
LogManager.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
package worker
import (
"context"
"github.com/m9rco/exile/kernel/common"
"github.com/m9rco/exile/kernel/utils"
"github.com/mongodb/mongo-go-driver/mongo"
"github.com/mongodb/mongo-go-driver/mongo/clientopt"
"time"
)
type LogManager struct {
client *mongo.Client
logCollection *mongo.Collection
logChan chan *common.JobLog
autoCommitChan chan *common.LogBatch
}
// initialize the InitLogSink
func InitLogSink() (err error) {
var (
client *mongo.Client
configure utils.IniParser
LogManagerSev LogManager
)
configure = common.Manage.GetSingleton("configure").(utils.IniParser)
// init a mango connect
if client, err = mongo.Connect(
context.TODO(),
configure.GetString("mongodb", "endpoints"),
clientopt.ConnectTimeout(time.Duration(configure.GetInt32("mongodb", "connect_timeout"))*time.Millisecond)); err != nil {
return
}
// switch the db and collection
common.Manage.SetSingleton("LogManager", LogManager{
client: client,
logCollection: client.Database(configure.GetString("mongodb", "db")).Collection(configure.GetString("mongodb", "logger")),
logChan: make(chan *common.JobLog, 1000),
autoCommitChan: make(chan *common.LogBatch, 1000),
})
LogManagerSev = common.Manage.GetSingleton("LogManager").(LogManager)
// start mongodb goroutine
go LogManagerSev.writeLoop()
return
}
// send logger
func (logSink *LogManager) Append(jobLog *common.JobLog) {
select {
case logSink.logChan <- jobLog:
default:
}
}
// batch write logger
func (logSink *LogManager) saveLogs(batch *common.LogBatch) {
logSink.logCollection.InsertMany(context.TODO(), batch.Logs)
}
// logger save goroutine
func (logSink *LogManager) writeLoop() {
var (
log *common.JobLog
logBatch *common.LogBatch
commitTimer *time.Timer
timeoutBatch *common.LogBatch
configure utils.IniParser
)
configure = common.Manage.GetSingleton("configure").(utils.IniParser)
for {
select {
case log = <-logSink.logChan:
if logBatch == nil {
logBatch = &common.LogBatch{}
// set the timeout automatically submit (1s)
commitTimer = time.AfterFunc(
time.Duration(configure.GetInt32("job", "commit_timeout"))*time.Millisecond,
func(batch *common.LogBatch) func() {
return func() {
logSink.autoCommitChan <- batch
}
}(logBatch),
)
}
// create new logger append
logBatch.Logs = append(logBatch.Logs, log)
// if batch size is full, that is send logger !
if len(logBatch.Logs) >= int(configure.GetInt32("job", "batch_size")) {
logSink.saveLogs(logBatch)
// clean logger batch
logBatch = nil
commitTimer.Stop()
}
case timeoutBatch = <-logSink.autoCommitChan: // overdue batch
if timeoutBatch != logBatch {
continue
}
// save loggers
logSink.saveLogs(timeoutBatch)
// clean logger batch
logBatch = nil
}
}
}