-
Notifications
You must be signed in to change notification settings - Fork 0
/
LogSink.go
113 lines (99 loc) · 2.54 KB
/
LogSink.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
package worker
import (
"context"
"time"
"github.com/luoruofeng/crontab/common"
"github.com/mongodb/mongo-go-driver/mongo"
"github.com/mongodb/mongo-go-driver/mongo/clientopt"
)
// mongodb存储日志
type LogSink struct {
client *mongo.Client
logCollection *mongo.Collection
logChan chan *common.JobLog
autoCommitChan chan *common.LogBatch
}
var (
// 单例
G_logSink *LogSink
)
// 批量写入日志
func (logSink *LogSink) saveLogs(batch *common.LogBatch) {
logSink.logCollection.InsertMany(context.TODO(), batch.Logs)
}
// 日志存储协程
func (logSink *LogSink) writeLoop() {
var (
log *common.JobLog
logBatch *common.LogBatch // 当前的批次
commitTimer *time.Timer
timeoutBatch *common.LogBatch // 超时批次
)
for {
select {
case log = <-logSink.logChan:
if logBatch == nil {
logBatch = &common.LogBatch{}
// 让这个批次超时自动提交(给1秒的时间)
commitTimer = time.AfterFunc(
time.Duration(G_config.JobLogCommitTimeout)*time.Millisecond,
func(batch *common.LogBatch) func() {
return func() {
logSink.autoCommitChan <- batch
}
}(logBatch),
)
}
// 把新日志追加到批次中
logBatch.Logs = append(logBatch.Logs, log)
// 如果批次满了, 就立即发送
if len(logBatch.Logs) >= G_config.JobLogBatchSize {
// 发送日志
logSink.saveLogs(logBatch)
// 清空logBatch
logBatch = nil
// 取消定时器
commitTimer.Stop()
}
case timeoutBatch = <-logSink.autoCommitChan: // 过期的批次
// 判断过期批次是否仍旧是当前的批次
if timeoutBatch != logBatch {
continue // 跳过已经被提交的批次
}
// 把批次写入到mongo中
logSink.saveLogs(timeoutBatch)
// 清空logBatch
logBatch = nil
}
}
}
func InitLogSink() (err error) {
var (
client *mongo.Client
)
// 建立mongodb连接
if client, err = mongo.Connect(
context.TODO(),
G_config.MongodbUri,
clientopt.ConnectTimeout(time.Duration(G_config.MongodbConnectTimeout)*time.Millisecond)); err != nil {
return
}
// 选择db和collection
G_logSink = &LogSink{
client: client,
logCollection: client.Database("cron").Collection("log"),
logChan: make(chan *common.JobLog, 1000),
autoCommitChan: make(chan *common.LogBatch, 1000),
}
// 启动一个mongodb处理协程
go G_logSink.writeLoop()
return
}
// 发送日志
func (logSink *LogSink) Append(jobLog *common.JobLog) {
select {
case logSink.logChan <- jobLog:
default:
// 队列满了就丢弃
}
}