forked from aliyun/aliyun-log-go-sdk
/
tasks.go
90 lines (81 loc) · 3.08 KB
/
tasks.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
package consumerLibrary
import (
"errors"
"fmt"
"runtime"
"time"
sls "github.com/aliyun/aliyun-log-go-sdk"
"github.com/go-kit/kit/log/level"
)
func (consumer *ShardConsumerWorker) consumerInitializeTask() (string, error) {
checkpoint, err := consumer.client.getCheckPoint(consumer.shardId)
if err != nil {
return checkpoint, err
}
if checkpoint != "" && err == nil {
consumer.consumerCheckPointTracker.setPersistentCheckPoint(checkpoint)
return checkpoint, nil
}
if consumer.client.option.CursorPosition == BEGIN_CURSOR {
cursor, err := consumer.client.getCursor(consumer.shardId, "begin")
if err != nil {
level.Warn(consumer.logger).Log("msg", "get beginCursor error", "shard", consumer.shardId, "error", err)
}
return cursor, err
}
if consumer.client.option.CursorPosition == END_CURSOR {
cursor, err := consumer.client.getCursor(consumer.shardId, "end")
if err != nil {
level.Warn(consumer.logger).Log("msg", "get endCursor error", "shard", consumer.shardId, "error", err)
}
return cursor, err
}
if consumer.client.option.CursorPosition == SPECIAL_TIMER_CURSOR {
cursor, err := consumer.client.getCursor(consumer.shardId, fmt.Sprintf("%v", consumer.client.option.CursorStartTime))
if err != nil {
level.Warn(consumer.logger).Log("msg", "get specialCursor error", "shard", consumer.shardId, "error", err)
}
return cursor, err
}
level.Info(consumer.logger).Log("msg", "CursorPosition setting error, please reset with BEGIN_CURSOR or END_CURSOR or SPECIAL_TIMER_CURSOR")
return "", errors.New("CursorPositionError")
}
func (consumer *ShardConsumerWorker) consumerFetchTask() (*sls.LogGroupList, string, error) {
logGroup, next_cursor, err := consumer.client.pullLogs(consumer.shardId, consumer.nextFetchCursor)
return logGroup, next_cursor, err
}
func (consumer *ShardConsumerWorker) consumerProcessTask() string {
// If the user's consumption function reports a panic error, it will be captured and retry until sucessed.
defer func() {
if r := recover(); r != nil {
stackBuf := make([]byte, 1<<16)
runtime.Stack(stackBuf, false)
level.Error(consumer.logger).Log("msg", "get panic in your process function", "error", r, "stack", string(stackBuf))
for {
if consumer.consumerRetryProcessTask() {
break
} else {
time.Sleep(time.Second * 2)
}
}
}
}()
if consumer.lastFetchLogGroupList != nil {
consumer.rollBackCheckpoint = consumer.process(consumer.shardId, consumer.lastFetchLogGroupList)
consumer.consumerCheckPointTracker.flushCheck()
}
return consumer.rollBackCheckpoint
}
func (consumer *ShardConsumerWorker) consumerRetryProcessTask() bool {
level.Info(consumer.logger).Log("msg", "Start retrying the process function")
defer func() {
if r := recover(); r != nil {
stackBuf := make([]byte, 1<<16)
runtime.Stack(stackBuf, false)
level.Error(consumer.logger).Log("msg", "get panic in your process function", "error", r, "stack", string(stackBuf))
}
}()
consumer.rollBackCheckpoint = consumer.process(consumer.shardId, consumer.lastFetchLogGroupList)
consumer.consumerCheckPointTracker.flushCheck()
return true
}