/
worker.go
154 lines (138 loc) · 4.11 KB
/
worker.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
package redischeduler
import (
"context"
"errors"
"fmt"
"github.com/go-redis/redis/v8"
"log"
"os"
"reflect"
"time"
)
type Worker interface {
Run()
}
type PartitionChannel struct {
partitionRedis *PartitionRedis
Channel chan WorkerTask
sleep time.Duration
logger *log.Logger
script *redis.Script
}
func (c *PartitionChannel) Run() {
for {
taskId, found, err := c.findTask()
if err != nil {
c.logger.Printf("PartitionChannel find task eror:%v\n", err)
}
if !found {
time.Sleep(c.sleep)
} else {
c.logger.Println("PartitionChannel task has found:", taskId)
c.Channel <- *NewWorkerTask(taskId)
}
}
}
func NewPartitionChannel(pr *PartitionRedis, sleep time.Duration, logger *log.Logger) *PartitionChannel {
pc := &PartitionChannel{
partitionRedis: pr,
Channel: make(chan WorkerTask),
sleep: sleep,
logger: logger,
script: redis.NewScript(`local function findTask(key)
redis.replicate_commands()
local t = redis.call('TIME')
t = t[1] * 1000 + t[2]/100
local tasks = redis.call("ZRANGEBYSCORE",key, 0, t, "limit", 0, 1)
if tasks[1] == nil then
return nil
else
if redis.call("ZREM",key, tasks[1]) == 1 then
redis.call("DEL", tasks[1])
return tasks[1]
end
end
return nil
end
return findTask(KEYS[1])`),
}
if logger == nil {
pc.logger = log.New(os.Stdout, fmt.Sprintf("PartitionChannel-%s ", pr.Node()), log.LstdFlags)
}
return pc
}
func (c *PartitionChannel) findTask() (taskId string, found bool, err error) {
ctx := context.Background()
partitionKey := c.partitionRedis.Node()
cmd := c.script.Run(ctx, c.partitionRedis.client, []string{partitionKey})
if cmd.Val() != nil {
found = true
taskId = cmd.Val().(string)
}
if cmd.Err() != redis.Nil {
err = cmd.Err()
}
return taskId, found, err
}
type SinglePartitionWorker struct {
Partition *Partition
PartitionChannels []*PartitionChannel
TaskInvoker Invoker
Logger *log.Logger
}
type SinglePartitionWorkerConfig struct {
PartitionId int
PartitionSize int
PartitionRedisConfig []*PartitionRedisConfig
TaskInvoker Invoker
Logger *log.Logger
}
func NewSinglePartitionWorker(config *SinglePartitionWorkerConfig) (w *SinglePartitionWorker, err error) {
if config.TaskInvoker == nil {
err = errors.New("TaskInvoker undefined!")
}
if config.Logger == nil {
config.Logger = log.New(os.Stdout, "SinglePartitionWorker ", log.LstdFlags)
}
partitionShards := len(config.PartitionRedisConfig)
var partitionChannels = make([]*PartitionChannel, partitionShards)
for i, c := range config.PartitionRedisConfig {
config.Logger.Println("Start init PartitionRedis, partitionId", c.PartitionId, "shardingId", c.ShardingId)
pr := NewPartitionRedis(config.PartitionRedisConfig[i])
partitionChannels[i] = NewPartitionChannel(pr, 1*time.Second, nil)
}
singlePartition := NewSinglePartition(config.PartitionId, config.PartitionSize, partitionShards)
w = &SinglePartitionWorker{
Partition: singlePartition,
PartitionChannels: partitionChannels,
TaskInvoker: config.TaskInvoker,
Logger: config.Logger,
}
return w, err
}
func (w *SinglePartitionWorker) Run() {
w.Logger.Println("Run SinglePartitionWorker! Partition id:", w.Partition.Id)
cases := make([]reflect.SelectCase, len(w.PartitionChannels))
for i, partitionChannel := range w.PartitionChannels {
w.Logger.Println("Run partitionChannel", partitionChannel.partitionRedis.Node())
go partitionChannel.Run()
cases[i] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(partitionChannel.Channel)}
}
for len(cases) > 0 {
chosen, value, ok := reflect.Select(cases)
if !ok {
w.Logger.Printf("The chosen channel %v has been closed\n", cases[chosen])
cases[chosen].Chan = reflect.ValueOf(nil)
cases = append(cases[:chosen], cases[chosen+1:]...)
continue
}
workerTask := value.Interface().(WorkerTask)
err := w.TaskInvoker.Call(workerTask)
if err != nil {
w.Logger.Printf("The chosen channel %v invoker error: %v\n", cases[chosen], err)
}
}
}
func (s *SinglePartitionWorker) RunFunc(f func()) {
f()
}