forked from Terry-Mao/gopush-cluster
-
Notifications
You must be signed in to change notification settings - Fork 0
/
message.go
228 lines (208 loc) · 6.82 KB
/
message.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
// Copyright © 2014 Terry Mao, LiuDing All rights reserved.
// This file is part of gopush-cluster.
// gopush-cluster is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// gopush-cluster is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with gopush-cluster. If not, see <http://www.gnu.org/licenses/>.
package rpc
import (
"encoding/json"
myzk "github.com/Terry-Mao/gopush-cluster/zk"
"github.com/golang/glog"
"github.com/samuel/go-zookeeper/zk"
"net/rpc"
"path"
"strings"
"time"
)
const (
// group id
PrivateGroupId = 0
PublicGroupId = 1
// message rpc service
MessageService = "MessageRPC"
MessageServiceGetPrivate = "MessageRPC.GetPrivate"
MessageServiceSavePrivate = "MessageRPC.SavePrivate"
MessageServiceDelPrivate = "MessageRPC.DelPrivate"
)
type MessageNodeEvent struct {
// addr:port
Key string
// event type
Event int
}
// The Message struct
type Message struct {
Msg json.RawMessage `json:"msg"` // message content
MsgId int64 `json:"mid"` // message id
GroupId uint `json:"gid"` // group id
}
// The Old Message struct (Compatible), TODO remove it.
type OldMessage struct {
Msg string `json:"msg"` // Message
MsgId int64 `json:"mid"` // Message id
GroupId uint `json:"gid"` // Group id
}
// Bytes get a message reply bytes.
func (m *Message) Bytes() ([]byte, error) {
byteJson, err := json.Marshal(m)
if err != nil {
glog.Errorf("json.Marshal(%v) error(%v)", m, err)
return nil, err
}
return byteJson, nil
}
// OldBytes get a message reply bytes(Compatible), TODO remove it.
func (m *Message) OldBytes() ([]byte, error) {
om := &OldMessage{Msg: string(m.Msg), MsgId: m.MsgId, GroupId: m.GroupId}
byteJson, err := json.Marshal(om)
if err != nil {
glog.Errorf("json.Marshal(%v) error(%v)", om, err)
return nil, err
}
return byteJson, nil
}
// Message SavePrivate Args
type MessageSavePrivateArgs struct {
Key string // subscriber key
Msg json.RawMessage // message content
MsgId int64 // message id
Expire uint // message expire second
}
// Message SavePublish Args
type MessageSavePublishArgs struct {
MsgID int64 // message id
Msg string // message content
Expire int64 // message expire second
}
// Message Get Args
type MessageGetPrivateArgs struct {
MsgId int64 // message id
Key string // subscriber key
}
// Message Get Response
type MessageGetResp struct {
Msgs []*Message // messages
}
var (
MessageRPC *RandLB
)
func init() {
MessageRPC, _ = NewRandLB(map[string]*rpc.Client{}, []string{}, MessageService, 0, 0, false)
}
// watchMessageRoot watch the message root path.
func watchMessageRoot(conn *zk.Conn, fpath string, ch chan *MessageNodeEvent) error {
for {
nodes, watch, err := myzk.GetNodesW(conn, fpath)
if err == myzk.ErrNodeNotExist {
glog.Warningf("zk don't have node \"%s\", retry in %d second", fpath, waitNodeDelay)
time.Sleep(waitNodeDelaySecond)
continue
} else if err == myzk.ErrNoChild {
glog.Warningf("zk don't have any children in \"%s\", retry in %d second", fpath, waitNodeDelay)
// all child died, kick all the nodes
for node, _ := range MessageRPC.Clients {
glog.V(1).Infof("node: \"%s\" send del node event", node)
ch <- &MessageNodeEvent{Event: eventNodeDel, Key: node}
}
time.Sleep(waitNodeDelaySecond)
continue
} else if err != nil {
glog.Errorf("getNodes error(%v), retry in %d second", err, waitNodeDelay)
time.Sleep(waitNodeDelaySecond)
continue
}
nodesMap := map[string]bool{}
// handle new add nodes
for _, node := range nodes {
data, _, err := conn.Get(path.Join(fpath, node))
if err != nil {
glog.Errorf("zk.Get(\"%s\") error(%v)", path.Join(fpath, node), err)
continue
}
// may contains many addrs split by ,
addrs := strings.Split(string(data), ",")
for _, addr := range addrs {
// if not exists in old map then trigger a add event
if _, ok := MessageRPC.Clients[addr]; !ok {
ch <- &MessageNodeEvent{Event: eventNodeAdd, Key: addr}
}
nodesMap[addr] = true
}
}
// handle delete nodes
for node, _ := range MessageRPC.Clients {
if _, ok := nodesMap[node]; !ok {
ch <- &MessageNodeEvent{Event: eventNodeDel, Key: node}
}
}
// blocking wait node changed
event := <-watch
glog.Infof("zk path: \"%s\" receive a event %v", fpath, event)
}
}
// handleNodeEvent add and remove MessageRPC.Clients, copy the src map to a new map then replace the variable.
func handleMessageNodeEvent(conn *zk.Conn, retry, ping time.Duration, ch chan *MessageNodeEvent) {
for {
ev := <-ch
// copy map from src
tmpMessageRPCMap := make(map[string]*rpc.Client, len(MessageRPC.Clients))
for k, v := range MessageRPC.Clients {
tmpMessageRPCMap[k] = v
}
// handle event
if ev.Event == eventNodeAdd {
glog.Infof("add message rpc node: \"%s\"", ev.Key)
// if exist old node info, reuse
if client, ok := MessageRPC.Clients[ev.Key]; ok && client != nil {
tmpMessageRPCMap[ev.Key] = client
glog.Infof("reuse message rpc node: \"%s\"", ev.Key)
} else {
rpcTmp, err := rpc.Dial("tcp", ev.Key)
if err != nil {
glog.Errorf("rpc.Dial(\"tcp\", \"%s\") error(%v)", ev.Key, err)
glog.Warningf("discard message rpc node: \"%s\", connect failed", ev.Key)
continue
}
tmpMessageRPCMap[ev.Key] = rpcTmp
}
} else if ev.Event == eventNodeDel {
glog.Infof("del message rpc node: \"%s\"", ev.Key)
delete(tmpMessageRPCMap, ev.Key)
// if exist old node info, close
if client, ok := MessageRPC.Clients[ev.Key]; ok && client != nil {
client.Close()
glog.Infof("close message rpc node: \"%s\"", ev.Key)
}
} else {
glog.Errorf("unknown node event: %d", ev.Event)
panic("unknown node event")
}
addrs := make([]string, 0, len(tmpMessageRPCMap))
for addr, _ := range tmpMessageRPCMap {
addrs = append(addrs, addr)
}
tmpMessageRPC, err := NewRandLB(tmpMessageRPCMap, addrs, MessageService, retry, ping, true)
if err != nil {
glog.Errorf("NewRandLR() error(%v)", err)
panic(err)
}
// atomic update
MessageRPC.Stop()
MessageRPC = tmpMessageRPC
glog.V(1).Infof("MessageRPC.Client length: %d", len(MessageRPC.Clients))
}
}
// InitMessage init a rand lb rpc for message module.
func InitMessage(conn *zk.Conn, fpath string, retry, ping time.Duration) {
// watch message path
ch := make(chan *MessageNodeEvent, 1024)
go handleMessageNodeEvent(conn, retry, ping, ch)
go watchMessageRoot(conn, fpath, ch)
}