Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 332 lines (257 sloc) 10.503 kb
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
1 import logging
aa23b7b @labisso Better ensure_path() behavior
labisso authored
2 from os.path import split
33cd152 @labisso Add default ACL support to KazooClient
labisso authored
3 import hashlib
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
4
5 from kazoo.zkclient import ZooKeeperClient, WatchedEvent, KeeperState,\
33cd152 @labisso Add default ACL support to KazooClient
labisso authored
6 EventType, NodeExistsException, NoNodeException, AclPermission
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
7 from kazoo.retry import KazooRetry
8
9 log = logging.getLogger(__name__)
10
11
12 class KazooState(object):
13 """High level connection state values
14
15 States inspired by Netflix Curator
16 """
17
18 # The connection has been lost but may be recovered.
19 # We should operate in a "safe mode" until then.
20 SUSPENDED = "SUSPENDED"
21
22 # The connection is alive and well.
23 CONNECTED = "CONNECTED"
24
25 # The connection has been confirmed dead.
26 LOST = "LOST"
27
28
33cd152 @labisso Add default ACL support to KazooClient
labisso authored
29 def make_digest_acl_credential(username, password):
30 credential = "%s:%s" % (username, password)
31 cred_hash = hashlib.sha1(credential).digest().encode('base64').strip()
32 return "%s:%s" % (username, cred_hash)
33
34 def make_acl(scheme, credential, read=False, write=False,
35 create=False, delete=False, admin=False, all=False):
36 if all:
37 permissions = AclPermission.ALL
38 else:
39 permissions = 0
40 if read:
41 permissions |= AclPermission.READ
42 if write:
43 permissions |= AclPermission.WRITE
44 if create:
45 permissions |= AclPermission.CREATE
46 if delete:
47 permissions |= AclPermission.DELETE
48 if admin:
49 permissions |= AclPermission.ADMIN
50
51 return dict(scheme=scheme, id=credential, perms=permissions)
52
53 def make_digest_acl(username, password, read=False, write=False,
54 create=False, delete=False, admin=False, all=False):
55 cred = make_digest_acl_credential(username, password)
56 return make_acl("digest", cred, read=read, write=write, create=create,
57 delete=delete, admin=admin, all=all)
58
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
59 class KazooClient(object):
60 """Higher-level ZooKeeper client.
61
62 Supports retries, namespacing, easier state monitoring; saves kittens.
63 """
64
33cd152 @labisso Add default ACL support to KazooClient
labisso authored
65 def __init__(self, hosts, namespace=None, timeout=10.0, max_retries=None, default_acl=None):
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
66 # remove any trailing slashes
67 if namespace:
68 namespace = namespace.rstrip('/')
69 if namespace:
70 validate_path(namespace)
71 self.namespace = namespace
72
ad86a5c @labisso Recursive path creation
labisso authored
73 self._needs_ensure_path = bool(namespace)
74
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
75 self.zk = ZooKeeperClient(hosts, watcher=self._session_watcher,
76 timeout=timeout)
77 self.retry = KazooRetry(max_retries)
78
79 self.state = KazooState.LOST
80 self.state_listeners = set()
81
33cd152 @labisso Add default ACL support to KazooClient
labisso authored
82 self.default_acl = default_acl
83
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
84 def _session_watcher(self, event):
85 """called by the underlying ZK client when the connection state changes
86 """
87
88 if event.type != EventType.SESSION:
89 return
90
91 if event.state == KeeperState.CONNECTED:
ad86a5c @labisso Recursive path creation
labisso authored
92 self._make_state_change(KazooState.CONNECTED)
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
93 elif event.state in (KeeperState.AUTH_FAILED,
94 KeeperState.EXPIRED_SESSION):
ad86a5c @labisso Recursive path creation
labisso authored
95 self._make_state_change(KazooState.LOST)
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
96 elif event.state == KeeperState.CONNECTING:
ad86a5c @labisso Recursive path creation
labisso authored
97 self._make_state_change(KazooState.SUSPENDED)
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
98
ad86a5c @labisso Recursive path creation
labisso authored
99 def _make_state_change(self, state):
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
100 # skip if state is current
101 if self.state == state:
102 return
103
104 self.state = state
105
106 listeners = list(self.state_listeners)
107 for listener in listeners:
108 try:
109 listener(state)
110 except Exception:
111 log.exception("Error in connection state listener")
112
33cd152 @labisso Add default ACL support to KazooClient
labisso authored
113 def _assure_namespace(self, acl=None):
ad86a5c @labisso Recursive path creation
labisso authored
114 if self._needs_ensure_path:
33cd152 @labisso Add default ACL support to KazooClient
labisso authored
115 self.ensure_path('/', acl=acl)
ad86a5c @labisso Recursive path creation
labisso authored
116 self._needs_ensure_path = False
117
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
118 def add_listener(self, listener):
119 """Add a function to be called for connection state changes
120 """
121 if not (listener and callable(listener)):
122 raise ValueError("listener must be callable")
123 self.state_listeners.add(listener)
124
125 def remove_listener(self, listener):
126 """Remove a listener function
127 """
128 self.state_listeners.discard(listener)
129
75473fd @labisso Add client_id support
labisso authored
130 @property
131 def client_id(self):
132 return self.zk.client_id
133
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
134 def connect(self, timeout=None):
135 """Initiate connection to ZK
136
137 @param timeout: time in seconds to wait for connection to succeed
138 """
139 self.zk.connect(timeout=timeout)
140
141 def close(self):
142 """Disconnect from ZooKeeper
143 """
144 self.zk.close()
145
a44e4f6 @labisso Support add_auth operation in KazooClient
labisso authored
146 def add_auth(self, scheme, credential):
147 """Send credentials to server
148
149 @param scheme: authentication scheme (default supported: "digest")
150 @param credential: the credential -- value depends on scheme
151 """
152 self.zk.add_auth(scheme, credential)
153
5ffa429 @labisso Add makepath feature to KazooClient.create()
labisso authored
154 def create(self, path, value, acl=None, ephemeral=False, sequence=False,
155 makepath=False):
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
156 """Create a ZNode
157
158 @param path: path of node
159 @param value: initial value of node
160 @param acl: permissions for node
161 @param ephemeral: boolean indicating whether node is ephemeral (tied to this session)
162 @param sequence: boolean indicating whether path is suffixed with a unique index
5ffa429 @labisso Add makepath feature to KazooClient.create()
labisso authored
163 @param makepath: boolean indicating whether to create path if it doesn't exist
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
164 @return: real path of the new node
165 """
33cd152 @labisso Add default ACL support to KazooClient
labisso authored
166 self._assure_namespace(acl=acl)
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
167
168 path = self.namespace_path(path)
33cd152 @labisso Add default ACL support to KazooClient
labisso authored
169
170 if acl is None and self.default_acl:
171 acl = self.default_acl
172
5ffa429 @labisso Add makepath feature to KazooClient.create()
labisso authored
173 try:
174 realpath = self.zk.create(path, value, acl=acl,
175 ephemeral=ephemeral, sequence=sequence)
176
177 except NoNodeException:
178 # some or all of the parent path doesn't exist. if makepath is set
179 # we will create it and retry. If it fails again, someone must be
180 # actively deleting ZNodes and we'd best bail out.
181 if not makepath:
182 raise
183
184 parent, _ = split(path)
185
186 # using the inner call directly because path is already namespaced
33cd152 @labisso Add default ACL support to KazooClient
labisso authored
187 self._inner_ensure_path(parent, acl)
5ffa429 @labisso Add makepath feature to KazooClient.create()
labisso authored
188
189 # now retry
190 realpath = self.zk.create(path, value, acl=acl,
191 ephemeral=ephemeral, sequence=sequence)
192
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
193 return self.unnamespace_path(realpath)
194
195 def exists(self, path, watch=None):
196 """Check if a node exists
197
198 @param path: path of node
199 @param watch: optional watch callback to set for future changes to this path
200 @return stat of the node if it exists, else None
201 """
ad86a5c @labisso Recursive path creation
labisso authored
202
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
203 path = self.namespace_path(path)
204 if watch:
205 watch = self.unnamespace_watch(watch)
206 return self.zk.exists(path, watch)
207
208 def get(self, path, watch=None):
209 """Get the value of a node
210
211 @param path: path of node
212 @param watch: optional watch callback to set for future changes to this path
213 @return tuple (value, stat) of node
214 """
ad86a5c @labisso Recursive path creation
labisso authored
215
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
216 path = self.namespace_path(path)
217 if watch:
218 watch = self.unnamespace_watch(watch)
219 return self.zk.get(path, watch)
220
221 def get_children(self, path, watch=None):
222 """Get a list of child nodes of a path
223
224 @param path: path of node to list
225 @param watch: optional watch callback to set for future changes to this path
226 @return: list of child node names
227 """
ad86a5c @labisso Recursive path creation
labisso authored
228
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
229 path = self.namespace_path(path)
230 if watch:
231 watch = self.unnamespace_watch(watch)
232 return self.zk.get_children(path, watch)
233
234 def set(self, path, data, version=-1):
235 """Set the value of a node
236
237 If the version of the node being updated is newer than the supplied
238 version (and the supplied version is not -1), a BadVersionException
239 will be raised.
240
241 @param path: path of node to set
242 @param data: new data value
243 @param version: version of node being updated, or -1
244 @return: updated node stat
245 """
ad86a5c @labisso Recursive path creation
labisso authored
246 self._assure_namespace()
247
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
248 path = self.namespace_path(path)
249 return self.zk.set(path, data, version)
250
251 def delete(self, path, version=-1):
252 """Delete a node
253
254 @param path: path of node to delete
255 @param version: version of node to delete, or -1 for any
256 """
257 path = self.namespace_path(path)
258 return self.zk.delete(path, version)
259
33cd152 @labisso Add default ACL support to KazooClient
labisso authored
260 def ensure_path(self, path, acl=None):
ad86a5c @labisso Recursive path creation
labisso authored
261 """Recursively create a path if it doesn't exist
262 """
263 path = self.namespace_path(path)
33cd152 @labisso Add default ACL support to KazooClient
labisso authored
264 self._inner_ensure_path(path, acl)
ad86a5c @labisso Recursive path creation
labisso authored
265
33cd152 @labisso Add default ACL support to KazooClient
labisso authored
266 def _inner_ensure_path(self, path, acl):
aa23b7b @labisso Better ensure_path() behavior
labisso authored
267 if self.zk.exists(path):
268 return
269
33cd152 @labisso Add default ACL support to KazooClient
labisso authored
270 if acl is None and self.default_acl:
271 acl = self.default_acl
272
aa23b7b @labisso Better ensure_path() behavior
labisso authored
273 parent, node = split(path)
ad86a5c @labisso Recursive path creation
labisso authored
274
aa23b7b @labisso Better ensure_path() behavior
labisso authored
275 if parent != "/":
33cd152 @labisso Add default ACL support to KazooClient
labisso authored
276 self._inner_ensure_path(parent, acl)
aa23b7b @labisso Better ensure_path() behavior
labisso authored
277 try:
33cd152 @labisso Add default ACL support to KazooClient
labisso authored
278 self.zk.create(path, "", acl=acl)
aa23b7b @labisso Better ensure_path() behavior
labisso authored
279 except NodeExistsException:
280 # someone else created the node. how sweet!
281 pass
ad86a5c @labisso Recursive path creation
labisso authored
282
b9fbe49 @labisso Added recursive_delete method to KazooClient
labisso authored
283 def recursive_delete(self, path):
284 """Recursively delete a ZNode and all of its children
285 """
a61e949 @labisso Make tests clean up after themselves
labisso authored
286 try:
287 children = self.get_children(path)
288 except NoNodeException:
289 return
290
b9fbe49 @labisso Added recursive_delete method to KazooClient
labisso authored
291 if children:
292 for child in children:
293 self.recursive_delete(path + "/" + child)
294 try:
295 self.delete(path)
296 except NoNodeException:
297 pass
298
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
299 def with_retry(self, func, *args, **kwargs):
300 """Run a method repeatedly in the face of transient ZK errors
301 """
302 return self.retry(func, *args, **kwargs)
303
304 def namespace_path(self, path):
305 if not self.namespace:
306 return path
307 validate_path(path)
aa23b7b @labisso Better ensure_path() behavior
labisso authored
308 path = self.namespace + path
309 path = path.rstrip('/')
310 return path
c32994e @labisso Added higher level ZK client, KazooClient
labisso authored
311
312 def unnamespace_path(self, path):
313 if not self.namespace:
314 return path
315 if path.startswith(self.namespace):
316 return path[len(self.namespace):]
317
318 def unnamespace_watch(self, watch):
319 def fixed_watch(event):
320 if event.path:
321 # make a new event with the fixed path
322 path = self.unnamespace_path(event.path)
323 event = WatchedEvent(event.type, event.state, path)
324
325 watch(event)
326
327 return fixed_watch
328
329 def validate_path(path):
330 if not path.startswith('/'):
331 raise ValueError("invalid path '%s'. must start with /" % path)
Something went wrong with that request. Please try again.