Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 542 lines (490 sloc) 21.72 kb
d5e07c2 mmmm goodies
Jeff Balogh authored
1 """
2 Code to configure logging using dictionaries, as per PEP 391.
3
4 http://bitbucket.org/vinay.sajip/dictconfig/src/tip/src/dictconfig.py
5 """
6 import logging.handlers
7 import re
8 import sys
9 import types
10
11 IDENTIFIER = re.compile('^[a-z_][a-z0-9_]*$', re.I)
12
45c51a7 @robhudson PEP8 pass
robhudson authored
13
d5e07c2 mmmm goodies
Jeff Balogh authored
14 def valid_ident(s):
15 m = IDENTIFIER.match(s)
16 if not m:
17 raise ValueError('Not a valid Python identifier: %r' % s)
18 return True
19
20 #
21 # This function is defined in logging only in recent versions of Python
22 #
23 try:
24 from logging import _checkLevel
25 except ImportError:
26 def _checkLevel(level):
27 if isinstance(level, int):
28 rv = level
29 elif str(level) == level:
30 if level not in logging._levelNames:
31 raise ValueError('Unknown level: %r' % level)
32 rv = logging._levelNames[level]
33 else:
34 raise TypeError('Level not an integer or a '
35 'valid string: %r' % level)
36 return rv
37
38 # The ConvertingXXX classes are wrappers around standard Python containers,
39 # and they serve to convert any suitable values in the container. The
40 # conversion converts base dicts, lists and tuples to their wrapped
41 # equivalents, whereas strings which match a conversion format are converted
42 # appropriately.
43 #
44 # Each wrapper should have a configurator attribute holding the actual
45 # configurator to use for conversion.
46
45c51a7 @robhudson PEP8 pass
robhudson authored
47
d5e07c2 mmmm goodies
Jeff Balogh authored
48 class ConvertingDict(dict):
49 """A converting dictionary wrapper."""
50
51 def __getitem__(self, key):
52 value = dict.__getitem__(self, key)
53 result = self.configurator.convert(value)
45c51a7 @robhudson PEP8 pass
robhudson authored
54 # If the converted value is different, save for next time
d5e07c2 mmmm goodies
Jeff Balogh authored
55 if value is not result:
56 self[key] = result
57 if type(result) in (ConvertingDict, ConvertingList,
58 ConvertingTuple):
59 result.parent = self
60 result.key = key
61 return result
62
63 def get(self, key, default=None):
64 value = dict.get(self, key, default)
65 result = self.configurator.convert(value)
45c51a7 @robhudson PEP8 pass
robhudson authored
66 # If the converted value is different, save for next time
d5e07c2 mmmm goodies
Jeff Balogh authored
67 if value is not result:
68 self[key] = result
69 if type(result) in (ConvertingDict, ConvertingList,
70 ConvertingTuple):
71 result.parent = self
72 result.key = key
73 return result
74
75 def pop(self, key, default=None):
76 value = dict.pop(self, key, default)
77 result = self.configurator.convert(value)
78 if value is not result:
79 if type(result) in (ConvertingDict, ConvertingList,
80 ConvertingTuple):
81 result.parent = self
82 result.key = key
83 return result
84
45c51a7 @robhudson PEP8 pass
robhudson authored
85
d5e07c2 mmmm goodies
Jeff Balogh authored
86 class ConvertingList(list):
87 """A converting list wrapper."""
88 def __getitem__(self, key):
89 value = list.__getitem__(self, key)
90 result = self.configurator.convert(value)
45c51a7 @robhudson PEP8 pass
robhudson authored
91 # If the converted value is different, save for next time
d5e07c2 mmmm goodies
Jeff Balogh authored
92 if value is not result:
93 self[key] = result
94 if type(result) in (ConvertingDict, ConvertingList,
95 ConvertingTuple):
96 result.parent = self
97 result.key = key
98 return result
99
100 def pop(self, idx=-1):
101 value = list.pop(self, idx)
102 result = self.configurator.convert(value)
103 if value is not result:
104 if type(result) in (ConvertingDict, ConvertingList,
105 ConvertingTuple):
106 result.parent = self
107 return result
108
45c51a7 @robhudson PEP8 pass
robhudson authored
109
d5e07c2 mmmm goodies
Jeff Balogh authored
110 class ConvertingTuple(tuple):
111 """A converting tuple wrapper."""
112 def __getitem__(self, key):
113 value = tuple.__getitem__(self, key)
114 result = self.configurator.convert(value)
115 if value is not result:
116 if type(result) in (ConvertingDict, ConvertingList,
117 ConvertingTuple):
118 result.parent = self
119 result.key = key
120 return result
121
45c51a7 @robhudson PEP8 pass
robhudson authored
122
d5e07c2 mmmm goodies
Jeff Balogh authored
123 class BaseConfigurator(object):
124 """
125 The configurator base class which defines some useful defaults.
126 """
127
128 CONVERT_PATTERN = re.compile(r'^(?P<prefix>[a-z]+)://(?P<suffix>.*)$')
129
130 WORD_PATTERN = re.compile(r'^\s*(\w+)\s*')
131 DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*')
132 INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*')
133 DIGIT_PATTERN = re.compile(r'^\d+$')
134
135 value_converters = {
45c51a7 @robhudson PEP8 pass
robhudson authored
136 'ext': 'ext_convert',
137 'cfg': 'cfg_convert',
d5e07c2 mmmm goodies
Jeff Balogh authored
138 }
139
45c51a7 @robhudson PEP8 pass
robhudson authored
140 # We might want to use a different one, e.g. importlib.
d5e07c2 mmmm goodies
Jeff Balogh authored
141 importer = __import__
142
143 def __init__(self, config):
144 self.config = ConvertingDict(config)
145 self.config.configurator = self
146
147 def resolve(self, s):
148 """
149 Resolve strings to objects using standard import and attribute
150 syntax.
151 """
152 name = s.split('.')
153 used = name.pop(0)
154 found = self.importer(used)
155 for frag in name:
156 used += '.' + frag
157 try:
158 found = getattr(found, frag)
159 except AttributeError:
160 self.importer(used)
161 found = getattr(found, frag)
162 return found
163
164 def ext_convert(self, value):
165 """Default converter for the ext:// protocol."""
166 return self.resolve(value)
167
168 def cfg_convert(self, value):
169 """Default converter for the cfg:// protocol."""
170 rest = value
171 m = self.WORD_PATTERN.match(rest)
172 if m is None:
173 raise ValueError("Unable to convert %r" % value)
174 else:
175 rest = rest[m.end():]
176 d = self.config[m.groups()[0]]
177 while rest:
178 m = self.DOT_PATTERN.match(rest)
179 if m:
180 d = d[m.groups()[0]]
181 else:
182 m = self.INDEX_PATTERN.match(rest)
183 if m:
184 idx = m.groups()[0]
185 if not self.DIGIT_PATTERN.match(idx):
186 d = d[idx]
187 else:
188 try:
45c51a7 @robhudson PEP8 pass
robhudson authored
189 # try as number first (most likely)
190 n = int(idx)
d5e07c2 mmmm goodies
Jeff Balogh authored
191 d = d[n]
192 except TypeError:
193 d = d[idx]
194 if m:
195 rest = rest[m.end():]
196 else:
197 raise ValueError('Unable to convert '
198 '%r at %r' % (value, rest))
45c51a7 @robhudson PEP8 pass
robhudson authored
199 # rest should be empty
d5e07c2 mmmm goodies
Jeff Balogh authored
200 return d
201
202 def convert(self, value):
203 """
204 Convert values to an appropriate type. dicts, lists and tuples are
205 replaced by their converting alternatives. Strings are checked to
206 see if they have a conversion format and are converted if they do.
207 """
208 if not isinstance(value, ConvertingDict) and isinstance(value, dict):
209 value = ConvertingDict(value)
210 value.configurator = self
211 elif not isinstance(value, ConvertingList) and isinstance(value, list):
212 value = ConvertingList(value)
213 value.configurator = self
45c51a7 @robhudson PEP8 pass
robhudson authored
214 elif not isinstance(value, ConvertingTuple) and isinstance(value, tuple):
d5e07c2 mmmm goodies
Jeff Balogh authored
215 value = ConvertingTuple(value)
216 value.configurator = self
45c51a7 @robhudson PEP8 pass
robhudson authored
217 elif isinstance(value, basestring): # str for py3k
d5e07c2 mmmm goodies
Jeff Balogh authored
218 m = self.CONVERT_PATTERN.match(value)
219 if m:
220 d = m.groupdict()
221 prefix = d['prefix']
222 converter = self.value_converters.get(prefix, None)
223 if converter:
224 suffix = d['suffix']
225 converter = getattr(self, converter)
226 value = converter(suffix)
227 return value
228
229 def configure_custom(self, config):
230 """Configure an object with a user-supplied factory."""
231 c = config.pop('()')
45c51a7 @robhudson PEP8 pass
robhudson authored
232 if (not hasattr(c, '__call__') and hasattr(types, 'ClassType') and
233 not isinstance(c, types.ClassType)):
d5e07c2 mmmm goodies
Jeff Balogh authored
234 c = self.resolve(c)
235 props = config.pop('.', None)
236 # Check for valid identifiers
237 kwargs = dict([(k, config[k]) for k in config if valid_ident(k)])
238 result = c(**kwargs)
239 if props:
240 for name, value in props.items():
241 setattr(result, name, value)
242 return result
243
244 def as_tuple(self, value):
245 """Utility function which converts lists to tuples."""
246 if isinstance(value, list):
247 value = tuple(value)
248 return value
249
45c51a7 @robhudson PEP8 pass
robhudson authored
250
d5e07c2 mmmm goodies
Jeff Balogh authored
251 class DictConfigurator(BaseConfigurator):
252 """
253 Configure logging using a dictionary-like object to describe the
254 configuration.
255 """
256
257 def configure(self):
258 """Do the configuration."""
259
260 config = self.config
261 if 'version' not in config:
262 raise ValueError("dictionary doesn't specify a version")
263 if config['version'] != 1:
264 raise ValueError("Unsupported version: %s" % config['version'])
265 incremental = config.pop('incremental', False)
266 EMPTY_DICT = {}
267 logging._acquireLock()
268 try:
269 if incremental:
270 handlers = config.get('handlers', EMPTY_DICT)
271 # incremental handler config only if handler name
272 # ties in to logging._handlers (Python 2.7)
273 if sys.version_info[:2] == (2, 7):
274 for name in handlers:
275 if name not in logging._handlers:
276 raise ValueError('No handler found with '
45c51a7 @robhudson PEP8 pass
robhudson authored
277 'name %r' % name)
d5e07c2 mmmm goodies
Jeff Balogh authored
278 else:
279 try:
280 handler = logging._handlers[name]
281 handler_config = handlers[name]
282 level = handler_config.get('level', None)
283 if level:
284 handler.setLevel(_checkLevel(level))
285 except StandardError, e:
286 raise ValueError('Unable to configure handler '
287 '%r: %s' % (name, e))
288 loggers = config.get('loggers', EMPTY_DICT)
289 for name in loggers:
290 try:
291 self.configure_logger(name, loggers[name], True)
292 except StandardError, e:
293 raise ValueError('Unable to configure logger '
294 '%r: %s' % (name, e))
295 root = config.get('root', None)
296 if root:
297 try:
298 self.configure_root(root, True)
299 except StandardError, e:
300 raise ValueError('Unable to configure root '
301 'logger: %s' % e)
302 else:
303 disable_existing = config.pop('disable_existing_loggers', True)
304
305 logging._handlers.clear()
306 del logging._handlerList[:]
307
308 # Do formatters first - they don't refer to anything else
309 formatters = config.get('formatters', EMPTY_DICT)
310 for name in formatters:
311 try:
312 formatters[name] = self.configure_formatter(
45c51a7 @robhudson PEP8 pass
robhudson authored
313 formatters[name])
d5e07c2 mmmm goodies
Jeff Balogh authored
314 except StandardError, e:
315 raise ValueError('Unable to configure '
316 'formatter %r: %s' % (name, e))
317 # Next, do filters - they don't refer to anything else, either
318 filters = config.get('filters', EMPTY_DICT)
319 for name in filters:
320 try:
321 filters[name] = self.configure_filter(filters[name])
322 except StandardError, e:
323 raise ValueError('Unable to configure '
324 'filter %r: %s' % (name, e))
325
326 # Next, do handlers - they refer to formatters and filters
327 # As handlers can refer to other handlers, sort the keys
328 # to allow a deterministic order of configuration
329 handlers = config.get('handlers', EMPTY_DICT)
330 for name in sorted(handlers):
331 try:
332 handler = self.configure_handler(handlers[name])
333 handler.name = name
334 handlers[name] = handler
335 except StandardError, e:
336 raise ValueError('Unable to configure handler '
337 '%r: %s' % (name, e))
338 # Next, do loggers - they refer to handlers and filters
339
45c51a7 @robhudson PEP8 pass
robhudson authored
340 # we don't want to lose the existing loggers,
341 # since other threads may have pointers to them.
342 # existing is set to contain all existing loggers,
343 # and as we go through the new configuration we
344 # remove any which are configured. At the end,
345 # what's left in existing is the set of loggers
346 # which were in the previous configuration but
347 # which are not in the new configuration.
d5e07c2 mmmm goodies
Jeff Balogh authored
348 root = logging.root
349 existing = root.manager.loggerDict.keys()
45c51a7 @robhudson PEP8 pass
robhudson authored
350 # The list needs to be sorted so that we can
351 # avoid disabling child loggers of explicitly
352 # named loggers. With a sorted list it is easier
353 # to find the child loggers.
d5e07c2 mmmm goodies
Jeff Balogh authored
354 existing.sort()
45c51a7 @robhudson PEP8 pass
robhudson authored
355 # We'll keep the list of existing loggers
356 # which are children of named loggers here...
d5e07c2 mmmm goodies
Jeff Balogh authored
357 child_loggers = []
45c51a7 @robhudson PEP8 pass
robhudson authored
358 # now set up the new ones...
d5e07c2 mmmm goodies
Jeff Balogh authored
359 loggers = config.get('loggers', EMPTY_DICT)
360 for name in loggers:
361 if name in existing:
362 i = existing.index(name)
363 prefixed = name + "."
364 pflen = len(prefixed)
365 num_existing = len(existing)
45c51a7 @robhudson PEP8 pass
robhudson authored
366 i = i + 1 # look at the entry after name
d5e07c2 mmmm goodies
Jeff Balogh authored
367 while (i < num_existing) and\
368 (existing[i][:pflen] == prefixed):
369 child_loggers.append(existing[i])
370 i = i + 1
371 existing.remove(name)
372 try:
373 self.configure_logger(name, loggers[name])
374 except StandardError, e:
375 raise ValueError('Unable to configure logger '
376 '%r: %s' % (name, e))
377
45c51a7 @robhudson PEP8 pass
robhudson authored
378 # Disable any old loggers. There's no point deleting
379 # them as other threads may continue to hold references
380 # and by disabling them, you stop them doing any logging.
381 # However, don't disable children of named loggers, as that's
382 # probably not what was intended by the user.
d5e07c2 mmmm goodies
Jeff Balogh authored
383 for log in existing:
384 logger = root.manager.loggerDict[log]
385 if log in child_loggers:
386 logger.level = logging.NOTSET
387 logger.handlers = []
388 logger.propagate = True
389 elif disable_existing:
390 logger.disabled = True
391
392 # And finally, do the root logger
393 root = config.get('root', None)
394 if root:
395 try:
396 self.configure_root(root)
397 except StandardError, e:
398 raise ValueError('Unable to configure root '
399 'logger: %s' % e)
400 finally:
401 logging._releaseLock()
402
403 def configure_formatter(self, config):
404 """Configure a formatter from a dictionary."""
405 if '()' in config:
45c51a7 @robhudson PEP8 pass
robhudson authored
406 factory = config['()'] # for use in exception handler
d5e07c2 mmmm goodies
Jeff Balogh authored
407 try:
408 result = self.configure_custom(config)
409 except TypeError, te:
410 if "'format'" not in str(te):
411 raise
45c51a7 @robhudson PEP8 pass
robhudson authored
412 # Name of parameter changed from fmt to format.
413 # Retry with old name.
414 # This is so that code can be used with older Python versions
415 # (e.g. by Django)
d5e07c2 mmmm goodies
Jeff Balogh authored
416 config['fmt'] = config.pop('format')
417 config['()'] = factory
418 result = self.configure_custom(config)
419 else:
420 fmt = config.get('format', None)
421 dfmt = config.get('datefmt', None)
422 result = logging.Formatter(fmt, dfmt)
423 return result
424
425 def configure_filter(self, config):
426 """Configure a filter from a dictionary."""
427 if '()' in config:
428 result = self.configure_custom(config)
429 else:
430 name = config.get('name', '')
431 result = logging.Filter(name)
432 return result
433
434 def add_filters(self, filterer, filters):
435 """Add filters to a filterer from a list of names."""
436 for f in filters:
437 try:
438 filterer.addFilter(self.config['filters'][f])
439 except StandardError, e:
440 raise ValueError('Unable to add filter %r: %s' % (f, e))
441
442 def configure_handler(self, config):
443 """Configure a handler from a dictionary."""
444 formatter = config.pop('formatter', None)
445 if formatter:
446 try:
447 formatter = self.config['formatters'][formatter]
448 except StandardError, e:
449 raise ValueError('Unable to set formatter '
450 '%r: %s' % (formatter, e))
451 level = config.pop('level', None)
452 filters = config.pop('filters', None)
453 if '()' in config:
454 c = config.pop('()')
45c51a7 @robhudson PEP8 pass
robhudson authored
455 if (not hasattr(c, '__call__') and hasattr(types, 'ClassType') and
456 not isinstance(c, types.ClassType)):
d5e07c2 mmmm goodies
Jeff Balogh authored
457 c = self.resolve(c)
458 factory = c
459 else:
460 klass = self.resolve(config.pop('class'))
45c51a7 @robhudson PEP8 pass
robhudson authored
461 # Special case for handler which refers to another handler
d5e07c2 mmmm goodies
Jeff Balogh authored
462 if issubclass(klass, logging.handlers.MemoryHandler) and\
463 'target' in config:
464 try:
465 config['target'] = self.config['handlers'][config['target']]
466 except StandardError, e:
467 raise ValueError('Unable to set target handler '
468 '%r: %s' % (config['target'], e))
469 elif issubclass(klass, logging.handlers.SMTPHandler) and\
470 'mailhost' in config:
471 config['mailhost'] = self.as_tuple(config['mailhost'])
472 elif issubclass(klass, logging.handlers.SysLogHandler) and\
473 'address' in config:
474 config['address'] = self.as_tuple(config['address'])
475 factory = klass
476 kwargs = dict([(k, config[k]) for k in config if valid_ident(k)])
477 try:
478 result = factory(**kwargs)
479 except TypeError, te:
480 if "'stream'" not in str(te):
481 raise
45c51a7 @robhudson PEP8 pass
robhudson authored
482 # The argument name changed from strm to stream
483 # Retry with old name.
484 # This is so that code can be used with older Python versions
485 # (e.g. by Django)
d5e07c2 mmmm goodies
Jeff Balogh authored
486 kwargs['strm'] = kwargs.pop('stream')
487 result = factory(**kwargs)
488 if formatter:
489 result.setFormatter(formatter)
490 if level is not None:
491 result.setLevel(_checkLevel(level))
492 if filters:
493 self.add_filters(result, filters)
494 return result
495
496 def add_handlers(self, logger, handlers):
497 """Add handlers to a logger from a list of names."""
498 for h in handlers:
499 try:
500 logger.addHandler(self.config['handlers'][h])
501 except StandardError, e:
502 raise ValueError('Unable to add handler %r: %s' % (h, e))
503
504 def common_logger_config(self, logger, config, incremental=False):
505 """
506 Perform configuration which is common to root and non-root loggers.
507 """
508 level = config.get('level', None)
509 if level is not None:
510 logger.setLevel(_checkLevel(level))
511 if not incremental:
45c51a7 @robhudson PEP8 pass
robhudson authored
512 # Remove any existing handlers
d5e07c2 mmmm goodies
Jeff Balogh authored
513 for h in logger.handlers[:]:
514 logger.removeHandler(h)
515 handlers = config.get('handlers', None)
516 if handlers:
517 self.add_handlers(logger, handlers)
518 filters = config.get('filters', None)
519 if filters:
520 self.add_filters(logger, filters)
521
522 def configure_logger(self, name, config, incremental=False):
523 """Configure a non-root logger from a dictionary."""
524 logger = logging.getLogger(name)
525 self.common_logger_config(logger, config, incremental)
526 propagate = config.get('propagate', None)
527 if propagate is not None:
528 logger.propagate = propagate
529
530 def configure_root(self, config, incremental=False):
531 """Configure a root logger from a dictionary."""
532 root = logging.getLogger()
533 self.common_logger_config(root, config, incremental)
534
45c51a7 @robhudson PEP8 pass
robhudson authored
535
d5e07c2 mmmm goodies
Jeff Balogh authored
536 dictConfigClass = DictConfigurator
537
45c51a7 @robhudson PEP8 pass
robhudson authored
538
d5e07c2 mmmm goodies
Jeff Balogh authored
539 def dictConfig(config):
540 """Configure logging using a dictionary."""
541 dictConfigClass(config).configure()
Something went wrong with that request. Please try again.