Skip to content
This repository
Browse code

Resolves #115 logging moved to configuration file.

  • Loading branch information...
commit 183d41c1cf0cabc8344433a1c12a8d9bb6cdd453 1 parent 375a7fe
Daniel Nephin dnephin authored
1  MANIFEST.in
@@ -2,6 +2,7 @@ include *.txt
2 2 include *.md
3 3 include Makefile
4 4 include tron/default_config.yaml
  5 +include tron/logging.conf
5 6 recursive-include debian *
6 7 recursive-include test *.py *.yaml
7 8 recursive-include docs *.txt *.rst *.yaml *.1 *.8
59 bin/trond
@@ -9,9 +9,10 @@ from __future__ import with_statement
9 9
10 10 import errno
11 11 import logging
  12 +import logging.config
12 13 import optparse
13 14 import os
14   -from pkg_resources import resource_string
  15 +import pkg_resources
15 16 import signal
16 17 import sys
17 18
@@ -27,16 +28,20 @@ import tron
27 28 from tron import commands
28 29 from tron import mcp
29 30 from tron import www
30   -from tron.utils import log_handler
31 31
32   -logger = logging.getLogger('bin.trond')
  32 +logger = logging.getLogger('trond')
33 33
34 34
  35 +TRON = 'tron'
  36 +DEFAULT_CONF = 'default_config.yaml'
  37 +DEFAULT_WORKING_CONF = 'tron_config.yaml'
  38 +DEFAULT_LOGGING_CONF = 'logging.conf'
  39 +
35 40 def create_default_config(config_path):
36 41 """Create a default empty configuration for first time installs"""
37   - default_config = resource_string('tron', 'default_config.yaml')
  42 + default = pkg_resources.resource_string(TRON, DEFAULT_CONF)
38 43 with open(config_path, "w") as config_file:
39   - config_file.write(default_config)
  44 + config_file.write(default)
40 45
41 46
42 47 def parse_options():
@@ -46,9 +51,8 @@ def parse_options():
46 51 help="Directory where tron's state and output is stored"
47 52 " (default %default)",
48 53 default="/var/lib/tron/")
49   - parser.add_option("--log-file", "-l", action="store", dest="log_file",
50   - help="Where the logs are stored (default %default)",
51   - default="/var/log/tron/tron.log")
  54 + parser.add_option("--log-conf", "-l", action="store", dest="log_conf",
  55 + help="Custom logging.conf file to use")
52 56 parser.add_option("--config-file", "-c", action="store",
53 57 dest="config_file", default=None,
54 58 help="Configuration file to load (default in working"
@@ -81,16 +85,17 @@ def parse_options():
81 85 parser.error("Bad working-dir option")
82 86
83 87 if options.config_file is None:
84   - options.config_file = os.path.join(options.working_dir,
85   - "tron_config.yaml")
  88 + options.config_file = os.path.join(
  89 + options.working_dir, DEFAULT_WORKING_CONF)
86 90
87 91 return options
88 92
89 93
90 94 def setup_logging(options):
91   - level = logging.WARNING
92   - twist_level = logging.WARNING
  95 + default = pkg_resources.resource_filename(TRON, DEFAULT_LOGGING_CONF)
  96 + logfile = options.log_conf or default
93 97
  98 + level = twist_level = None
94 99 if options.verbose > 0:
95 100 level = logging.INFO
96 101 twist_level = logging.WARNING
@@ -100,32 +105,22 @@ def setup_logging(options):
100 105 if options.verbose > 2:
101 106 twist_level = logging.DEBUG
102 107
103   - root = logging.getLogger('')
104   -
105   - try:
106   - handler = log_handler.ReOpeningFileHandler(options.log_file)
107   - except IOError, e:
108   - print >>sys.stderr, e
109   - sys.exit()
  108 + tron_logger = logging.getLogger('tron')
  109 + twisted_logger = logging.getLogger('twisted')
110 110
111   - fmt_str = '%(asctime)s %(name)s %(levelname)s %(message)s'
112   - formatter = logging.Formatter(fmt_str)
113   - handler.setFormatter(formatter)
114   -
115   - root.addHandler(handler)
116   - root.setLevel(level)
117   - logging.getLogger('twisted').setLevel(twist_level)
  111 + logging.config.fileConfig(logfile)
  112 + if level is not None:
  113 + tron_logger.setLevel(level)
  114 + if twist_level is not None:
  115 + twisted_logger.setLevel(twist_level)
118 116
119 117 # Hookup twisted to standard logging
120   - observer = log.PythonLoggingObserver()
121   - observer.start()
  118 + log.PythonLoggingObserver().start()
122 119
123 120 # Show stack traces for errors in twisted deferreds.
124 121 if options.debug:
125 122 defer.setDebugging(True)
126 123
127   - return handler
128   -
129 124
130 125 # This is rather crazy thing to do, but I don't really like 'twistd', but it
131 126 # has all kinds of useful functions in it for daemonizing stuff. So rather than
@@ -199,7 +194,7 @@ class TronApplicationRunner(_twistd_unix.UnixApplicationRunner):
199 194 self.preApplication()
200 195
201 196 logger.debug("init: setup_logging")
202   - self.log_handler = setup_logging(self.options)
  197 + setup_logging(self.options)
203 198
204 199 logger.debug("init: postApplication")
205 200 self.postApplication()
@@ -240,7 +235,7 @@ class TronApplicationRunner(_twistd_unix.UnixApplicationRunner):
240 235 def sighup_handler(signum, frame):
241 236 logger.info("SIGHUP Caught!")
242 237 reactor.callLater(0, master_control.live_reconfig)
243   - reactor.callLater(0, self.log_handler.reopen)
  238 + reactor.callLater(0, setup_logging, self.options)
244 239
245 240 signal.signal(signal.SIGHUP, sighup_handler)
246 241
17 docs/config.rst
Source Rendered
@@ -205,23 +205,6 @@ Context variables only available to Services:
205 205 total number of instances).
206 206
207 207
208   -.. _config_logging:
209   -
210   -Logging
211   --------
212   -
213   -**syslog_address** (optional)
214   - Include this if you want to enable logging to syslog. Accepts paths as
215   - strings and ``[address, port]`` lists for sockets. Typical values for
216   - various platforms are::
217   -
218   - Linux: "/dev/log"
219   - OS X: "/var/run/syslog"
220   - Windows: ["localhost", 514]
221   -
222   -Example::
223   -
224   - syslog_address: "/dev/log"
225 208
226 209 Nodes
227 210 -----
4 docs/developing.rst
Source Rendered
@@ -31,10 +31,10 @@ directory structure with your test jobs and configs. To put everything in one
31 31 directory, launch `trond` like this::
32 32
33 33 > mkdir wd
34   - > bin/trond --working-dir=wd --log-file=wd/tron.log --pid-file=wd/tron.pid --verbose
  34 + > bin/trond --working-dir=wd --pid-file=wd/tron.pid --verbose
35 35
36 36 You may be tempted to run with ``--nodaemon``, but all the interesting output
37   -goes to ``tron.log``, so you're better off running ``tail -f wd/tron.log`` in a
  37 +goes to ``tron.log``, so you're better off running ``tail -f tron.log`` in a
38 38 terminal. Kill ``trond`` when you're done with ``cat wd/tron.pid | xargs
39 39 kill``.
40 40
10 docs/man_trond.rst
Source Rendered
@@ -25,8 +25,8 @@ Options
25 25 --working-dir=WORKING_DIR
26 26 Directory where tron's state and output is stored (default /var/lib/tron/)
27 27
28   --l LOG_FILE, --log-file=LOG_FILE
29   - Where the logs are stored (default /var/log/tron/tron.log)
  28 +-l LOG_CONF, --log-conf=LOG_CONF
  29 + Logging configuration file to setup python logger
30 30
31 31 -c CONFIG_FILE, --config-file=CONFIG_FILE
32 32 Configuration file to load (default in working dir)
@@ -67,6 +67,12 @@ State file
67 67 trond saves state here when it terminates and reloads it when it starts
68 68 up again.
69 69
  70 +
  71 +Logging
  72 +-------
  73 +TODO documentation
  74 +
  75 +
70 76 Bugs
71 77 ----
72 78
7 docs/man_tronfig.rst
Source Rendered
@@ -48,13 +48,6 @@ command_context
48 48 the command string. For example, if you include `animal: cat`, then the
49 49 command `cat %(animal)s` will become `cat cat`.
50 50
51   -syslog_address
52   - Include this if you want to enable logging to syslog. Accepts paths as strings
53   - and [address, port] lists for sockets. Typical values for various platforms are::
54   -
55   - Linux: "/dev/log"
56   - OS X: "/var/run/syslog"
57   - Windows: ["localhost", 514]
58 51
59 52 notification_options
60 53 Who to email failures to.
12 tests/config/config_parse_test.py
... ... @@ -1,5 +1,4 @@
1 1 import datetime
2   -import platform
3 2 import shutil
4 3 import StringIO
5 4 import tempfile
@@ -39,15 +38,6 @@
39 38 """
40 39
41 40
42   -def syslog_address_for_platform():
43   - if platform.system() == 'Darwin':
44   - return '/var/run/syslog'
45   - elif platform.system() == 'Windows':
46   - return ['localhost', 514]
47   - else:
48   - return '/dev/log'
49   -
50   -
51 41 class OldConfigTest(TestCase):
52 42 OLD_BASE_CONFIG = """
53 43 --- !TronConfiguration
@@ -157,7 +147,6 @@ def test_attributes(self):
157 147 test_config = load_config(StringIO.StringIO(self.config))
158 148 expected = TronConfig(
159 149 working_dir='/tmp',
160   - syslog_address=None,
161 150 command_context=FrozenDict(**{
162 151 'python': '/usr/bin/python',
163 152 'batch_dir': '/tron/batch/test/foo'
@@ -311,7 +300,6 @@ def test_attributes(self):
311 300 # we could just do a big assert_equal here, but it would be hella hard
312 301 # to debug failures that way.
313 302 assert_equal(test_config.working_dir, expected.working_dir)
314   - assert_equal(test_config.syslog_address, expected.syslog_address)
315 303 assert_equal(test_config.command_context, expected.command_context)
316 304 assert_equal(test_config.ssh_options, expected.ssh_options)
317 305 assert_equal(test_config.notification_options, expected.notification_options)
2  tests/config/reconfigure_test.py
@@ -7,7 +7,6 @@
7 7 from testify import TestCase, run, setup, assert_equal, teardown
8 8 from tron import mcp
9 9 from tron.config import config_parse
10   -from tests.config.config_parse_test import syslog_address_for_platform
11 10
12 11 class ConfigTest(TestCase):
13 12
@@ -67,7 +66,6 @@ def config_2(self, wd):
67 66 agent=True,
68 67 identities=['tests/test_id_rsa'],
69 68 ),
70   - syslog_address=syslog_address_for_platform(),
71 69 nodes=[
72 70 dict(name='node0', hostname='batch0'),
73 71 dict(name='node1', hostname='batch1'),
33 tests/logging.conf
... ... @@ -0,0 +1,33 @@
  1 +[loggers]
  2 +keys=root, twisted, tron
  3 +
  4 +[handlers]
  5 +keys=fileHandler
  6 +
  7 +[formatters]
  8 +keys=defaultFormatter
  9 +
  10 +[logger_root]
  11 +level=DEBUG
  12 +handlers=fileHandler
  13 +
  14 +[logger_twisted]
  15 +level=WARNING
  16 +handlers=fileHandler
  17 +qualname=twisted
  18 +propagate=0
  19 +
  20 +[logger_tron]
  21 +level=INFO
  22 +handlers=fileHandler
  23 +qualname=tron
  24 +propagate=0
  25 +
  26 +[handler_fileHandler]
  27 +class=logging.FileHandler
  28 +level=DEBUG
  29 +formatter=defaultFormatter
  30 +args=('tron.log',)
  31 +
  32 +[formatter_defaultFormatter]
  33 +format=%(asctime)s %(name)s %(levelname)s %(message)s
5 tests/sandbox.py
@@ -84,8 +84,9 @@ def __init__(self):
84 84 self.trond_bin = os.path.join(self.tron_bin, 'trond')
85 85 self.tronfig_bin = os.path.join(self.tron_bin, 'tronfig')
86 86 self.tronview_bin = os.path.join(self.tron_bin, 'tronview')
87   -
88 87 self.log_file = os.path.join(self.tmp_dir, 'tron.log')
  88 + self.log_conf = 'tests/logging.conf'
  89 +
89 90 self.pid_file = os.path.join(self.tmp_dir, 'tron.pid')
90 91 self.config_file = os.path.join(self.tmp_dir, 'tron_config.yaml')
91 92
@@ -95,11 +96,11 @@ def __init__(self):
95 96 self.run_time = None
96 97
97 98 self.trond_debug_args = ['--working-dir=%s' % self.tmp_dir,
98   - '--log-file=%s' % self.log_file,
99 99 '--pid-file=%s' % self.pid_file,
100 100 '--port=%d' % self.port,
101 101 '--host=%s' % self.host,
102 102 '--config=%s' % self.config_file,
  103 + '--log-conf=%s' % self.log_conf,
103 104 '--verbose', '--verbose']
104 105
105 106 self.tron_server_address = '%s:%d' % (self.host, self.port)
7 tron/config/config_parse.py
@@ -82,7 +82,6 @@ def config_object_factory(name, required=None, optional=None):
82 82 'TronConfig',
83 83 optional=[
84 84 'working_dir', # str
85   - 'syslog_address', # str
86 85 'command_context', # FrozenDict of str
87 86 'ssh_options', # ConchOptions
88 87 'notification_options',# NotificationOptions or None
@@ -365,10 +364,6 @@ def valid_working_dir(wd):
365 364 return valid_str('working_dir', wd, optional=True)
366 365
367 366
368   -def valid_syslog(syslog):
369   - return valid_str('syslog', syslog, optional=True)
370   -
371   -
372 367 def valid_command_context(context):
373 368 # context can be any dict.
374 369 return FrozenDict(**valid_dict('command_context', context or {}))
@@ -632,7 +627,6 @@ class ValidateConfig(Validator):
632 627 config_class = TronConfig
633 628 defaults = {
634 629 'working_dir': None,
635   - 'syslog_address': None,
636 630 'command_context': None,
637 631 'ssh_options': valid_ssh_options({}),
638 632 'notification_options': None,
@@ -644,7 +638,6 @@ class ValidateConfig(Validator):
644 638 }
645 639 validators = {
646 640 'working_dir': valid_working_dir,
647   - 'syslog_address': valid_syslog,
648 641 'command_context': valid_command_context,
649 642 'ssh_options': valid_ssh_options,
650 643 'notification_options': valid_notification_options,
7 tron/default_config.yaml
@@ -6,13 +6,6 @@ ssh_options:
6 6 # - /home/tron/.ssh/id_dsa
7 7 agent: true
8 8
9   -## Uncomment if you want logging to syslog. Typical values for different
10   -## platforms:
11   -## Linux: "/dev/log"
12   -## OS X: "/var/run/syslog"
13   -## Windows: ["localhost", 514]
14   -# syslog_address: /dev/log
15   -
16 9 # notification_options:
17 10 ## In case of trond failures, where should we send notifications to ?
18 11 # smtp_host: localhost
42 tron/logging.conf
... ... @@ -0,0 +1,42 @@
  1 +[loggers]
  2 +keys=root, twisted, tron
  3 +
  4 +[handlers]
  5 +keys=timedRotatingFileHandler, syslogHandler
  6 +
  7 +[formatters]
  8 +keys=defaultFormatter, syslogFormatter
  9 +
  10 +[logger_root]
  11 +level=DEBUG
  12 +handlers=timedRotatingFileHandler
  13 +
  14 +[logger_twisted]
  15 +level=WARNING
  16 +handlers=timedRotatingFileHandler
  17 +qualname=twisted
  18 +propagate=0
  19 +
  20 +[logger_tron]
  21 +level=WARNING
  22 +handlers=timedRotatingFileHandler, syslogHandler
  23 +qualname=tron
  24 +propagate=0
  25 +
  26 +[handler_timedRotatingFileHandler]
  27 +class=logging.handlers.TimedRotatingFileHandler
  28 +level=DEBUG
  29 +formatter=defaultFormatter
  30 +args=('tron.log', 'D')
  31 +
  32 +[handler_syslogHandler]
  33 +class=logging.handlers.SysLogHandler
  34 +level=WARNING
  35 +formatter=syslogFormatter
  36 +args=('/dev/log',)
  37 +
  38 +[formatter_defaultFormatter]
  39 +format=%(asctime)s %(name)s %(levelname)s %(message)s
  40 +
  41 +[formatter_syslogFormatter]
  42 +format=tron[%(process)d]: %(message)s
34 tron/mcp.py
@@ -3,7 +3,6 @@
3 3 import logging.handlers
4 4 import os
5 5 import shutil
6   -import socket
7 6 import time
8 7 import yaml
9 8
@@ -236,9 +235,6 @@ def __init__(self, working_dir, config_file, context=None):
236 235 # Control writing of the state file
237 236 self.state_handler = StateHandler(self, working_dir)
238 237
239   - root = logging.getLogger('')
240   - self.base_logging_handlers = list(root.handlers)
241   -
242 238 ### CONFIGURATION ###
243 239
244 240 def live_reconfig(self):
@@ -291,7 +287,6 @@ def apply_config(self, conf, skip_env_dependent=False):
291 287 trond machine.
292 288 """
293 289 self._apply_working_directory(conf.working_dir)
294   - self._apply_loggers(conf.syslog_address)
295 290
296 291 if not skip_env_dependent:
297 292 ssh_options = self._ssh_options_from_config(conf.ssh_options)
@@ -333,35 +328,6 @@ def _apply_working_directory(self, wd):
333 328
334 329 self.state_handler.working_dir = wd
335 330
336   - def _apply_loggers(self, syslog_address):
337   - root = logging.getLogger('')
338   - handlers_to_be_removed = set(h for h in root.handlers
339   - if h not in self.base_logging_handlers)
340   -
341   - # Only change handlers if they will actually be different from the old
342   - # handlers
343   - new_handlers = []
344   - if syslog_address:
345   - if not isinstance(syslog_address, basestring):
346   - self.syslog_address = tuple(self.syslog_address)
347   -
348   - try:
349   - h = logging.handlers.SysLogHandler(syslog_address)
350   - fmt_str = "tron[%(process)d]: %(message)s"
351   - h.setFormatter(logging.Formatter(fmt_str))
352   - new_handlers.append(h)
353   - except socket.error:
354   - raise ConfigError('%s is not a valid syslog address' %
355   - syslog_address)
356   -
357   - for h in handlers_to_be_removed:
358   - log.info('Removing logging handler %s', h)
359   - root.removeHandler(h)
360   -
361   - for h in new_handlers:
362   - log.info('Adding logging handler %s', h)
363   - root.addHandler(h)
364   -
365 331 def _ssh_options_from_config(self, ssh_conf):
366 332 ssh_options = ConchOptions()
367 333 if ssh_conf.agent:
29 tron/utils/log_handler.py
... ... @@ -1,29 +0,0 @@
1   -"""The default logging module doesn't provide a way to re-open log files based
2   -on SIGHUP
3   -"""
4   -
5   -import logging
6   -
7   -
8   -class ReOpeningFileHandler(logging.FileHandler):
9   -
10   - def __init__(self, filename, mode='a', encoding=None):
11   - self.filename = filename
12   - self.mode = mode
13   - self.stream = None
14   - self.open_stream()
15   -
16   - logging.StreamHandler.__init__(self, self.stream)
17   -
18   - def open_stream(self):
19   - assert self.stream is None
20   - self.stream = open(self.filename, self.mode)
21   -
22   - def close_stream(self):
23   - self.flush()
24   - self.stream.close()
25   - self.stream = None
26   -
27   - def reopen(self):
28   - self.close_stream()
29   - self.open_stream()

0 comments on commit 183d41c

Please sign in to comment.
Something went wrong with that request. Please try again.