Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 717 lines (502 sloc) 19.452 kb
416babb Initial work in progress commit
Gavin M. Roy authored
1 """Command-line application wrapper with configuration and daemonization
2 support.
3
4 """
5 __author__ = 'Gavin M. Roy'
6 __email__ = 'gmr@meetme.com'
7 __since__ = '2012-04-11'
9ac0f3d Add the ability to add new configuration keys to validate the presenc…
Gavin M. Roy authored
8 __version__ = '1.2.0'
416babb Initial work in progress commit
Gavin M. Roy authored
9
10 import daemon
11 import grp
12 import logging
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
13 try:
14 from logging.config import dictConfig
15 except ImportError:
16 from logutils.dictconfig import dictConfig
416babb Initial work in progress commit
Gavin M. Roy authored
17 import optparse
18 import os
19 from daemon import pidfile
20 import pwd
21 import signal
1263928 Ready for 1.0 release
Gavin M. Roy authored
22 import sys
416babb Initial work in progress commit
Gavin M. Roy authored
23 import time
24 import yaml
25
26 _APPNAME = 'clihelper'
27 _APPLICATION = 'Application'
28 _DAEMON = 'Daemon'
29 _LOGGING = 'Logging'
30 _CONFIG_KEYS = [_APPLICATION, _DAEMON, _LOGGING]
31 _CONFIG_FILE = None
32 _CONTROLLER = None
33 _DESCRIPTION = 'Command Line Daemon'
34 _PIDFILE = '/var/run/%s.pid'
35
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
36 logger = logging.getLogger(__name__)
37
416babb Initial work in progress commit
Gavin M. Roy authored
38
39 class Controller(object):
40 """Extend the Controller class with your own application implementing the
41 Controller._process method. If you do not want to use sleep based looping
42 but rather an IOLoop or some other long-lived blocking loop, redefine
43 the Controller._loop method.
44
45 """
46 _SLEEP_UNIT = 1
47 _WAKE_INTERVAL = 60 # How many seconds to sleep before waking up
48
49 _STATE_IDLE = 0
50 _STATE_RUNNING = 1
51 _STATE_SLEEPING = 2
52 _STATE_SHUTTING_DOWN = 3
53
54 def __init__(self, options, arguments):
55 """Create an instance of the controller passing in the debug flag,
56 the options and arguments from the cli parser.
57
58 :param optparse.Values options: OptionParser option values
59 :param list arguments: Left over positional cli arguments
60
61 """
62 # Default state
63 self._set_state(self._STATE_IDLE)
64
65 # Carry these for possible later use
66 self._options = options
67 self._arguments = arguments
68
69 # Carry debug around for when/if HUP is called or the value is needed
70 self._debug = options.foreground
71
72 # Create a new instance of a configuration object
9ac0f3d Add the ability to add new configuration keys to validate the presenc…
Gavin M. Roy authored
73 self._config = get_configuration()
416babb Initial work in progress commit
Gavin M. Roy authored
74
75 def _get_application_config(self):
76 """Get the configuration data the application itself
77
78 :rtype: dict
79
80 """
9ac0f3d Add the ability to add new configuration keys to validate the presenc…
Gavin M. Roy authored
81 return get_configuration().get(_APPLICATION)
416babb Initial work in progress commit
Gavin M. Roy authored
82
83 def _get_config(self, key):
84 """Get the configuration data for the specified key
85
86 :param str key: The key to get config data for
87 :rtype: any
88
89 """
90 return self._get_application_config().get(key)
91
92 def _get_wake_interval(self):
93 """Return the wake interval in seconds.
94
95 :rtype: int
96
97 """
98 return self._get_application_config().get('wake_interval',
99 self._WAKE_INTERVAL)
100
d3fcbdc More complete test coverage
Gavin M. Roy authored
101 def _loop(self): #pragma: no cover
416babb Initial work in progress commit
Gavin M. Roy authored
102 """The process loop, loop until we are running no more."""
103 # Loop while we are running
104 while self.is_running:
105
106 # Process actions for the application
107 self._process()
108
109 # Sleep
110 try:
111 self._sleep()
112 except KeyboardInterrupt:
113 self._running = False
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
114 logger.info('CTRL-C received, shutting down')
416babb Initial work in progress commit
Gavin M. Roy authored
115 break
116 except SystemExit:
117 self._running = False
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
118 logger.info('Exit signal received, shutting down')
416babb Initial work in progress commit
Gavin M. Roy authored
119 break
120
121 def _on_sighup(self, _frame):
122 """Called when SIGHUP is received, shutdown internal runtime state,
123 reloads configuration and then calls Controller.run().
124
125 :param frame _frame: The stack frame when called
126
127 """
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
128 logger.info('Received SIGHUP, restarting internal state')
416babb Initial work in progress commit
Gavin M. Roy authored
129 self._shutdown()
130 self._shutdown_complete()
131 self._reload_configuration()
132 self.run()
133
134 def _on_sigterm(self, frame):
135 """Called when SIGTERM is received, override to implement."""
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
136 logger.info('Received SIGTERM at frame %r', frame)
416babb Initial work in progress commit
Gavin M. Roy authored
137 self._shutdown()
138
139 def _on_sigusr1(self, _frame):
140 """Called when SIGUSR1 is received. Reloads configuration and reruns
141 the logger/logging setup.
142
143 :param frame _frame: The stack frame when called
144
145 """
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
146 logger.info('Received SIGUSR1, reloading configuration')
416babb Initial work in progress commit
Gavin M. Roy authored
147 self._reload_configuration()
148
d3fcbdc More complete test coverage
Gavin M. Roy authored
149 def _on_sigusr2(self, frame): #pragma: no cover
416babb Initial work in progress commit
Gavin M. Roy authored
150 """Called when SIGUSR2 is received, override to implement."""
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
151 logger.info('Received SIGUSR2 at frame %r', frame)
416babb Initial work in progress commit
Gavin M. Roy authored
152
153 def _process(self):
154 """To be implemented by the extending class. Is called after every sleep
155 interval in the main application loop.
156
157 """
158 raise NotImplementedError
159
160 def _reload_configuration(self):
161 """Reload the configuration by creating a new instance of the
162 Configuration object and re-setup logging. Extend behavior by
163 overriding object while calling super.
164
165 """
166 # Delete the config object, creating a new one
167 del self._config
9ac0f3d Add the ability to add new configuration keys to validate the presenc…
Gavin M. Roy authored
168 self._config = get_configuration()
416babb Initial work in progress commit
Gavin M. Roy authored
169
170 # Re-Setup logging
171 _setup_logging(self._debug)
172
173 def _set_state(self, state):
174 """Set the runtime state of the Controller.
175
176 :param int state: The runtime state
177 :raises: ValueError
178
179 """
180 if state not in [self._STATE_IDLE,
181 self._STATE_RUNNING,
d3fcbdc More complete test coverage
Gavin M. Roy authored
182 self._STATE_SLEEPING,
416babb Initial work in progress commit
Gavin M. Roy authored
183 self._STATE_SHUTTING_DOWN]:
184 raise ValueError('Invalid Runtime State')
185
186 # Set the value
187 self._state = state
188
189 # Log the change
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
190 logger.debug('Runtime state changed to %i', self._state)
416babb Initial work in progress commit
Gavin M. Roy authored
191
d3fcbdc More complete test coverage
Gavin M. Roy authored
192 def _setup(self): #pragma: no cover
416babb Initial work in progress commit
Gavin M. Roy authored
193 """Override to provide any required setup steps."""
194 pass
195
196 def _shutdown(self):
197 """Override to implement shutdown steps."""
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
198 logger.debug('Shutting down')
416babb Initial work in progress commit
Gavin M. Roy authored
199 self._set_state(self._STATE_SHUTTING_DOWN)
200
201 def _shutdown_complete(self):
202 """Sets the state back to idle when shutdown steps are complete."""
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
203 logger.debug('Shutdown complete')
416babb Initial work in progress commit
Gavin M. Roy authored
204 self._set_state(self._STATE_IDLE)
205
206 def _sleep(self):
207 """Sleep for the configured sleep interval or until the app has been
208 told to shutdown.
209
210 """
211 # Calculate when the application should wake
d3fcbdc More complete test coverage
Gavin M. Roy authored
212 wake_time = self._wake_time()
416babb Initial work in progress commit
Gavin M. Roy authored
213
214 # Set the state to sleeping
215 self._set_state(self._STATE_SLEEPING)
216
217 # While we've not exceeded the end_time and we're still running
d3fcbdc More complete test coverage
Gavin M. Roy authored
218 while wake_time > time.time() and self.is_running:
416babb Initial work in progress commit
Gavin M. Roy authored
219 time.sleep(self._SLEEP_UNIT)
220
221 # Set the state back to running
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
222 logger.debug('Waking')
416babb Initial work in progress commit
Gavin M. Roy authored
223 self._set_state(self._STATE_RUNNING)
224
d3fcbdc More complete test coverage
Gavin M. Roy authored
225 def _wake_time(self):
226 """Calculate the wakeup time for sleeping
227
228 :rtype: int
229
230 """
231 wake_interval = self._get_wake_interval()
232 end_time = int(time.time() + wake_interval)
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
233 logger.debug('Sleeping %i seconds, waking at %.2f',
d3fcbdc More complete test coverage
Gavin M. Roy authored
234 wake_interval, end_time)
235 return end_time
236
416babb Initial work in progress commit
Gavin M. Roy authored
237 @property
238 def is_running(self):
239 """Returns True if the controller is running
240
241 :rtype: bool
242
243 """
244 return self._state in [self._STATE_RUNNING, self._STATE_SLEEPING]
245
246 @property
247 def is_shutting_down(self):
248 """Returns True if the controller is shutting down
249
250 :rtype: bool
251
252 """
253 return self._state == self._STATE_SHUTTING_DOWN
254
255 def run(self):
256 """The core method for starting the application. Will setup logging,
257 toggle the runtime state flag, block on loop, then call shutdown.
258
259 """
260 # Call this now because the app may be in a new process
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
261 logger.info('Process running')
416babb Initial work in progress commit
Gavin M. Roy authored
262
263 # Call the _setup method
264 self._setup()
265
266 # Toggle that we are running
267 self._set_state(self._STATE_RUNNING)
268
269 # Loop until we're not
270 self._loop()
271
272 # Wait until shutdown is complete
273 while self.is_shutting_down:
274 self._sleep()
275
d3fcbdc More complete test coverage
Gavin M. Roy authored
276 # Signal that shutdown is complete
277 self._shutdown_complete()
278
416babb Initial work in progress commit
Gavin M. Roy authored
279
280 def _cli_options(option_callback):
281 """Setup the option parser and return the options and arguments.
282
283 :param method option_callback: If passed, is called after the foreground
284 option is added to the option parser
285 parameters. The parser will be passed
286 as an argument to the callback.
287 :rtype tuple: optparse.Values, list
288
289 """
290 parser = _new_option_parser()
291
292 # Set default attributes
293 parser.usage = "usage: %prog -c <configfile> [options]"
294 parser.version = "%%prog v%s" % _VERSION
295 parser.description = _DESCRIPTION
296
297 # Add default options
298 parser.add_option("-c", "--config",
299 action="store",
1263928 Ready for 1.0 release
Gavin M. Roy authored
300 dest="configuration",
416babb Initial work in progress commit
Gavin M. Roy authored
301 default=False,
302 help="Path to the configuration file.")
303
304 parser.add_option("-f", "--foreground",
305 action="store_true",
306 dest="foreground",
307 default=False,
308 help="Run interactively in console")
309
310 # If the option callback is specified, call it with the parser instance
311 if option_callback:
312 option_callback(parser)
313
314 # Parse our options and arguments
315 return parser.parse_args()
316
317
318
319 def _get_daemon_config():
320 """Return the daemon specific configuration values
321
322 :rtype: dict
323
324 """
9ac0f3d Add the ability to add new configuration keys to validate the presenc…
Gavin M. Roy authored
325 return get_configuration().get(_DAEMON)
416babb Initial work in progress commit
Gavin M. Roy authored
326
327
328 def _get_daemon_context():
329 """Return an instance of the daemon.DaemonContext class.
330
331 :rtype: daemon.DaemonContext
332
333 """
334 # Return the daemon configuration values
335 config = _get_daemon_config()
336
337 # Create the new context to daemonize with
338 context = _new_daemon_context()
339
340 # If user is specified in the config, set it for the context
341 if config.get('user'):
342 context.uid = _get_uid(config['user'])
343
344 # If group is specified in the config, set it for the context
345 if config.get('group'):
346 context.gid = _get_gid(config['group'])
347
348 # Set the pidfile to write when app has started
349 context.pidfile = pidfile.PIDLockFile(config.get('pidfile', _PIDFILE))
350
351 # Setup the signal map
352 context.signal_map = {signal.SIGHUP: _on_sighup,
353 signal.SIGTERM: _on_sigterm,
354 signal.SIGUSR1: _on_sigusr1,
355 signal.SIGUSR2: _on_sigusr2}
356
357 # This will be used by the caller to daemonize the application
358 return context
359
360
361 def _get_gid(group):
362 """Return the group id for the specified group.
363
364 :param str group: The group name to get the id for
365 :rtype: int
366
367 """
368 return grp.getgrnam(group).gr_gid
369
370
371 def _get_logging_config():
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
372 """Return the configuration data for dictConfig
416babb Initial work in progress commit
Gavin M. Roy authored
373
374 :rtype: dict
375
376 """
9ac0f3d Add the ability to add new configuration keys to validate the presenc…
Gavin M. Roy authored
377 return get_configuration().get(_LOGGING)
416babb Initial work in progress commit
Gavin M. Roy authored
378
379
380 def _get_pidfile_path():
381 """Return the pidfile path for the daemon context.
382
383 :rtype: str
384
385 """
386 config = _get_daemon_config()
387 return config.get('pidfile', _PIDFILE % _APPNAME)
388
389
390 def _get_uid(username):
391 """Return the user id for the specified username
392
393 :param str username: The user to get the UID for
394 :rtype: int
395
396 """
397 return pwd.getpwnam(username).pw_uid
398
399
400 def _load_config():
401 """Load the configuration from disk returning a dictionary object
402 representing the configuration values.
403
404 :rtype: dict
405 :raises: OSError
406
407 """
408 # Validate the config file exists
409 _validate_config_file()
410
411 # Read the config file off the filesystem
412 content = _read_config_file()
413
414 # Return the parsed content
415 return _parse_yaml(content)
416
c252278 Slight cleanup/bugfix
Gavin M. Roy authored
417
416babb Initial work in progress commit
Gavin M. Roy authored
418 def _new_daemon_context():
419 """Return a new daemon context.
420
421 :rtype: daemon.DaemonContext
422
423 """
424 return daemon.DaemonContext()
425
426
427 def _new_option_parser():
428 """Return a new optparse.OptionParser instance.
429
430 :rtype: optparse.OptionParser
431
432 """
433 return optparse.OptionParser()
434
435
436 def _on_sighup(_signal, frame):
437 """Received when SIGHUP is received.
438
439 :param int _signal: The signal number
440 :param frame frame: The stack frame when received
441
442 """
443 _CONTROLLER._on_sighup(frame)
444
445
446 def _on_sigterm(_signal, frame):
447 """Received when SIGTERM is received.
448
449 :param int _signal: The signal number
450 :param frame frame: The stack frame when received
451
452 """
453 _CONTROLLER._on_sigterm(frame)
454
455
456 def _on_sigusr1(_signal, frame):
457 """Received when SIGUSR1 is received.
458
459 :param int _signal: The signal number
460 :param frame frame: The stack frame when received
461
462 """
463 _CONTROLLER._on_sigusr1(frame)
464
465
466 def _on_sigusr2(_signal, frame):
467 """Received when SIGUSR1 is received.
468
469 :param int _signal: The signal number
470 :param frame frame: The stack frame when received
471
472 """
473 _CONTROLLER._on_sigusr2(frame)
474
475
476 def _parse_yaml(content):
477 """Parses a YAML string and returns a dictionary object.
478
479 :param str content: The YAML content
480 :rtype: dict
481
482 """
483 return yaml.load(content)
484
c252278 Slight cleanup/bugfix
Gavin M. Roy authored
485
416babb Initial work in progress commit
Gavin M. Roy authored
486 def _read_config_file():
487 """Return the contents of the file specified in _CONFIG_FILE.
488
489 :rtype: str
490
491 """
492 with open(_CONFIG_FILE, 'r') as handle:
493 return handle.read()
494
c252278 Slight cleanup/bugfix
Gavin M. Roy authored
495
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
496 def _remove_debug_only_from_handlers(logging_config):
497 """Iterate through each handler removing the invalid dictConfig key of
498 debug_only.
499
500 :param dict logging_config: The logging configuration for dictConfig
501
502 """
503 for handler in logging_config['handlers']:
504 if 'debug_only' in logging_config['handlers'][handler]:
505 del logging_config['handlers'][handler]['debug_only']
506
507
508 def _remove_debug_only_handlers(logging_config):
509 """Remove any handlers with an attribute of debug_only that is True and
510 remove the references to said handlers from any loggers that are referencing
511 them.
512
513 :param dict logging_config: The logging configuration for dictConfig
514
515 """
516 remove = list()
517 for handler in logging_config['handlers']:
518 if logging_config['handlers'][handler].get('debug_only'):
519 remove.append(handler)
520
521 # Iterate through the handlers to remove and remove them
522 for handler in remove:
523 del logging_config['handlers'][handler]
524 _remove_handler_from_loggers(logging_config['loggers'], handler)
525
526
527 def _remove_handler_from_loggers(loggers, handler):
528 """Remove any reference of the specified handler from the loggers in the
529 logging_config dictionary.
530
531 :param dict loggers: The loggers section of the logging configuration
532 :param str handler: The name of the handler to remove references to
533
534 """
535 for logger in loggers:
536 try:
537 loggers[logger]['handlers'].remove(handler)
538 except ValueError:
539 pass
540
541
416babb Initial work in progress commit
Gavin M. Roy authored
542 def _setup_logging(debug):
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
543 """Setup the logging configuration and assign the logger. If debug is False
544 strip any handlers and their references from the configuration.
416babb Initial work in progress commit
Gavin M. Roy authored
545
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
546 :param bool debug: The app is in debug mode
416babb Initial work in progress commit
Gavin M. Roy authored
547
548 """
95a458d Update to use logutils instead of logging-config.
Gavin M. Roy authored
549 # Get the configuration
550 logging_config = _get_logging_config()
551
552 # Process debug only handlers
553 if not debug:
554 _remove_debug_only_handlers(logging_config)
555
556 # Remove any references to debug_only
557 _remove_debug_only_from_handlers(logging_config)
558
559 # Run the Dictionary Configuration
560 dictConfig(logging_config)
416babb Initial work in progress commit
Gavin M. Roy authored
561
562
563 def _validate_config_file():
564 """Validates the configuration file is set and that it exists.
565
566 :rtype: bool
567 :raises: ValueError, OSError
568
569 """
570 if not _CONFIG_FILE:
571 raise ValueError('Missing internal reference to configuration file')
572
573 if not os.path.exists(_CONFIG_FILE):
c252278 Slight cleanup/bugfix
Gavin M. Roy authored
574 raise OSError('"%s" does not exist' % _CONFIG_FILE)
416babb Initial work in progress commit
Gavin M. Roy authored
575
576 return True
577
9ac0f3d Add the ability to add new configuration keys to validate the presenc…
Gavin M. Roy authored
578 def add_config_key(key):
579 """Add a top-level key to the expected configuration values for validation
580
581 :param str key: The key to add to the configuraiton keys
582
583 """
584 global _CONFIG_KEYS
585 _CONFIG_KEYS.append(key)
586
587 def get_configuration():
588 """Return the configuration object, validating that the required top-level
589 keys exists.
590
591 :rtype: dict
592
593 """
594 # Load the configuration file from disk
595 configuration = _load_config()
596
597 # Validate all the top-level items are there
598 for key in _CONFIG_KEYS:
599 if key not in configuration:
600 raise ValueError('Missing required configuration parameter: %s',
601 key)
602
603 # Return the configuration dictionary
604 return configuration
605
c252278 Slight cleanup/bugfix
Gavin M. Roy authored
606
416babb Initial work in progress commit
Gavin M. Roy authored
607 def run(controller, option_callback=None):
608 """Called by the implementing application to run the application.
609 ControllerClass is a class that extends cliapp.Controller.
610
611 :param Controller controller: Implementing class extending Controller
612 :param method option_callback: If passed, is called after the foreground
613 option is added to the option parser
614 parameters.
615
616 """
617 options, arguments = _cli_options(option_callback)
618
619 # Setup the config file
1263928 Ready for 1.0 release
Gavin M. Roy authored
620 try:
621 set_configuration_file(options.configuration)
622 except ValueError as error:
623 print 'Error: %s\n' % error
624 sys.exit(1)
416babb Initial work in progress commit
Gavin M. Roy authored
625
626 if options.foreground:
627 _setup_logging(True)
628 process = controller(options, arguments)
629 set_controller(process)
630 try:
631 return process.run()
632 except KeyboardInterrupt:
633 logging.info('CTRL-C caught, shutting down')
634 process._on_sigterm(None)
635 logging.info('Shutdown')
c252278 Slight cleanup/bugfix
Gavin M. Roy authored
636 return
416babb Initial work in progress commit
Gavin M. Roy authored
637
638 # Run the process with the daemon context
639 with _get_daemon_context():
640 _setup_logging(False)
641 process = controller(options, arguments)
642 set_controller(process)
643 process.run()
644
645
646 def set_appname(appname):
647 """Sets the application name for the instance of the application.
648
649 :param str appname: The application name
650
651 """
652 global _APPNAME
653 _APPNAME = appname
654
655
656 def set_configuration_file(filename):
657 """Sets the path for the configuration file.
658
659 :param str filename: The full path to the configuration file
660 :raises: ValueError
661
662 """
663 global _CONFIG_FILE
664
665 # Make sure the configuration file was specified
666 if not filename:
667 raise ValueError('Missing required configuration file value')
668
669 # Set the config file to the global variable
670 _CONFIG_FILE = filename
671
672 # Validate the file exists
673 _validate_config_file()
674
675
676 def set_controller(controller):
677 """Sets the controller for the instance of the application.
678
679 :param Controller controller: The Controller object
680
681 """
682 global _CONTROLLER
683 _CONTROLLER = controller
684
685
686 def set_description(description):
687 """Sets the description for the instance of the application.
688
689 :param str description: The app description
690
691 """
692 global _DESCRIPTION
693 _DESCRIPTION = description
694
695
696 def set_version(version):
697 """Sets the version for the instance of the application.
698
699 :param str version: The version #
700
701 """
702 global _VERSION
703 _VERSION = version
704
705
706 def setup(appname, description, version):
707 """Setup the application with one method instead of calling all four.
708
709 :param str appname: The application name
710 :param str description: The app description
711 :param str version: The version #
712
713 """
714 set_appname(appname)
715 set_description(description)
716 set_version(version)
Something went wrong with that request. Please try again.