Skip to content

Commit

Permalink
Support expansions of ENV_* vars in all options.
Browse files Browse the repository at this point in the history
  • Loading branch information
@dexterbt1 @mnaberez
dexterbt1 authored and mnaberez committed Jun 4, 2014
1 parent 9b6fa1a commit 2d6ca34
Show file tree
Hide file tree
Showing 3 changed files with 451 additions and 40 deletions.
67 changes: 67 additions & 0 deletions docs/configuration.rst
View file
Expand Up @@ -1417,3 +1417,70 @@ And a section in the config file meant to configure it.
[rpcinterface:another]
supervisor.rpcinterface_factory = my.package:make_another_rpcinterface
retries = 1
Environment Variable Interpolation
----------------------------------

There may be a time where it is necessary to avoid hardcoded values in your
configuration file (such as paths, port numbers, username, etc). Some teams
may also put their supervisord.conf files under source control but may want
to avoid committing sensitive information into the repository.

With this, **all** the environment variables inherited by the ``supervisord``
process are available and can be interpolated / expanded in **any**
configuration value, under **any** section.

Your configuration values may contain Python expressions for expanding
the environment variables using the ``ENV_`` prefix. The sample syntax is
``foo_key=%(ENV_FOO)s``, where the value of the environment variable ``FOO``
will be assigned to the ``foo_key``. The string values of environment
variables will be converted properly to their correct types.

.. note::
- some sections such as ``[program:x]`` have other extra expansion options.
- environment variables in the configuration will be required, otherwise
supervisord will refuse to start.
- any changes to the variable requires a restart in the ``supervisord``
daemon.


An example configuration snippet with customizable values:

.. code-block:: ini
[supervisord]
logfile = %(ENV_MYSUPERVISOR_BASEDIR)s/%(ENV_MYSUPERVISOR_LOGFILE)s
logfile_maxbytes = %(ENV_MYSUPERVISOR_LOGFILE_MAXBYTES)s
logfile_backups=10
loglevel = info
pidfile = %(ENV_MYSUPERVISOR_BASEDIR)s/supervisor.pid
nodaemon = false
minfds = 1024
minprocs = 200
umask = 022
user = %(ENV_USER)s
[program:cat]
command=/bin/cat -x -y --optz=%(ENV_CAT_OPTZ)s
process_name=%(program_name)s
numprocs=%(ENV_CAT_NUMPROCS)s
directory=%(ENV_CAT_DIR)s
umask=022
priority=999
autostart=true
autorestart=true
exitcodes=0,2
user=%(ENV_USER)s
redirect_stderr=false
stopwaitsecs=10
The above sample config will require the following environment variables to be set:

- ``MYSUPERVISOR_BASEDIR``
- ``MYSUPERVISOR_LOGFILE``
- ``MYSUPERVISOR_LOGFILE_MAXBYTES``
- ``USER``
- ``CAT_OPTZ``
- ``CAT_NUMPROCS``
- ``CAT_DIRECTORY``

101 changes: 61 additions & 40 deletions supervisor/options.py
View file
Expand Up @@ -379,7 +379,14 @@ def get_plugins(self, parser, factory_key, section_prefix):
except ImportError:
raise ValueError('%s cannot be resolved within [%s]' % (
factory_spec, section))
items = parser.items(section)
items_tmp = parser.items(section)
items = []
for ikv in items_tmp:
ik, iv_tmp = ikv
iexpansions = {}
iexpansions.update(environ_expansions())
iv = expand(iv_tmp, iexpansions, ik)
items.append((ik, iv))
items.remove((factory_key, factory_spec))
factories.append((name, factory, dict(items)))

Expand Down Expand Up @@ -555,11 +562,14 @@ def read_config(self, fp):
if need_close:
fp.close()

expansions = {'here':self.here}
expansions.update(environ_expansions())
if parser.has_section('include'):
if not parser.has_option('include', 'files'):
raise ValueError(".ini file has [include] section, but no "
"files setting")
files = parser.get('include', 'files')
files = expand(files, expansions, 'include.files')
files = files.split()
if hasattr(fp, 'name'):
base = os.path.dirname(os.path.abspath(fp.name))
Expand All @@ -583,7 +593,14 @@ def read_config(self, fp):
sections = parser.sections()
if not 'supervisord' in sections:
raise ValueError('.ini file does not include supervisord section')
get = parser.getdefault

common_expansions = {'here':self.here}
def get(opt, default, **kwargs):
expansions = kwargs.get('expansions', {})
expansions.update(common_expansions)
kwargs['expansions'] = expansions
return parser.getdefault(opt, default, **kwargs)

section.minfds = integer(get('minfds', 1024))
section.minprocs = integer(get('minprocs', 200))

Expand All @@ -608,8 +625,6 @@ def read_config(self, fp):
section.nocleanup = boolean(get('nocleanup', 'false'))
section.strip_ansi = boolean(get('strip_ansi', 'false'))

expansions = {'here':self.here}
expansions.update(environ_expansions())
environ_str = get('environment', '')
environ_str = expand(environ_str, expansions, 'environment')
section.environment = dict_of_key_value_pairs(environ_str)
Expand All @@ -634,7 +649,14 @@ def process_groups_from_parser(self, parser):
groups = []
all_sections = parser.sections()
homogeneous_exclude = []
get = parser.saneget

common_expansions = {'here':self.here}
def get(section, opt, default, **kwargs):
expansions = kwargs.get('expansions', {})
expansions.update(common_expansions)
kwargs['expansions'] = expansions
return parser.saneget(section, opt, default, **kwargs)


# process heterogeneous groups
for section in all_sections:
Expand Down Expand Up @@ -717,6 +739,7 @@ def process_groups_from_parser(self, parser):
continue
program_name = process_or_group_name(section.split(':', 1)[1])
priority = integer(get(section, 'priority', 999))
fcgi_expansions = {'program_name': program_name}

# find proc_uid from "user" option
proc_user = get(section, 'user', None)
Expand All @@ -741,15 +764,11 @@ def process_groups_from_parser(self, parser):
raise ValueError('Invalid socket_mode value %s'
% socket_mode)

socket = get(section, 'socket', None)
socket = get(section, 'socket', None, expansions=fcgi_expansions)
if not socket:
raise ValueError('[%s] section requires a "socket" line' %
section)

expansions = {'here':self.here,
'program_name':program_name}
expansions.update(environ_expansions())
socket = expand(socket, expansions, 'socket')
try:
socket_config = self.parse_fcgi_socket(socket, proc_uid,
socket_owner, socket_mode)
Expand Down Expand Up @@ -803,8 +822,19 @@ def processes_from_section(self, parser, section, group_name,
if klass is None:
klass = ProcessConfig
programs = []
get = parser.saneget

program_name = process_or_group_name(section.split(':', 1)[1])
host_node_name = platform.node()
common_expansions = {'here':self.here,
'program_name':program_name,
'host_node_name':host_node_name,
'group_name':group_name}
def get(section, opt, *args, **kwargs):
expansions = kwargs.get('expansions', {})
expansions.update(common_expansions)
kwargs['expansions'] = expansions
return parser.saneget(section, opt, *args, **kwargs)

priority = integer(get(section, 'priority', 999))
autostart = boolean(get(section, 'autostart', 'true'))
autorestart = auto_restart(get(section, 'autorestart', 'unexpected'))
Expand All @@ -818,7 +848,7 @@ def processes_from_section(self, parser, section, group_name,
redirect_stderr = boolean(get(section, 'redirect_stderr','false'))
numprocs = integer(get(section, 'numprocs', 1))
numprocs_start = integer(get(section, 'numprocs_start', 0))
environment_str = get(section, 'environment', '')
environment_str = get(section, 'environment', '', do_expand=False)
stdout_cmaxbytes = byte_size(get(section,'stdout_capture_maxbytes','0'))
stdout_events = boolean(get(section, 'stdout_events_enabled','false'))
stderr_cmaxbytes = byte_size(get(section,'stderr_capture_maxbytes','0'))
Expand All @@ -844,7 +874,7 @@ def processes_from_section(self, parser, section, group_name,
'program section %s does not specify a command' % section)

process_name = process_or_group_name(
get(section, 'process_name', '%(program_name)s'))
get(section, 'process_name', '%(program_name)s', do_expand=False))

if numprocs > 1:
if not '%(process_num)' in process_name:
Expand All @@ -859,13 +889,9 @@ def processes_from_section(self, parser, section, group_name,
"Cannot set stopasgroup=true and killasgroup=false"
)

host_node_name = platform.node()
for process_num in range(numprocs_start, numprocs + numprocs_start):
expansions = {'here':self.here,
'process_num':process_num,
'program_name':program_name,
'host_node_name':host_node_name,
'group_name':group_name}
expansions = common_expansions
expansions.update({'process_num': process_num})
expansions.update(environ_expansions())

environment = dict_of_key_value_pairs(
Expand Down Expand Up @@ -1620,23 +1646,26 @@ def read_string(self, s):
except AttributeError:
return self.readfp(s)

def getdefault(self, option, default=_marker):
def saneget(self, section, option, default=_marker, do_expand=True,
expansions={}):
expansions.update(environ_expansions())
try:
return self.get(self.mysection, option)
optval = self.get(section, option)
if isinstance(optval, basestring) and do_expand:
return expand(optval,
expansions,
"%s.%s" % (section, option))
return optval
except ConfigParser.NoOptionError:
if default is _marker:
raise
else:
return default

def saneget(self, section, option, default=_marker):
try:
return self.get(section, option)
except ConfigParser.NoOptionError:
if default is _marker:
raise
else:
return default
def getdefault(self, option, default=_marker, expansions={}, **kwargs):
return self.saneget(self.mysection, option, default=default,
expansions=expansions, **kwargs)


class Config(object):
def __ne__(self, other):
Expand Down Expand Up @@ -2018,24 +2047,16 @@ def expand(s, expansions, name):
'Format string %r for %r is badly formatted' % (s, name)
)

_environ_expansions = None

def environ_expansions():
"""Return dict of environment variables, suitable for use in string
expansions.
Every environment variable is prefixed by 'ENV_'.
"""
global _environ_expansions

if _environ_expansions:
return _environ_expansions

_environ_expansions = {}
x = {}
for key, value in os.environ.items():
_environ_expansions['ENV_%s' % key] = value

return _environ_expansions
x['ENV_%s' % key] = value
return x

def make_namespec(group_name, process_name):
# we want to refer to the process by its "short name" (a process named
Expand Down

0 comments on commit 2d6ca34

Please sign in to comment.