Permalink
Cannot retrieve contributors at this time
Fetching contributors…
Cannot retrieve contributors at this time
| # vi: ts=4 expandtab | |
| # | |
| # Copyright (C) 2012 Canonical Ltd. | |
| # Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. | |
| # Copyright (C) 2012 Yahoo! Inc. | |
| # | |
| # Author: Scott Moser <scott.moser@canonical.com> | |
| # Author: Juerg Haefliger <juerg.haefliger@hp.com> | |
| # Author: Joshua Harlow <harlowja@yahoo-inc.com> | |
| # | |
| # This program is free software: you can redistribute it and/or modify | |
| # it under the terms of the GNU General Public License version 3, as | |
| # published by the Free Software Foundation. | |
| # | |
| # This program is distributed in the hope that it will be useful, | |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| # GNU General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU General Public License | |
| # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| from time import time | |
| import contextlib | |
| import os | |
| import six | |
| from six.moves.configparser import ( | |
| NoSectionError, NoOptionError, RawConfigParser) | |
| from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE, | |
| CFG_ENV_NAME) | |
| from cloudinit import log as logging | |
| from cloudinit import type_utils | |
| from cloudinit import util | |
| LOG = logging.getLogger(__name__) | |
| class LockFailure(Exception): | |
| pass | |
| class DummyLock(object): | |
| pass | |
| class DummySemaphores(object): | |
| def __init__(self): | |
| pass | |
| @contextlib.contextmanager | |
| def lock(self, _name, _freq, _clear_on_fail=False): | |
| yield DummyLock() | |
| def has_run(self, _name, _freq): | |
| return False | |
| def clear(self, _name, _freq): | |
| return True | |
| def clear_all(self): | |
| pass | |
| class FileLock(object): | |
| def __init__(self, fn): | |
| self.fn = fn | |
| def __str__(self): | |
| return "<%s using file %r>" % (type_utils.obj_name(self), self.fn) | |
| def canon_sem_name(name): | |
| return name.replace("-", "_") | |
| class FileSemaphores(object): | |
| def __init__(self, sem_path): | |
| self.sem_path = sem_path | |
| @contextlib.contextmanager | |
| def lock(self, name, freq, clear_on_fail=False): | |
| name = canon_sem_name(name) | |
| try: | |
| yield self._acquire(name, freq) | |
| except: | |
| if clear_on_fail: | |
| self.clear(name, freq) | |
| raise | |
| def clear(self, name, freq): | |
| name = canon_sem_name(name) | |
| sem_file = self._get_path(name, freq) | |
| try: | |
| util.del_file(sem_file) | |
| except (IOError, OSError): | |
| util.logexc(LOG, "Failed deleting semaphore %s", sem_file) | |
| return False | |
| return True | |
| def clear_all(self): | |
| try: | |
| util.del_dir(self.sem_path) | |
| except (IOError, OSError): | |
| util.logexc(LOG, "Failed deleting semaphore directory %s", | |
| self.sem_path) | |
| def _acquire(self, name, freq): | |
| # Check again if its been already gotten | |
| if self.has_run(name, freq): | |
| return None | |
| # This is a race condition since nothing atomic is happening | |
| # here, but this should be ok due to the nature of when | |
| # and where cloud-init runs... (file writing is not a lock...) | |
| sem_file = self._get_path(name, freq) | |
| contents = "%s: %s\n" % (os.getpid(), time()) | |
| try: | |
| util.write_file(sem_file, contents) | |
| except (IOError, OSError): | |
| util.logexc(LOG, "Failed writing semaphore file %s", sem_file) | |
| return None | |
| return FileLock(sem_file) | |
| def has_run(self, name, freq): | |
| if not freq or freq == PER_ALWAYS: | |
| return False | |
| cname = canon_sem_name(name) | |
| sem_file = self._get_path(cname, freq) | |
| # This isn't really a good atomic check | |
| # but it suffices for where and when cloudinit runs | |
| if os.path.exists(sem_file): | |
| return True | |
| # this case could happen if the migrator module hadn't run yet | |
| # but the item had run before we did canon_sem_name. | |
| if cname != name and os.path.exists(self._get_path(name, freq)): | |
| LOG.warn("%s has run without canonicalized name [%s].\n" | |
| "likely the migrator has not yet run. It will run next boot.\n" | |
| "run manually with: cloud-init single --name=migrator" | |
| % (name, cname)) | |
| return True | |
| return False | |
| def _get_path(self, name, freq): | |
| sem_path = self.sem_path | |
| if not freq or freq == PER_INSTANCE: | |
| return os.path.join(sem_path, name) | |
| else: | |
| return os.path.join(sem_path, "%s.%s" % (name, freq)) | |
| class Runners(object): | |
| def __init__(self, paths): | |
| self.paths = paths | |
| self.sems = {} | |
| def _get_sem(self, freq): | |
| if freq == PER_ALWAYS or not freq: | |
| return None | |
| sem_path = None | |
| if freq == PER_INSTANCE: | |
| # This may not exist, | |
| # so thats why we still check for none | |
| # below if say the paths object | |
| # doesn't have a datasource that can | |
| # provide this instance path... | |
| sem_path = self.paths.get_ipath("sem") | |
| elif freq == PER_ONCE: | |
| sem_path = self.paths.get_cpath("sem") | |
| if not sem_path: | |
| return None | |
| if sem_path not in self.sems: | |
| self.sems[sem_path] = FileSemaphores(sem_path) | |
| return self.sems[sem_path] | |
| def run(self, name, functor, args, freq=None, clear_on_fail=False): | |
| sem = self._get_sem(freq) | |
| if not sem: | |
| sem = DummySemaphores() | |
| if not args: | |
| args = [] | |
| if sem.has_run(name, freq): | |
| LOG.debug("%s already ran (freq=%s)", name, freq) | |
| return (False, None) | |
| with sem.lock(name, freq, clear_on_fail) as lk: | |
| if not lk: | |
| raise LockFailure("Failed to acquire lock for %s" % name) | |
| else: | |
| LOG.debug("Running %s using lock (%s)", name, lk) | |
| if isinstance(args, (dict)): | |
| results = functor(**args) | |
| else: | |
| results = functor(*args) | |
| return (True, results) | |
| class ConfigMerger(object): | |
| def __init__(self, paths=None, datasource=None, | |
| additional_fns=None, base_cfg=None, | |
| include_vendor=True): | |
| self._paths = paths | |
| self._ds = datasource | |
| self._fns = additional_fns | |
| self._base_cfg = base_cfg | |
| self._include_vendor = include_vendor | |
| # Created on first use | |
| self._cfg = None | |
| def _get_datasource_configs(self): | |
| d_cfgs = [] | |
| if self._ds: | |
| try: | |
| ds_cfg = self._ds.get_config_obj() | |
| if ds_cfg and isinstance(ds_cfg, (dict)): | |
| d_cfgs.append(ds_cfg) | |
| except: | |
| util.logexc(LOG, "Failed loading of datasource config object " | |
| "from %s", self._ds) | |
| return d_cfgs | |
| def _get_env_configs(self): | |
| e_cfgs = [] | |
| if CFG_ENV_NAME in os.environ: | |
| e_fn = os.environ[CFG_ENV_NAME] | |
| try: | |
| e_cfgs.append(util.read_conf(e_fn)) | |
| except: | |
| util.logexc(LOG, 'Failed loading of env. config from %s', | |
| e_fn) | |
| return e_cfgs | |
| def _get_instance_configs(self): | |
| i_cfgs = [] | |
| # If cloud-config was written, pick it up as | |
| # a configuration file to use when running... | |
| if not self._paths: | |
| return i_cfgs | |
| cc_paths = ['cloud_config'] | |
| if self._include_vendor: | |
| cc_paths.append('vendor_cloud_config') | |
| for cc_p in cc_paths: | |
| cc_fn = self._paths.get_ipath_cur(cc_p) | |
| if cc_fn and os.path.isfile(cc_fn): | |
| try: | |
| i_cfgs.append(util.read_conf(cc_fn)) | |
| except: | |
| util.logexc(LOG, 'Failed loading of cloud-config from %s', | |
| cc_fn) | |
| return i_cfgs | |
| def _read_cfg(self): | |
| # Input config files override | |
| # env config files which | |
| # override instance configs | |
| # which override datasource | |
| # configs which override | |
| # base configuration | |
| cfgs = [] | |
| if self._fns: | |
| for c_fn in self._fns: | |
| try: | |
| cfgs.append(util.read_conf(c_fn)) | |
| except: | |
| util.logexc(LOG, "Failed loading of configuration from %s", | |
| c_fn) | |
| cfgs.extend(self._get_env_configs()) | |
| cfgs.extend(self._get_instance_configs()) | |
| cfgs.extend(self._get_datasource_configs()) | |
| if self._base_cfg: | |
| cfgs.append(self._base_cfg) | |
| return util.mergemanydict(cfgs) | |
| @property | |
| def cfg(self): | |
| # None check to avoid empty case causing re-reading | |
| if self._cfg is None: | |
| self._cfg = self._read_cfg() | |
| return self._cfg | |
| class ContentHandlers(object): | |
| def __init__(self): | |
| self.registered = {} | |
| self.initialized = [] | |
| def __contains__(self, item): | |
| return self.is_registered(item) | |
| def __getitem__(self, key): | |
| return self._get_handler(key) | |
| def is_registered(self, content_type): | |
| return content_type in self.registered | |
| def register(self, mod, initialized=False, overwrite=True): | |
| types = set() | |
| for t in mod.list_types(): | |
| if overwrite: | |
| types.add(t) | |
| else: | |
| if not self.is_registered(t): | |
| types.add(t) | |
| for t in types: | |
| self.registered[t] = mod | |
| if initialized and mod not in self.initialized: | |
| self.initialized.append(mod) | |
| return types | |
| def _get_handler(self, content_type): | |
| return self.registered[content_type] | |
| def items(self): | |
| return list(self.registered.items()) | |
| class Paths(object): | |
| def __init__(self, path_cfgs, ds=None): | |
| self.cfgs = path_cfgs | |
| # Populate all the initial paths | |
| self.cloud_dir = path_cfgs.get('cloud_dir', '/var/lib/cloud') | |
| self.instance_link = os.path.join(self.cloud_dir, 'instance') | |
| self.boot_finished = os.path.join(self.instance_link, "boot-finished") | |
| self.upstart_conf_d = path_cfgs.get('upstart_dir') | |
| self.seed_dir = os.path.join(self.cloud_dir, 'seed') | |
| # This one isn't joined, since it should just be read-only | |
| template_dir = path_cfgs.get('templates_dir', '/etc/cloud/templates/') | |
| self.template_tpl = os.path.join(template_dir, '%s.tmpl') | |
| self.lookups = { | |
| "handlers": "handlers", | |
| "scripts": "scripts", | |
| "vendor_scripts": "scripts/vendor", | |
| "sem": "sem", | |
| "boothooks": "boothooks", | |
| "userdata_raw": "user-data.txt", | |
| "userdata": "user-data.txt.i", | |
| "obj_pkl": "obj.pkl", | |
| "cloud_config": "cloud-config.txt", | |
| "vendor_cloud_config": "vendor-cloud-config.txt", | |
| "data": "data", | |
| "vendordata_raw": "vendor-data.txt", | |
| "vendordata": "vendor-data.txt.i", | |
| } | |
| # Set when a datasource becomes active | |
| self.datasource = ds | |
| # get_ipath_cur: get the current instance path for an item | |
| def get_ipath_cur(self, name=None): | |
| ipath = self.instance_link | |
| add_on = self.lookups.get(name) | |
| if add_on: | |
| ipath = os.path.join(ipath, add_on) | |
| return ipath | |
| # get_cpath : get the "clouddir" (/var/lib/cloud/<name>) | |
| # for a name in dirmap | |
| def get_cpath(self, name=None): | |
| cpath = self.cloud_dir | |
| add_on = self.lookups.get(name) | |
| if add_on: | |
| cpath = os.path.join(cpath, add_on) | |
| return cpath | |
| # _get_ipath : get the instance path for a name in pathmap | |
| # (/var/lib/cloud/instances/<instance>/<name>) | |
| def _get_ipath(self, name=None): | |
| if not self.datasource: | |
| return None | |
| iid = self.datasource.get_instance_id() | |
| if iid is None: | |
| return None | |
| ipath = os.path.join(self.cloud_dir, 'instances', str(iid)) | |
| add_on = self.lookups.get(name) | |
| if add_on: | |
| ipath = os.path.join(ipath, add_on) | |
| return ipath | |
| # get_ipath : get the instance path for a name in pathmap | |
| # (/var/lib/cloud/instances/<instance>/<name>) | |
| # returns None + warns if no active datasource.... | |
| def get_ipath(self, name=None): | |
| ipath = self._get_ipath(name) | |
| if not ipath: | |
| LOG.warn(("No per instance data available, " | |
| "is there an datasource/iid set?")) | |
| return None | |
| else: | |
| return ipath | |
| # This config parser will not throw when sections don't exist | |
| # and you are setting values on those sections which is useful | |
| # when writing to new options that may not have corresponding | |
| # sections. Also it can default other values when doing gets | |
| # so that if those sections/options do not exist you will | |
| # get a default instead of an error. Another useful case where | |
| # you can avoid catching exceptions that you typically don't | |
| # care about... | |
| class DefaultingConfigParser(RawConfigParser): | |
| DEF_INT = 0 | |
| DEF_FLOAT = 0.0 | |
| DEF_BOOLEAN = False | |
| DEF_BASE = None | |
| def get(self, section, option): | |
| value = self.DEF_BASE | |
| try: | |
| value = RawConfigParser.get(self, section, option) | |
| except NoSectionError: | |
| pass | |
| except NoOptionError: | |
| pass | |
| return value | |
| def set(self, section, option, value=None): | |
| if not self.has_section(section) and section.lower() != 'default': | |
| self.add_section(section) | |
| RawConfigParser.set(self, section, option, value) | |
| def remove_option(self, section, option): | |
| if self.has_option(section, option): | |
| RawConfigParser.remove_option(self, section, option) | |
| def getboolean(self, section, option): | |
| if not self.has_option(section, option): | |
| return self.DEF_BOOLEAN | |
| return RawConfigParser.getboolean(self, section, option) | |
| def getfloat(self, section, option): | |
| if not self.has_option(section, option): | |
| return self.DEF_FLOAT | |
| return RawConfigParser.getfloat(self, section, option) | |
| def getint(self, section, option): | |
| if not self.has_option(section, option): | |
| return self.DEF_INT | |
| return RawConfigParser.getint(self, section, option) | |
| def stringify(self, header=None): | |
| contents = '' | |
| with six.StringIO() as outputstream: | |
| self.write(outputstream) | |
| outputstream.flush() | |
| contents = outputstream.getvalue() | |
| if header: | |
| contents = "\n".join([header, contents]) | |
| return contents |