forked from pantsbuild/pants
/
config.py
292 lines (221 loc) · 9.34 KB
/
config.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
# coding=utf-8
# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
from __future__ import absolute_import, division, print_function, unicode_literals
import configparser
import getpass
import io
import itertools
import os
from contextlib import contextmanager
import six
from twitter.common.collections import OrderedSet
from pants.base.build_environment import get_buildroot, get_pants_cachedir, get_pants_configdir
from pants.util.eval import parse_expression
from pants.util.meta import AbstractClass
class Config(AbstractClass):
"""Encapsulates ini-style config file loading and access.
Supports recursive variable substitution using standard python format strings. E.g.,
%(var_name)s will be replaced with the value of var_name.
"""
DEFAULT_SECTION = configparser.DEFAULTSECT
class ConfigError(Exception):
pass
class ConfigValidationError(ConfigError):
pass
@classmethod
def load_file_contents(cls, file_contents, seed_values=None):
"""Loads config from the given string payloads.
A handful of seed values will be set to act as if specified in the loaded config file's DEFAULT
section, and be available for use in substitutions. The caller may override some of these
seed values.
:param list[FileContents] file_contents: Load from these FileContents. Later instances take
precedence over earlier ones. If empty, returns an
empty config.
:param seed_values: A dict with optional override seed values for buildroot, pants_workdir,
pants_supportdir and pants_distdir.
"""
@contextmanager
def opener(file_content):
with io.StringIO(file_content.content.decode('utf-8')) as fh:
yield fh
return cls._meta_load(opener, file_contents, seed_values)
@classmethod
def load(cls, config_paths, seed_values=None):
"""Loads config from the given paths.
A handful of seed values will be set to act as if specified in the loaded config file's DEFAULT
section, and be available for use in substitutions. The caller may override some of these
seed values.
:param list config_paths: Load from these paths. Later instances take precedence over earlier
ones. If empty, returns an empty config.
:param seed_values: A dict with optional override seed values for buildroot, pants_workdir,
pants_supportdir and pants_distdir.
"""
@contextmanager
def opener(f):
with open(f, 'r') as fh:
yield fh
return cls._meta_load(opener, config_paths, seed_values)
@classmethod
def _meta_load(cls, open_ctx, config_items, seed_values=None):
if not config_items:
return _EmptyConfig()
single_file_configs = []
for config_item in config_items:
parser = cls._create_parser(seed_values)
with open_ctx(config_item) as ini:
parser.read_file(ini)
config_path = config_item.path if hasattr(config_item, 'path') else config_item
single_file_configs.append(_SingleFileConfig(config_path, parser))
return _ChainedConfig(single_file_configs)
@classmethod
def _create_parser(cls, seed_values=None):
"""Creates a config parser that supports %([key-name])s value substitution.
A handful of seed values will be set to act as if specified in the loaded config file's DEFAULT
section, and be available for use in substitutions. The caller may override some of these
seed values.
:param seed_values: A dict with optional override seed values for buildroot, pants_workdir,
pants_supportdir and pants_distdir.
"""
seed_values = seed_values or {}
buildroot = seed_values.get('buildroot', get_buildroot())
all_seed_values = {
'buildroot': buildroot,
'homedir': os.path.expanduser('~'),
'user': getpass.getuser(),
'pants_bootstrapdir': get_pants_cachedir(),
'pants_configdir': get_pants_configdir(),
}
def update_dir_from_seed_values(key, default):
all_seed_values[key] = seed_values.get(key, os.path.join(buildroot, default))
update_dir_from_seed_values('pants_workdir', '.pants.d')
update_dir_from_seed_values('pants_supportdir', 'build-support')
update_dir_from_seed_values('pants_distdir', 'dist')
return configparser.ConfigParser(all_seed_values)
def get(self, section, option, type_=six.string_types, default=None):
"""Retrieves option from the specified section (or 'DEFAULT') and attempts to parse it as type.
If the specified section does not exist or is missing a definition for the option, the value is
looked up in the DEFAULT section. If there is still no definition found, the default value
supplied is returned.
"""
return self._getinstance(section, option, type_, default)
def _getinstance(self, section, option, type_, default=None):
if not self.has_option(section, option):
return default
raw_value = self.get_value(section, option)
# We jump through some hoops here to deal with the fact that `six.string_types` is a tuple of
# types.
if (type_ == six.string_types or
(isinstance(type_, type) and issubclass(type_, six.string_types))):
return raw_value
key = '{}.{}'.format(section, option)
return parse_expression(name=key, val=raw_value, acceptable_types=type_,
raise_type=self.ConfigError)
# Subclasses must implement.
def configs(self):
"""Returns the underlying single-file configs represented by this object."""
raise NotImplementedError()
def sources(self):
"""Returns the sources of this config as a list of filenames."""
raise NotImplementedError()
def sections(self):
"""Returns the sections in this config (not including DEFAULT)."""
raise NotImplementedError()
def has_section(self, section):
"""Returns whether this config has the section."""
raise NotImplementedError()
def has_option(self, section, option):
"""Returns whether this config specified a value the option."""
raise NotImplementedError()
def get_value(self, section, option):
"""Returns the value of the option in this config as a string, or None if no value specified."""
raise NotImplementedError()
def get_source_for_option(self, section, option):
"""Returns the path to the source file the given option was defined in.
:param string section: the scope of the option.
:param string option: the name of the option.
:returns: the path to the config file, or None if the option was not defined by a config file.
:rtype: string
"""
raise NotImplementedError
class _EmptyConfig(Config):
"""A dummy config with no data at all."""
def sources(self):
return []
def configs(self):
return []
def sections(self):
return []
def has_section(self, section):
return False
def has_option(self, section, option):
return False
def get_value(self, section, option):
return None
def get_source_for_option(self, section, option):
return None
class _SingleFileConfig(Config):
"""Config read from a single file."""
def __init__(self, configpath, configparser):
super(_SingleFileConfig, self).__init__()
self.configpath = configpath
self.configparser = configparser
def configs(self):
return [self]
def sources(self):
return [self.configpath]
def sections(self):
return self.configparser.sections()
def has_section(self, section):
return self.configparser.has_section(section)
def has_option(self, section, option):
return self.configparser.has_option(section, option)
def get_value(self, section, option):
return self.configparser.get(section, option)
def get_source_for_option(self, section, option):
if self.has_option(section, option):
return self.sources()[0]
return None
class _ChainedConfig(Config):
"""Config read from multiple sources."""
def __init__(self, configs):
"""
:param configs: A list of Config instances to chain.
Later instances take precedence over earlier ones.
"""
super(_ChainedConfig, self).__init__()
self._configs = list(reversed(configs))
def configs(self):
return self._configs
def sources(self):
# NB: Present the sources in the order we were given them.
return list(itertools.chain.from_iterable(cfg.sources() for cfg in reversed(self._configs)))
def sections(self):
ret = OrderedSet()
for cfg in self._configs:
ret.update(cfg.sections())
return ret
def has_section(self, section):
for cfg in self._configs:
if cfg.has_section(section):
return True
return False
def has_option(self, section, option):
for cfg in self._configs:
if cfg.has_option(section, option):
return True
return False
def get_value(self, section, option):
for cfg in self._configs:
try:
return cfg.get_value(section, option)
except (configparser.NoSectionError, configparser.NoOptionError):
pass
if not self.has_section(section):
raise configparser.NoSectionError(section)
raise configparser.NoOptionError(option, section)
def get_source_for_option(self, section, option):
for cfg in self._configs:
if cfg.has_option(section, option):
return cfg.get_source_for_option(section, option)
return None