/
config.py
360 lines (293 loc) · 12.9 KB
/
config.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
# -*- coding: utf-8 -*-
# Copyright 2014-2021 CERN
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Authors:
# - Mario Lassnig <mario.lassnig@cern.ch>, 2014-2019
# - Vincent Garonne <vincent.garonne@cern.ch>, 2015-2017
# - Cedric Serfon <cedric.serfon@cern.ch>, 2017
# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
# - Brandon White <bjwhite@fnal.gov>, 2019
# - Martin Barisits <martin.barisits@cern.ch>, 2019-2021
# - Radu Carpa <radu.carpa@cern.ch>, 2021
from dogpile.cache import make_region
from dogpile.cache.api import NoValue
from sqlalchemy import func
from rucio.common.config import config_get
from rucio.common.exception import ConfigNotFound
from rucio.db.sqla import models
from rucio.db.sqla.session import read_session, transactional_session
REGION = make_region().configure('dogpile.cache.memcached',
expiration_time=3600,
arguments={'url': config_get('cache', 'url', False, '127.0.0.1:11211',
check_config_table=False), 'distributed_lock': True})
SECTIONS_CACHE_KEY = 'sections'
def _has_section_cache_key(section):
return 'has_section_%s' % section
def _options_cache_key(section):
return 'options_%s' % section
def _has_option_cache_key(section, option):
return 'has_option_%s_%s' % (section, option)
def _items_cache_key(section):
return 'items_%s' % section
def _value_cache_key(section, option):
return 'get_%s_%s' % (section, option)
@read_session
def sections(use_cache=True, expiration_time=3600, session=None):
"""
Return a list of the sections available.
:param use_cache: Boolean if the cache should be used.
:param expiration_time: Time after that the cached value gets ignored.
:param session: The database session in use.
:returns: ['section_name', ...]
"""
all_sections = NoValue()
if use_cache:
all_sections = read_from_cache(SECTIONS_CACHE_KEY, expiration_time)
if isinstance(all_sections, NoValue):
query = session.query(models.Config.section).distinct().all()
all_sections = [section[0] for section in query]
write_to_cache(SECTIONS_CACHE_KEY, all_sections)
return all_sections
@transactional_session
def add_section(section, session=None):
"""
Add a section to the configuration.
:param session: The database session in use.
:param section: The name of the section.
"""
raise NotImplementedError('Irrelevant - sections cannot exist without options')
@read_session
def has_section(section, use_cache=True, expiration_time=3600, session=None):
"""
Indicates whether the named section is present in the configuration.
:param section: The name of the section.
:param use_cache: Boolean if the cache should be used.
:param expiration_time: Time after that the cached value gets ignored.
:param session: The database session in use.
:returns: True/False
"""
has_section_key = 'has_section_%s' % section
has_section = NoValue()
if use_cache:
has_section = read_from_cache(has_section_key, expiration_time)
if isinstance(has_section, NoValue):
query = session.query(models.Config).filter_by(section=section)
has_section = True if query.first() else False
write_to_cache(has_section_key, has_section)
return has_section
@read_session
def options(section, use_cache=True, expiration_time=3600, session=None):
"""
Returns a list of options available in the specified section.
:param section: The name of the section.
:param use_cache: Boolean if the cache should be used.
:param expiration_time: Time after that the cached value gets ignored.
:param session: The database session in use.
:returns: ['option', ...]
"""
options_key = _options_cache_key(section)
options = NoValue()
if use_cache:
options = read_from_cache(options_key, expiration_time)
if isinstance(options, NoValue):
query = session.query(models.Config.opt).filter_by(section=section).distinct().all()
options = [option[0] for option in query]
write_to_cache(options_key, options)
return options
@read_session
def has_option(section, option, use_cache=True, expiration_time=3600, session=None):
"""
Check if the given section exists and contains the given option.
:param section: The name of the section.
:param option: The name of the option.
:param use_cache: Boolean if the cache should be used.
:param expiration_time: Time after that the cached value gets ignored.
:param session: The database session in use.
:returns: True/False
"""
has_option_key = _has_option_cache_key(section, option)
has_option = NoValue()
if use_cache:
has_option = read_from_cache(has_option_key, expiration_time)
if isinstance(has_option, NoValue):
query = session.query(models.Config).filter_by(section=section, opt=option)
has_option = True if query.first() else False
write_to_cache(has_option_key, has_option)
return has_option
@read_session
def get(section, option, default=None, use_cache=True, expiration_time=3600, session=None):
"""
Get an option value for the named section. Value can be auto-coerced to string, int, float, bool, None.
Caveat emptor: Strings, regardless the case, matching 'on'/off', 'true'/'false', 'yes'/'no' are converted to bool.
0/1 are converted to int, and not to bool.
:param section: The name of the section.
:param option: The name of the option.
:param default: The default value if no value is found.
:param use_cache: Boolean if the cache should be used.
:param expiration_time: Time after that the cached value gets ignored.
:param session: The database session in use.
:returns: The auto-coerced value.
"""
value_key = _value_cache_key(section, option)
value = NoValue()
if use_cache:
value = read_from_cache(value_key, expiration_time)
if isinstance(value, NoValue):
tmp = session.query(models.Config.value).filter_by(section=section, opt=option).first()
if tmp is not None:
value = __convert_type(tmp[0])
write_to_cache(value_key, tmp[0])
elif default is None:
raise ConfigNotFound
else:
value = default
write_to_cache(value_key, str(value)) # Also write default to cache
else:
value = __convert_type(value)
return value
@read_session
def items(section, use_cache=True, expiration_time=3600, session=None):
"""
Return a list of (option, value) pairs for each option in the given section. Values are auto-coerced as in get().
:param section: The name of the section.
:param use_cache: Boolean if the cache should be used.
:param expiration_time: Time after that the cached value gets ignored.
:param session: The database session in use.
:returns: [('option', auto-coerced value), ...]
"""
items_key = _items_cache_key(section)
items = NoValue()
if use_cache:
items = read_from_cache(items_key, expiration_time)
if isinstance(items, NoValue):
items = session.query(models.Config.opt, models.Config.value).filter_by(section=section).all()
write_to_cache(items_key, items)
return [(item[0], __convert_type(item[1])) for item in items]
@transactional_session
def set(section, option, value, session=None):
"""
Set the given option to the specified value. If the option doesn't exist, it is created.
:param section: The name of the section.
:param option: The name of the option.
:param value: The content of the value.
:param session: The database session in use.
"""
if not has_option(section=section, option=option, use_cache=False, session=session):
section_existed = has_section(section=section)
new_option = models.Config(section=section, opt=option, value=value)
new_option.save(session=session)
delete_from_cache(key=_value_cache_key(section, option))
delete_from_cache(key=_has_option_cache_key(section, option))
delete_from_cache(key=_items_cache_key(section))
if not section_existed:
delete_from_cache(key=SECTIONS_CACHE_KEY)
delete_from_cache(key=_has_section_cache_key(section))
else:
old_value = session.query(models.Config.value).filter_by(section=section,
opt=option).first()[0]
if old_value != str(value):
old_option = models.ConfigHistory(section=section,
opt=option,
value=old_value)
old_option.save(session=session)
session.query(models.Config).filter_by(section=section, opt=option).update({'value': str(value)})
delete_from_cache(key=_value_cache_key(section, option))
delete_from_cache(key=_items_cache_key(section))
@transactional_session
def remove_section(section, session=None):
"""
Remove the specified section from the specified section.
:param section: The name of the section.
:param session: The database session in use.
:returns: True/False.
"""
if not has_section(section=section, session=session):
return False
else:
for old in session.query(models.Config.value).filter_by(section=section).all():
old_option = models.ConfigHistory(section=old[0],
opt=old[1],
value=old[2])
old_option.save(session=session)
delete_from_cache(key=_has_option_cache_key(old[0], old[1]))
delete_from_cache(key=_value_cache_key(old[0], old[1]))
session.query(models.Config).filter_by(section=section).delete()
delete_from_cache(key=SECTIONS_CACHE_KEY)
delete_from_cache(key=_items_cache_key(section))
return True
@transactional_session
def remove_option(section, option, session=None):
"""
Remove the specified option from the configuration.
:param section: The name of the section.
:param option: The name of the option.
:param session: The database session in use.
:returns: True/False
"""
if not has_option(section=section, option=option, session=session, use_cache=False):
return False
else:
old_option = models.ConfigHistory(section=section,
opt=option,
value=session.query(models.Config.value).filter_by(section=section,
opt=option).first()[0])
old_option.save(session=session)
session.query(models.Config).filter_by(section=section, opt=option).delete()
if not session.query(func.count('*')).select_from(models.Config).filter_by(section=section).scalar():
# we deleted the last config entry in the section. Invalidate the section cache
delete_from_cache(key=SECTIONS_CACHE_KEY)
delete_from_cache(key=_has_section_cache_key(section))
delete_from_cache(key=_items_cache_key(section))
delete_from_cache(key=_has_option_cache_key(section, option))
delete_from_cache(key=_value_cache_key(section, option))
return True
def __convert_type(value):
'''
__convert_type
'''
if value.lower() in ['true', 'yes', 'on']:
return True
elif value.lower() in ['false', 'no', 'off']:
return False
for conv in (int, float):
try:
return conv(value)
except:
pass
return value
def read_from_cache(key, expiration_time=3600):
"""
Try to read a value from a cache.
:param key: Key that stores the value.
:param expiration_time: Time in seconds that a value should not be older than.
"""
key = key.replace(' ', '')
value = REGION.get(key, expiration_time=expiration_time)
return value
def write_to_cache(key, value):
"""
Set a value on a key in a cache.
:param key: Key that stores the value.
:param value: Value to be stored.
"""
key = key.replace(' ', '')
REGION.set(key, value)
def delete_from_cache(key):
"""
Delete from cache any data stored for the given key
:param key: Key that stores the value.
"""
key = key.replace(' ', '')
REGION.delete(key)