Skip to content

Commit

Permalink
Added config options for changing series loader and fields memcached …
Browse files Browse the repository at this point in the history
…keys (#41)

Added config options for changing series loader and fields memcached keys
  • Loading branch information
lcharkiewicz authored and pkittenis committed Sep 22, 2017
1 parent 341fad4 commit 1c6839e
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 9 deletions.
3 changes: 3 additions & 0 deletions graphite-api.yaml.example
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,9 @@ influxdb:
# Memcache (compressed) max value length in MB. The default of 1MB is
# the memcached server's default - this value should match server's value.
# max_value: 1
# Memcache key names for fields and series loader mutex
# series_loader_mutex_key: influxgraph_series_loader
# fields_key: infl_fields_key

## (Optional) Aggregation function configuration
#
Expand Down
24 changes: 15 additions & 9 deletions influxgraph/classes/finder.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ class InfluxDBFinder(object):
__fetch_multi__ = 'influxdb'
__slots__ = ('client', 'aggregation_functions',
'memcache', 'memcache_host', 'memcache_ttl',
'memcache_series_loader_mutex_key', 'memcache_fields_key',
'deltas', 'retention_policies', 'index', 'reader',
'index_lock', 'index_path', 'graphite_templates',
'loader_limit', 'fill_param')
Expand All @@ -76,6 +77,10 @@ def __init__(self, config):
memcache_host = memcache_conf.get('host')
self.memcache_ttl = memcache_conf.get(
'ttl', MEMCACHE_SERIES_DEFAULT_TTL)
self.memcache_series_loader_mutex_key = memcache_conf.get(
'series_loader_mutex_key', SERIES_LOADER_MUTEX_KEY)
self.memcache_fields_key = memcache_conf.get(
'fields_key', _MEMCACHE_FIELDS_KEY)
self.memcache = make_memcache_client(
memcache_host, memcache_max_value=memcache_conf.get('max_value', 1))
self.aggregation_functions = _compile_aggregation_patterns(
Expand Down Expand Up @@ -121,15 +126,15 @@ def _start_loader(self, series_loader_interval):
# requests to be served before series loader has completed at
# least once.
if _SERIES_LOADER_LOCK.acquire(block=False):
if self.memcache.get(SERIES_LOADER_MUTEX_KEY):
if self.memcache.get(self.memcache_series_loader_mutex_key):
logger.debug("Series loader mutex exists %s - "
"skipping series load",
SERIES_LOADER_MUTEX_KEY)
self.memcache_series_loader_mutex_key)
else:
logger.info("Starting initial series list load - this may "
"take several minutes on databases with a large "
"number of series..")
self.memcache.set(SERIES_LOADER_MUTEX_KEY, 1,
self.memcache.set(self.memcache_series_loader_mutex_key, 1,
time=series_loader_interval)
try:
if self.graphite_templates:
Expand Down Expand Up @@ -280,13 +285,14 @@ def _series_loader(self, interval=900):
time.sleep(interval)
if _SERIES_LOADER_LOCK.acquire(block=False):
_SERIES_LOADER_LOCK.release()
if self.memcache.get(SERIES_LOADER_MUTEX_KEY):
if self.memcache.get(self.memcache_series_loader_mutex_key):
logger.debug("Series loader mutex exists %s - "
"skipping series load",
SERIES_LOADER_MUTEX_KEY)
self.memcache_series_loader_mutex_key)
time.sleep(interval)
continue
self.memcache.set(SERIES_LOADER_MUTEX_KEY, 1, time=interval)
self.memcache.set(self.memcache_series_loader_mutex_key, 1,
time=interval)
start_time = datetime.datetime.now()
logger.debug("Starting series list loader..")
_SERIES_LOADER_LOCK.acquire()
Expand Down Expand Up @@ -613,8 +619,8 @@ def load_index(self):

def get_field_keys(self):
"""Get field keys for all measurements"""
field_keys = self.memcache.get(_MEMCACHE_FIELDS_KEY) if self.memcache \
else None
field_keys = self.memcache.get(self.memcache_fields_key) \
if self.memcache else None
if field_keys:
logger.debug("Found cached field keys")
return field_keys
Expand All @@ -624,7 +630,7 @@ def get_field_keys(self):
for ((key, _), vals) in data.items():
field_keys[key] = [val['fieldKey'] for val in vals]
if self.memcache:
if not self.memcache.set(_MEMCACHE_FIELDS_KEY, field_keys,
if not self.memcache.set(self.memcache_fields_key, field_keys,
time=self.memcache_ttl,
min_compress_len=1):
logger.error("Could not add field key list to memcache - "
Expand Down

0 comments on commit 1c6839e

Please sign in to comment.