Skip to content
Permalink
Browse files

Disable metadata management at runtime

  • Loading branch information...
adriengentil committed Jun 11, 2019
1 parent 3d61b3a commit 62977763a2a46ccc4fcfc6faabfd3a8d753762c7
Showing with 128 additions and 3 deletions.
  1. +16 −2 biggraphite/plugins/carbon.py
  2. +50 −1 biggraphite/utils.py
  3. +16 −0 tests/plugins/test_carbon.py
  4. +46 −0 tests/test_utils.py
@@ -16,6 +16,7 @@

import time
import datetime
import os

import prometheus_client
from six.moves import queue
@@ -85,6 +86,8 @@ class BigGraphiteDatabase(database.TimeSeriesDatabase):

# See class pydoc for the rational.
_SYNC_EVERY_N_WRITES = 10
_DISABLE_METADATA_FILENAME = \
os.getenv('GRAPHITE_ROOT', utils.DEFAULT_GRAPHITE_ROOT) + '/disable_metadata'

# Limit by default to 300 creations per seconds. This is mostly
# to give priority to other threads. A typical carbon instance
@@ -113,6 +116,10 @@ def __init__(self, settings):
"BG_CREATION_RATE_LIMIT", self._CREATION_RATE_LIMIT
)

# We want the metadata to be enabled when the file doesn't exists
watcher = utils.FileWatcher(self._DISABLE_METADATA_FILENAME)
self.metadata_switch = utils.FeatureSwitch(watcher, True)

utils.start_admin(bg_settings.settings_from_confattr(settings))
self.reactor.addSystemEventTrigger("before", "shutdown", self._flush)
self.reactor.callInThread(self._createMetrics)
@@ -285,15 +292,22 @@ def _background(self):
if self._accessor:
self.reactor.callInThread(self.accessor.background)

# check if metadata are enabled
metadata_enabled = self.metadata_switch.enabled()
self.metadata_switch.watch()
if metadata_enabled != self.metadata_switch.enabled():
log.cache("Metadata enabled: %s" % self.metadata_switch.enabled())

def _createAsync(self, metric, metric_name):
"""Add metric to the queue of metrics to create.
Args:
metric: a Metric object
metric_name: the original, un-encoded metric name
"""
self._metricsToCreate.put((metric, metric_name))
CREATES_ENQUEUED.inc()
if self.metadata_switch.enabled():
self._metricsToCreate.put((metric, metric_name))
CREATES_ENQUEUED.inc()

def _createMetrics(self):
# We create metrics in a separate thread because this is potentially
@@ -16,11 +16,13 @@

import logging
import os
import threading

import prometheus_client

DEFAULT_LOG_LEVEL = "WARNING"
DEFAULT_ADMIN_PORT = None
DEFAULT_GRAPHITE_ROOT = '/opt/graphite'


log = logging.getLogger(__name__)
@@ -38,6 +40,53 @@ class ConfigError(Error):
pass


class FileWatcher:
"""Watcher that checks if a file exists."""

def __init__(self, filename):
"""The constructor takes the filename to be watched."""
self._filename = filename

def watch(self):
"""Returns True if the file exists."""
return os.path.exists(self._filename)


class FeatureSwitch:
"""FeatureSwitch checks if a feature is enabled or not."""

def __init__(self, watcher, default_value):
"""Constructor for FeatureSwitch.
Args:
watcher: the watcher object used to determine if the feature is enabled or not
default_value: bool, value when the watcher object returns False
"""
self._watcher = watcher
self._default_value = default_value

# flag holding watcher's state
# Using an Event because this class is meant to be shared by several threads
self._flag = threading.Event()

# initializate the internal state
self.watch()

def enabled(self):
"""Check if the feature is enabled or not."""
if self._default_value:
return not self._flag.is_set()
else:
return self._flag.is_set()

def watch(self):
"""Refresh the internal state with the value returned by the watcher object."""
if self._watcher.watch():
self._flag.set()
else:
self._flag.clear()


def start_admin(settings):
"""Start the admin interface.
@@ -86,7 +135,7 @@ def setup_graphite_root_path(carbon_util_file):
"""
if os.path.dirname(carbon_util_file) == "/opt/graphite/lib/carbon":
if "GRAPHITE_ROOT" not in os.environ:
os.environ["GRAPHITE_ROOT"] = "/opt/graphite"
os.environ["GRAPHITE_ROOT"] = DEFAULT_GRAPHITE_ROOT


def round_down(rounded, divider):
@@ -18,6 +18,7 @@

bg_test_utils.prepare_graphite_imports() # noqa

import mock
import unittest

from carbon import database
@@ -117,6 +118,21 @@ def test_create_async(self):
retention = self._plugin.getMetadata(metric_name, "retention")
self.assertEqual(retention, metric.metadata.retention)

@mock.patch('biggraphite.utils.FeatureSwitch.enabled')
def test_create_async_should_not_create_metric_when_metadata_are_disabled(
self,
feature_switch_mock):
feature_switch_mock.return_value = False

metric_name = "a.metric.that.shoud.not.be.created"
metric = bg_test_utils.make_metric_with_defaults(metric_name)

self._plugin._createAsyncOrig(metric, metric_name)
self.assertFalse(self._plugin.exists(metric_name))

self._plugin._createOneMetric()
self.assertFalse(self._plugin.exists(metric_name))

def test_nosuchmetric(self):
other_metric = _TEST_METRIC + "-nosuchmetric"
self.assertRaises(
@@ -255,5 +255,51 @@ def __drop_all_metrics(self):
self.accessor.drop_all_metrics()


class TestFeatureSwitch(unittest.TestCase):

class WatcherMock:
"""WatcherMock used to mock a watcher in FeatureSwitch object."""

def __init__(self):
"""Constructor set the internal state."""
self._state = False

def set(self):
"""Set the internal state to True."""
self._state = True

def clear(self):
"""Set the internal state to False."""
self._state = False

def watch(self):
"""Return the internal state value."""
return self._state

def test_feature_switch_should_return_watcher_state_when_default_value_is_false(self):
watcher = self.WatcherMock()
feature_switch = bg_utils.FeatureSwitch(watcher, False)

# default watcher state is False
self.assertFalse(feature_switch.enabled())

# set watcher state to True
watcher.set()
feature_switch.watch()
self.assertTrue(feature_switch.enabled())

def test_feature_switch_should_return_opposite_watcher_state_when_default_value_is_true(self):
watcher = self.WatcherMock()
feature_switch = bg_utils.FeatureSwitch(watcher, True)

# default watcher state is False
self.assertTrue(feature_switch.enabled())

# set watcher state to True
watcher.set()
feature_switch.watch()
self.assertFalse(feature_switch.enabled())


if __name__ == "__main__":
unittest.main()

0 comments on commit 6297776

Please sign in to comment.
You can’t perform that action at this time.