Skip to content
Permalink
Browse files

Make metadata rate limit configurable

  • Loading branch information...
adriengentil committed Jun 4, 2019
1 parent 2f11256 commit 3d61b3a9936b7351a84ad0484a50d6e080cd1468
Showing with 12 additions and 5 deletions.
  1. +1 −0 CONFIGURATION.md
  2. +10 −5 biggraphite/plugins/carbon.py
  3. +1 −0 biggraphite/settings.py
@@ -26,6 +26,7 @@ This is the main backend for BigGraphite and the one that should be used in prod
- ```BG_CASSANDRA_META_WRITE_CONSISTENCY```: Data write consistency (default: ```ONE```)
- ```BG_CASSANDRA_REPLICA_ID```: Identifier of this replica (default: ```0```)
- ```BG_CASSANDRA_READ_ON_SAMPLING_RATE```: Sampling rate to update metadata field ```read_on```. Setting to ```0``` disables updating ````read-on```` (default: ```0.1```)
- ```BG_CREATION_RATE_LIMIT```: Maximun number of new metadata to create per second (default: ```300```)


You can also fine-tune the schema of your tables directly as needed. If you
@@ -86,6 +86,12 @@ class BigGraphiteDatabase(database.TimeSeriesDatabase):
# See class pydoc for the rational.
_SYNC_EVERY_N_WRITES = 10

# Limit by default to 300 creations per seconds. This is mostly
# to give priority to other threads. A typical carbon instance
# can handle up to 200k metrics per second so it will take
# ~10 minutes to check all metrics.
_CREATION_RATE_LIMIT = 300

def __init__(self, settings):
"""Create a BigGraphiteDatabase."""
try:
@@ -103,6 +109,9 @@ def __init__(self, settings):
self._sync_every_n_writes = settings.get(
"BG_SYNC_EVERY_N_WRITES", self._SYNC_EVERY_N_WRITES
)
self._creation_rate_limit = settings.get(
"BG_CREATION_RATE_LIMIT", self._CREATION_RATE_LIMIT
)

utils.start_admin(bg_settings.settings_from_confattr(settings))
self.reactor.addSystemEventTrigger("before", "shutdown", self._flush)
@@ -295,11 +304,7 @@ def _createMetrics(self):
while self.reactor.running:
try:
self._createOneMetric()
# Hard limit to 300 creations per seconds. This is mostly
# to give priority to other threads. A typical carbon instance
# can handle up to 200k metrics per second so it will take
# ~10 minutes to check all metrics.
time.sleep(0.003)
time.sleep(1 / float(self._creation_rate_limit))
except Exception:
log.err()
# Give the system time to recover, errors might be related
@@ -58,6 +58,7 @@ def strvalidator(value):
"loglevel": str,
"storage_dir": str,
"admin_port": strtoint,
"creation_rate_limit": strtoint,
}


0 comments on commit 3d61b3a

Please sign in to comment.
You can’t perform that action at this time.