Merge pull request #2118 from replay/tagging_xorgid_rebased

pass headers to tag db

webapp/graphite/render/evaluator.py

@@ -147,7 +147,10 @@ def extractPathExpression(requestContext, tokens, replacements=None):
       # if we're prefetching seriesByTag, look up the matching series and prefetch those
       if tokens.call.funcname == 'seriesByTag':
         if STORE.tagdb:
-          for series in STORE.tagdb.find_series(tuple([t.string[1:-1] for t in tokens.call.args if t.string])):
+          for series in STORE.tagdb.find_series(
+              tuple([t.string[1:-1] for t in tokens.call.args if t.string]),
+              requestContext=requestContext,
+          ):
             pathExpressions.add(series)
       else:
         for a in tokens.call.args:

webapp/graphite/render/functions.py

@@ -4274,7 +4274,7 @@ def seriesByTag(requestContext, *tagExpressions):
     log.info('seriesByTag called but no TagDB configured')
     return []
 
-  taggedSeries = STORE.tagdb.find_series(tagExpressions)
+  taggedSeries = STORE.tagdb.find_series(tagExpressions, requestContext=requestContext)
   if not taggedSeries:
     return []
 

webapp/graphite/tags/base.py

@@ -17,7 +17,7 @@ def __init__(self):
     """Initialize the tag db."""
 
   @logtime
-  def find_series(self, tags, timer=None):
+  def find_series(self, tags, timer=None, requestContext=None):
     """
     Find series by tag, accepts a list of tag specifiers and returns a list of matching paths.
 
@@ -45,27 +45,27 @@ def find_series(self, tags, timer=None):
     if result is not None:
       timer.set_msg('completed (cached) in')
     else:
-      result = self._find_series(tags)
+      result = self._find_series(tags, requestContext)
       cache.set(cacheKey, result, settings.TAGDB_CACHE_DURATION)
 
     return result
 
   @abc.abstractmethod
-  def _find_series(self, tags):
+  def _find_series(self, tags, requestContext=None):
     """
     Internal function called by find_series, follows the same semantics allowing base class to implement caching
     """
 
   @abc.abstractmethod
-  def get_series(self, path):
+  def get_series(self, path, requestContext=None):
     """
     Get series by path, accepts a path string and returns a TaggedSeries object describing the series.
 
     If the path is not found in the TagDB, returns None.
     """
 
   @abc.abstractmethod
-  def list_tags(self, tagFilter=None):
+  def list_tags(self, tagFilter=None, requestContext=None):
     """
     List defined tags, returns a list of dictionaries describing the tags stored in the TagDB.
 
@@ -83,7 +83,7 @@ def list_tags(self, tagFilter=None):
     """
 
   @abc.abstractmethod
-  def get_tag(self, tag, valueFilter=None):
+  def get_tag(self, tag, valueFilter=None, requestContext=None):
     """
     Get details of a particular tag, accepts a tag name and returns a dict describing the tag.
 
@@ -106,7 +106,7 @@ def get_tag(self, tag, valueFilter=None):
     """
 
   @abc.abstractmethod
-  def list_values(self, tag, valueFilter=None):
+  def list_values(self, tag, valueFilter=None, requestContext=None):
     """
     List values for a particular tag, returns a list of dictionaries describing the values stored in the TagDB.
 
@@ -126,18 +126,18 @@ def list_values(self, tag, valueFilter=None):
     """
 
   @abc.abstractmethod
-  def tag_series(self, series):
+  def tag_series(self, series, requestContext=None):
     """
     Enter series into database.  Accepts a series string, upserts into the TagDB and returns the canonicalized series name.
     """
 
   @abc.abstractmethod
-  def del_series(self, series):
+  def del_series(self, series, requestContext=None):
     """
     Remove series from database.  Accepts a series string and returns True
     """
 
-  def auto_complete_tags(self, exprs, tagPrefix=None, limit=None):
+  def auto_complete_tags(self, exprs, tagPrefix=None, limit=None, requestContext=None):
     """
     Return auto-complete suggestions for tags based on the matches for the specified expressions, optionally filtered by tag prefix
     """
@@ -147,13 +147,18 @@ def auto_complete_tags(self, exprs, tagPrefix=None, limit=None):
       limit = int(limit)
 
     if not exprs:
-      return [tagInfo['tag'] for tagInfo in self.list_tags(tagFilter=('^(' + tagPrefix + ')' if tagPrefix else None))[:limit]]
+      return [
+        tagInfo['tag'] for tagInfo in self.list_tags(
+          tagFilter='^(' + tagPrefix + ')' if tagPrefix else None,
+          requestContext=requestContext,
+        )[:limit]
+      ]
 
     result = []
 
     searchedTags = set([self.parse_tagspec(expr)[0] for expr in exprs])
 
-    for path in self.find_series(exprs):
+    for path in self.find_series(exprs, requestContext=requestContext):
       tags = self.parse(path).tags
       for tag in tags:
         if tag in searchedTags:
@@ -173,7 +178,7 @@ def auto_complete_tags(self, exprs, tagPrefix=None, limit=None):
 
     return result
 
-  def auto_complete_values(self, exprs, tag, valuePrefix=None, limit=None):
+  def auto_complete_values(self, exprs, tag, valuePrefix=None, limit=None, requestContext=None):
     """
     Return auto-complete suggestions for tags and values based on the matches for the specified expressions, optionally filtered by tag and/or value prefix
     """
@@ -183,11 +188,17 @@ def auto_complete_values(self, exprs, tag, valuePrefix=None, limit=None):
       limit = int(limit)
 
     if not exprs:
-      return [v['value'] for v in self.list_values(tag, valueFilter=('^(' + valuePrefix + ')' if valuePrefix else None))[:limit]]
+      return [
+        v['value'] for v in self.list_values(
+          tag,
+          valueFilter='^(' + valuePrefix + ')' if valuePrefix else None,
+          requestContext=requestContext,
+        )[:limit]
+      ]
 
     result = []
 
-    for path in self.find_series(exprs):
+    for path in self.find_series(exprs, requestContext=requestContext):
       tags = self.parse(path).tags
       if tag not in tags:
         continue

webapp/graphite/tags/http.py

@@ -18,11 +18,11 @@ def __init__(self):
     self.username = settings.TAGDB_HTTP_USER
     self.password = settings.TAGDB_HTTP_PASSWORD
 
-  def request(self, method, url, fields=None):
+  def request(self, method, url, fields=None, requestContext=None):
     if not fields:
       fields = {}
 
-    headers = {}
+    headers = requestContext.get('forwardHeaders') if requestContext else {}
     if 'Authorization' not in headers and self.username and self.password:
       headers['Authorization'] = 'Basic ' + ('%s:%s' % (self.username, self.password)).encode('base64')
 
@@ -39,42 +39,50 @@ def request(self, method, url, fields=None):
 
     return json.loads(result.data.decode('utf-8'))
 
-  def _find_series(self, tags):
-    return self.request('GET', '/tags/findSeries?' + '&'.join([('expr=%s' % quote(tag)) for tag in tags]))
+  def _find_series(self, tags, requestContext=None):
+    return self.request(
+      'GET',
+      '/tags/findSeries?' + '&'.join([('expr=%s' % quote(tag)) for tag in tags]),
+      requestContext=requestContext,
+    )
 
-  def get_series(self, path):
+  def get_series(self, path, requestContext=None):
     parsed = self.parse(path)
 
-    seriesList = self.find_series([('%s=%s' % (tag, parsed.tags[tag])) for tag in parsed.tags])
+    seriesList = self.find_series(
+      [('%s=%s' % (tag, parsed.tags[tag])) for tag in parsed.tags],
+      requestContext=requestContext,
+    )
 
     if parsed.path in seriesList:
       return parsed
 
-  def list_tags(self, tagFilter=None):
-    return self.request('GET', '/tags', {'filter': tagFilter})
+  def list_tags(self, tagFilter=None, requestContext=None):
+    return self.request('GET', '/tags', {'filter': tagFilter}, requestContext)
 
-  def get_tag(self, tag, valueFilter=None):
-    return self.request('GET', '/tags/' + tag, {'filter': valueFilter})
+  def get_tag(self, tag, valueFilter=None, requestContext=None):
+    return self.request('GET', '/tags/' + tag, {'filter': valueFilter}, requestContext)
 
-  def list_values(self, tag, valueFilter=None):
-    tagInfo = self.get_tag(tag, valueFilter)
+  def list_values(self, tag, valueFilter=None, requestContext=None):
+    tagInfo = self.get_tag(tag, valueFilter, requestContext)
     if not tagInfo:
       return []
 
     return tagInfo['values']
 
-  def tag_series(self, series):
-    return self.request('POST', '/tags/tagSeries', {'path': series})
+  def tag_series(self, series, requestContext=None):
+    return self.request('POST', '/tags/tagSeries', {'path': series}, requestContext)
 
-  def del_series(self, series):
-    return self.request('POST', '/tags/delSeries', {'path': series})
+  def del_series(self, series, requestContext=None):
+    return self.request('POST', '/tags/delSeries', {'path': series}, requestContext)
 
-  def auto_complete_tags(self, exprs, tagPrefix=None, limit=None):
+  def auto_complete_tags(self, exprs, tagPrefix=None, limit=None, requestContext=None):
     """
     Return auto-complete suggestions for tags based on the matches for the specified expressions, optionally filtered by tag prefix
     """
     if not settings.TAGDB_HTTP_AUTOCOMPLETE:
-      return super(HttpTagDB, self).auto_complete_tags(exprs, tagPrefix=tagPrefix, limit=limit)
+      return super(HttpTagDB, self).auto_complete_tags(
+        exprs, tagPrefix=tagPrefix, limit=limit, requestContext=requestContext)
 
     if limit is None:
       limit = settings.TAGDB_AUTOCOMPLETE_LIMIT
@@ -84,12 +92,13 @@ def auto_complete_tags(self, exprs, tagPrefix=None, limit=None):
 
     return self.request('GET', url)
 
-  def auto_complete_values(self, exprs, tag, valuePrefix=None, limit=None):
+  def auto_complete_values(self, exprs, tag, valuePrefix=None, limit=None, requestContext=None):
     """
     Return auto-complete suggestions for tags and values based on the matches for the specified expressions, optionally filtered by tag and/or value prefix
     """
     if not settings.TAGDB_HTTP_AUTOCOMPLETE:
-      return super(HttpTagDB, self).auto_complete_values(exprs, tag, valuePrefix=valuePrefix, limit=limit)
+      return super(HttpTagDB, self).auto_complete_values(
+        exprs, tag, valuePrefix=valuePrefix, limit=limit, requestContext=requestContext)
 
     if limit is None:
       limit = settings.TAGDB_AUTOCOMPLETE_LIMIT

webapp/graphite/tags/localdatabase.py

@@ -88,7 +88,7 @@ def find_series_query(self, tags):
 
     return sql, params, filters
 
-  def _find_series(self, tags):
+  def _find_series(self, tags, requestContext=None):
     sql, params, filters = self.find_series_query(tags)
 
     def matches_filters(path):
@@ -114,7 +114,7 @@ def matches_filters(path):
 
       return [row[0] for row in cursor if matches_filters(row[0])]
 
-  def get_series(self, path):
+  def get_series(self, path, requestContext=None):
     with connection.cursor() as cursor:
       sql = 'SELECT s.id, t.tag, v.value'
       sql += ' FROM tags_series AS s'
@@ -134,7 +134,7 @@ def get_series(self, path):
 
       return TaggedSeries(tags['name'], tags, series_id=series_id)
 
-  def list_tags(self, tagFilter=None):
+  def list_tags(self, tagFilter=None, requestContext=None):
     with connection.cursor() as cursor:
       sql = 'SELECT t.id, t.tag'
       sql += ' FROM tags_tag AS t'
@@ -152,7 +152,7 @@ def list_tags(self, tagFilter=None):
 
       return [{'id': tag_id, 'tag': tag} for (tag_id, tag) in cursor]
 
-  def get_tag(self, tag, valueFilter=None):
+  def get_tag(self, tag, valueFilter=None, requestContext=None):
     with connection.cursor() as cursor:
       sql = 'SELECT t.id, t.tag'
       sql += ' FROM tags_tag AS t'
@@ -173,7 +173,7 @@ def get_tag(self, tag, valueFilter=None):
       'values': self.list_values(tag, valueFilter=valueFilter),
     }
 
-  def list_values(self, tag, valueFilter=None ):
+  def list_values(self, tag, valueFilter=None, requestContext=None):
     with connection.cursor() as cursor:
       sql = 'SELECT v.id, v.value, COUNT(st.id)'
       sql += ' FROM tags_tagvalue AS v'
@@ -236,7 +236,7 @@ def _regexp_not_operator(connection):
       return '!~*'
     raise Exception('Database vendor ' + connection.vendor + ' does not support regular expressions')
 
-  def tag_series(self, series):
+  def tag_series(self, series, requestContext=None):
     # extract tags and normalize path
     parsed = self.parse(series)
 
@@ -286,7 +286,7 @@ def tag_series(self, series):
 
     return path
 
-  def del_series(self, series):
+  def del_series(self, series, requestContext=None):
     # extract tags and normalize path
     parsed = self.parse(series)
 

webapp/graphite/tags/redis.py

@@ -28,7 +28,7 @@ def __init__(self):
 
     self.r = Redis(host=settings.TAGDB_REDIS_HOST,port=settings.TAGDB_REDIS_PORT,db=settings.TAGDB_REDIS_DB)
 
-  def _find_series(self, tags):
+  def _find_series(self, tags, requestContext=None):
     selector = None
     selector_cnt = None
     filters = []
@@ -142,7 +142,7 @@ def _find_series(self, tags):
 
     return sorted(results)
 
-  def get_series(self, path):
+  def get_series(self, path, requestContext=None):
     tags = {}
 
     tags = self.r.hgetall('series:' + path + ':tags')
@@ -151,14 +151,14 @@ def get_series(self, path):
 
     return TaggedSeries(tags['name'], tags)
 
-  def list_tags(self, tagFilter=None):
+  def list_tags(self, tagFilter=None, requestContext=None):
     return sorted([
       {'tag': tag}
       for tag in self.r.sscan_iter('tags')
       if not tagFilter or re.match(tagFilter, tag) is not None
     ], key=lambda x: x['tag'])
 
-  def get_tag(self, tag, valueFilter=None):
+  def get_tag(self, tag, valueFilter=None, requestContext=None):
     if not self.r.sismember('tags', tag):
       return None
 
@@ -167,14 +167,14 @@ def get_tag(self, tag, valueFilter=None):
       'values': self.list_values(tag, valueFilter=valueFilter),
     }
 
-  def list_values(self, tag, valueFilter=None):
+  def list_values(self, tag, valueFilter=None, requestContext=None):
     return sorted([
       {'value': value, 'count': self.r.scard('tags:' + tag + ':values:' + value)}
       for value in self.r.sscan_iter('tags:' + tag + ':values')
       if not valueFilter or re.match(valueFilter, value) is not None
     ], key=lambda x: x['value'])
 
-  def tag_series(self, series):
+  def tag_series(self, series, requestContext=None):
     # extract tags and normalize path
     parsed = self.parse(series)
 
@@ -195,7 +195,7 @@ def tag_series(self, series):
 
     return path
 
-  def del_series(self, series):
+  def del_series(self, series, requestContext=None):
     # extract tags and normalize path
     parsed = self.parse(series)