A lot of annotator features

debian/snmpcollector.install

@@ -1,6 +1,4 @@
 opt/snmpcollector/src/* /usr/share/snmpcollector/
-opt/snmpcollector/snmpcollector-test /usr/bin/
-opt/snmpcollector/snmpcollector-trigger /usr/bin/
 etc/snmpcollector.yaml
 etc/default/snmpcollector
 etc/init.d/snmpcollector

debian/snmpcollector.links

@@ -0,0 +1,2 @@
+/usr/share/snmpcollector/trigger.py /usr/bin/snmpcollector-trigger
+/usr/share/snmpcollector/snmptest.py /usr/bin/snmpcollector-test

src/snmpcollector/Makefile

@@ -15,8 +15,10 @@ install:
 	python setup.py install --root $(DESTDIR) $(COMPILE)
 	mkdir -p $(DESTDIR)/opt/snmpcollector/src/
 	cp src/*.py $(DESTDIR)/opt/snmpcollector/src/
-	cp snmpcollector-test $(DESTDIR)/opt/snmpcollector/
-	cp snmpcollector-trigger $(DESTDIR)/opt/snmpcollector/
+	ln -sf /opt/snmpcollector/src/trigger.py \
+		$(DESTDIR)/opt/snmpcollector/snmpcollector-trigger
+	ln -sf /opt/snmpcollector/src/snmptest.py \
+		$(DESTDIR)/opt/snmpcollector/snmpcollector-test
 	install -D -m600 etc/snmpcollector.yaml $(DESTDIR)/etc/
 	install -D etc/snmpcollector.default $(DESTDIR)/etc/default/snmpcollector
 	install -D snmpcollector.init $(DESTDIR)/etc/init.d/snmpcollector

src/snmpcollector/etc/snmpcollector.yaml

@@ -57,6 +57,20 @@ snmp:
 
 annotator:
 
+  # Labelification is used to turn strings into labels on metrics that
+  # otherwise do not have any numeric data. The value will be fixed to 1
+  # and the string value will be moved to a label called 'value' and 'hex'.
+  # Use this if you don't have any sensible OID to annotate with the value or
+  # there isn't a 1:1 match between the index and the value you wish to use.
+  #
+  # 'value' contains the human readable characters only and is striped.
+  # 'hex' is the raw data but hex encoded.
+  # If the raw string value is empty the result is dropped
+  labelify:
+    - .1.3.6.1.2.1.47.1.1.1.1.9  # entPhysicalFirmwareRev
+    - .1.3.6.1.2.1.47.1.1.1.1.11 # entPhysicalSerialNum
+    - .1.3.6.1.2.1.47.1.1.1.1.13 # entPhysicalModelName
+
   annotations:
     - annotate:
         - .1.3.6.1.2.1.2.2.1            # ifTable
@@ -97,27 +111,6 @@ annotator:
       with:
         essid: .1.3.6.1.4.1.14179.2.1.1.1.2  # bsnDot11EssSsid
 
-  aggregate:
-    switch:
-      layers:
-        - access
-        - wifi
-        - dist
-        - core
-      properties:
-        model:
-          walk: .1.3.6.1.2.1.47.1.1.1.1.13 # entPhysicalModelName
-          find: non-empty-string
-        firmware:
-          walk: .1.3.6.1.2.1.47.1.1.1.1.9  # entPhysicalFirmwareRev
-          find: non-empty-string
-
-  convert:
-    .1.3.6.1.2.1.47.1.1.1.1.11: # entPhysicalSerialNum
-      value: 1
-      label: sn
-      keep-only: non-empty-string
-
 collection:
   Default OIDs:
     models:

src/snmpcollector/src/annotator.py

@@ -1,17 +1,23 @@
 #!/usr/bin/env python2
-import base64
+import binascii
 import collections
 import logging
 import time
 
 import actions
 import config
+import snmp
 import stage
 
 
 class Annotator(object):
   """Annotation step where results are given meaningful labels."""
 
+  LABEL_TYPES = set(['OCTETSTR', 'IPADDR'])
+  ALLOWED_CHARACTERS = (
+      '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+      '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ ')
+
   def __init__(self):
     super(Annotator, self).__init__()
     self.mibcache = {}
@@ -26,7 +32,7 @@ def mibresolver(self):
     return self._mibresolver
 
   def do_result(self, run, target, results, stats):
-    annotations = config.get('annotator', 'annotations')
+    annotations = config.get('annotator', 'annotations') or []
 
     # Calculate map to skip annotation if we're sure we're not going to annotate
     # TODO(bluecmd): This could be cached
@@ -36,32 +42,47 @@ def do_result(self, run, target, results, stats):
         # Add '.' to not match .1.2.3 if we want to annotate 1.2.30
         annotation_map[annotate + '.'] = annotation['with']
 
+    labelification = set(
+        [x + '.' for x in config.get('annotator', 'labelify') or []])
+
+    # Pre-fill the OID/Enum cache to allow annotations to get enum values
+    for (oid, ctxt), result in results.iteritems():
+      resolve = self.mibcache.get(oid, None)
+      if resolve is None:
+        resolve = self.mibresolver.resolve(oid)
+        self.mibcache[oid] = resolve
+      if resolve is None:
+        logging.warning('Failed to look up OID %s, ignoring', oid)
+        continue
+
     # Calculate annotator map
     split_oid_map = collections.defaultdict(dict)
-    for oid, result in results.iteritems():
-      # We only support the last part of an OID as index for annotations
-      key, index = oid.rsplit('.', 1)
-      key += '.'
-      split_oid_map[key][index] = result.value
+    for (oid, ctxt), result in results.iteritems():
+      resolve = self.mibcache.get(oid, None)
+      if resolve is None:
+        continue
+      name, _ = resolve
+
+      _, index = name.split('.', 1)
+      key = oid[:-(len(index))]
+      split_oid_map[(key, ctxt)][index] = result.value
 
     annotated_results = {}
-    for oid, result in results.iteritems():
+    for (oid, ctxt), result in results.iteritems():
+      resolve = self.mibcache.get(oid, None)
+      if resolve is None:
+        continue
+
       # Record some stats on how long time it took to get this metric
       elapsed = (time.time() - target.timestamp) * 1000 * 1000
       labels = {}
-      vlan = ''
-
-      if '@' in oid:
-        oid, vlan = oid.split('@')
+      vlan = None
 
-      name = self.mibcache.get(oid, None)
-      if name is None:
-        name = self.mibresolver.resolve(oid)
-        self.mibcache[oid] = name
+      # TODO(bluecmd): If we support more contexts we need to be smarter here
+      if not ctxt is None:
+        vlan = ctxt
 
-      if name is None:
-        logging.warning('Failed to look up OID %s, ignoring', oid)
-        continue
+      name, enum = resolve
 
       if not '::' in name:
         logging.warning('OID %s resolved to %s (no MIB), ignoring', oid, name)
@@ -70,37 +91,112 @@ def do_result(self, run, target, results, stats):
       mib, part = name.split('::', 1)
       obj, index = part.split('.', 1) if '.' in part else (part, None)
 
-      labels = {'vlan': vlan}
-      labels.update(self.annotate(oid, annotation_map, split_oid_map, results))
-
-      annotated_results[oid] = actions.AnnotatedResultEntry(
+      labels = {}
+      if not vlan is None:
+        labels['vlan'] = vlan
+      labels.update(
+          self.annotate(
+            oid, index, ctxt, annotation_map, split_oid_map, results))
+
+      # Handle labelification
+      if oid[:-len(index)] in labelification:
+        # Skip empty strings or non-strings that are up for labelification
+        if result.value == '' or result.type not in self.LABEL_TYPES:
+          continue
+        labels['value'] = self.string_to_label_value(result.value)
+        labels['hex'] = binascii.hexlify(result.value)
+        result = snmp.ResultTuple('NaN', 'ANNOTATED')
+
+      # Do something almost like labelification for enums
+      if enum:
+        enum_value = enum.get(result.value, None)
+        if enum_value is None:
+          logging.warning('Got invalid enum value for %s (%s), not labling',
+              oid, result.value)
+        else:
+          labels['enum'] = enum_value
+
+      annotated_results[(oid, vlan)] = actions.AnnotatedResultEntry(
           result, mib, obj, index, labels)
 
     yield actions.AnnotatedResult(target, annotated_results, stats)
     logging.debug('Annotation completed for %d metrics for %s',
         len(annotated_results), target.host)
 
-  def annotate(self, oid, annotation_map, split_oid_map, results):
+  def annotate(self, oid, index, ctxt, annotation_map, split_oid_map, results):
     for key in annotation_map:
       if oid.startswith(key):
         break
     else:
       return {}
 
-    # We only support the last part of an OID as index for annotations
-    _, index = oid.rsplit('.', 1)
+    saved_index = index
     labels = {}
-    for label, annotation_oid in annotation_map[key].iteritems():
-      annotation_key = annotation_oid + '.'
-      part = split_oid_map.get(annotation_key, None)
-      if not part:
-        continue
-      value = part.get(index, None)
-      if not value:
+    for label, annotation_path in annotation_map[key].iteritems():
+      # Parse the annotation path
+      annotation_keys = [x.strip() + '.' for x in annotation_path.split('>')]
+
+      value = self.jump_to_value(
+          annotation_keys, oid, ctxt, index, split_oid_map, results)
+      if value is None:
         continue
+
       labels[label] = value.replace('"', '\\"')
     return labels
 
+  def jump_to_value(self, keys, oid, ctxt, index, split_oid_map, results):
+    # Jump across the path seperated like:
+    # OID.idx:value1
+    # OID2.value1:value2
+    # OID3.value3:final
+    # label=final
+    for key in keys:
+      use_value = key[0] == '$'
+      if use_value:
+        key = key[1:]
+
+      # Try to associate with context first
+      part = split_oid_map.get((key, ctxt), None)
+      if not part:
+        # Fall back to the global context
+        part = split_oid_map.get((key, None), None)
+        # Do not allow going back into context when you have jumped into
+        # the global one.
+        # TODO(bluecmd): I have no reason *not* to support this more than
+        # it feels like an odd behaviour and not something I would be
+        # expecting the software to do, so let's not do that unless we find
+        # a usecase in the future.
+        ctxt = None
+        if not part:
+          continue
+
+      # We either use the last index or the OID value, deterimed by
+      # use_value above.
+      if use_value:
+        index = results[(oid, ctxt)].value
+
+      oid = ''.join((key, index))
+      index = part.get(index, None)
+      if not index:
+        return None
+
+    value = index
+
+    # Try enum resolution
+    _, enum = self.mibcache[oid]
+    if enum:
+      enum_value = enum.get(value, None)
+      if enum_value is None:
+        logging.warning('Got invalid enum value for %s (%s), ignoring',
+            oid, value)
+        return None
+      value = enum_value
+    return value
+
+  def string_to_label_value(self, value):
+    value = ''.join(x for x in value.strip() if x in self.ALLOWED_CHARACTERS)
+    return value.strip()
+
 
 if __name__ == '__main__':
   annotator = stage.Stage(Annotator())