snmp support filters and batch custom tags

conf/input.snmp/snmp.toml

@@ -73,6 +73,8 @@ agents = [
 #oid = "RFC1213-MIB::sysName.0"
 #name = "source"
 #is_tag = true
+# filters = ["A:ifIndex:^2$","B:ifOperStatus:1", "C:ifDescr:^eno*"]
+# filters_expression = "(A && B) || C"
 
 #[[instances.table]]
 #oid = "IF-MIB::ifTable"

go.mod

@@ -115,6 +115,7 @@ require (
 )
 
 require (
+	github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
 	github.com/alibabacloud-go/cms-20190101/v8 v8.0.0
 	github.com/alibabacloud-go/cms-export-20211101/v2 v2.0.0
 	github.com/alibabacloud-go/darabonba-openapi/v2 v2.0.0

go.sum

@@ -101,6 +101,7 @@ github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 h1:KeNholpO2xKjgaa
 github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962/go.mod h1:kC29dT1vFpj7py2OvG1khBdQpo3kInWP+6QipLbdngo=
 github.com/HdrHistogram/hdrhistogram-go v1.1.0 h1:6dpdDPTRoo78HxAJ6T1HfMiKSnqhgRRqzCuPshRkQ7I=
 github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
+github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc=
 github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=

inputs/snmp/instances.go

@@ -33,6 +33,8 @@ type Instance struct {
 	connectionCache []snmpConnection
 
 	translator Translator
+
+	Mappings map[string]map[string]string `toml:"mappings"`
 }
 
 func (ins *Instance) Init() error {
@@ -90,13 +92,17 @@ func (ins *Instance) Gather(slist *types.SampleList) {
 				Fields: ins.Fields,
 			}
 			topTags := map[string]string{}
-			if err := ins.gatherTable(slist, gs, t, topTags, false); err != nil {
+			extraTags := map[string]string{}
+			if m, ok := ins.Mappings[agent]; ok {
+				extraTags = m
+			}
+			if err := ins.gatherTable(slist, gs, t, topTags, extraTags, false); err != nil {
 				log.Printf("agent %s ins: %s", agent, err)
 			}
 
 			// Now is the real tables.
 			for _, t := range ins.Tables {
-				if err := ins.gatherTable(slist, gs, t, topTags, true); err != nil {
+				if err := ins.gatherTable(slist, gs, t, topTags, extraTags, true); err != nil {
 					log.Printf("agent %s ins: gathering table %s error: %s", agent, t.Name, err)
 				}
 			}
@@ -105,7 +111,7 @@ func (ins *Instance) Gather(slist *types.SampleList) {
 	wg.Wait()
 }
 
-func (ins *Instance) gatherTable(slist *types.SampleList, gs snmpConnection, t Table, topTags map[string]string, walk bool) error {
+func (ins *Instance) gatherTable(slist *types.SampleList, gs snmpConnection, t Table, topTags, extraTags map[string]string, walk bool) error {
 	rt, err := t.Build(gs, walk, ins.translator)
 	if err != nil {
 		return err
@@ -132,6 +138,9 @@ func (ins *Instance) gatherTable(slist *types.SampleList, gs snmpConnection, t T
 		if _, ok := tr.Tags[ins.AgentHostTag]; !ok {
 			tr.Tags[ins.AgentHostTag] = gs.Host()
 		}
+		for k, v := range extraTags {
+			tr.Tags[k] = v
+		}
 		slist.PushSamples(prefix, tr.Fields, tr.Tags)
 	}
 

inputs/snmp/snmp.go

@@ -59,6 +59,8 @@ type ClientConfig struct {
 type Snmp struct {
 	config.PluginConfig
 	Instances []*Instance `toml:"instances"`
+
+	Mappings map[string]map[string]string `toml:"mappings"`
 }
 
 func init() {
@@ -78,6 +80,18 @@ func (s *Snmp) Name() string {
 func (s *Snmp) GetInstances() []inputs.Instance {
 	ret := make([]inputs.Instance, len(s.Instances))
 	for i := 0; i < len(s.Instances); i++ {
+		if len(s.Instances[i].Mappings) == 0 {
+			s.Instances[i].Mappings = s.Mappings
+		} else {
+			m := make(map[string]map[string]string)
+			for k, v := range s.Mappings {
+				m[k] = v
+			}
+			for k, v := range s.Instances[i].Mappings {
+				m[k] = v
+			}
+			s.Instances[i].Mappings = m
+		}
 		ret[i] = s.Instances[i]
 	}
 	return ret

inputs/snmp/table.go

@@ -7,16 +7,20 @@ import (
 	"log"
 	"math"
 	"net"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/Knetic/govaluate"
 	"github.com/gosnmp/gosnmp"
 )
 
 const (
-	indexFilterPrefix  = "ifIndex:"
-	ifTypeFilterPrefix = "ifType:"
+	commonFormat = 2
+	fullFormat   = 3
+
+	defaultExprPrefix = "expr"
 )
 
 // Table holds the configuration for an SNMP table.
@@ -42,7 +46,16 @@ type Table struct {
 
 	IncludeFilter []string `toml:"include_filter"`
 
-	idxFilter map[string]bool `toml:"-"`
+	Filters          []string `toml:"filters"`
+	FilterExpression string   `toml:"filters_expression"`
+
+	filterFormat int                `toml:"-"`
+	filtersMap   map[string]*Filter `toml:"-"`
+}
+
+type Filter struct {
+	key string
+	re  *regexp.Regexp
 }
 
 // Init builds & initializes the nested fields.
@@ -56,15 +69,53 @@ func (t *Table) Init(tr Translator) error {
 	if t.initialized {
 		return nil
 	}
+	if len(t.IncludeFilter) != 0 {
+		log.Println("W! include_filter is deprecated, please use filters instead")
+		t.Filters = append(t.Filters, t.IncludeFilter...)
+	}
 
-	t.idxFilter = make(map[string]bool)
+	if len(t.Filters) != 0 {
+		t.filtersMap = make(map[string]*Filter)
+		filterExpression := ""
+		for idx, filter := range t.Filters {
+			fields := strings.Split(filter, ":")
+			if t.filterFormat == 0 {
+				t.filterFormat = len(fields)
+			}
+			if t.filterFormat != len(fields) {
+				return fmt.Errorf("invalid filter format: %s, format must be {A}:{oid}:{match} or {oid}:{matrch}", filter)
+			}
+			switch t.filterFormat {
+			case commonFormat:
+				exprKey := fmt.Sprintf("%s%d", defaultExprPrefix, idx)
+				t.filtersMap[exprKey] = &Filter{
+					key: fields[0],
+					re:  regexp.MustCompile(fields[1]),
+				}
+				if t.FilterExpression == "" {
+					if filterExpression == "" {
+						filterExpression = exprKey
+					} else {
+						filterExpression = fmt.Sprintf("%s||%s", filterExpression, exprKey)
+					}
+				}
 
-	if len(t.IncludeFilter) != 0 {
-		for _, filter := range t.IncludeFilter {
-			if strings.HasPrefix(filter, indexFilterPrefix) {
-				t.idxFilter[strings.TrimPrefix(filter, indexFilterPrefix)] = true
+			case fullFormat:
+				t.filtersMap[fields[0]] = &Filter{
+					key: fields[1],
+					re:  regexp.MustCompile(fields[2]),
+				}
+
+				if t.FilterExpression == "" {
+					return fmt.Errorf("filters_expression cannot be empty when filters are defined as {A}:{oid}:{match}")
+				}
+			default:
+				return fmt.Errorf("invalid filter format: %s, format must be {A}:{oid}:{match} or {oid}:{matrch}", filter)
 			}
 		}
+		if t.FilterExpression == "" {
+			t.FilterExpression = filterExpression
+		}
 	}
 
 	if err := t.initBuild(tr); err != nil {
@@ -226,15 +277,6 @@ func (e *walkError) Unwrap() error {
 	return e.err
 }
 
-func (t Table) isIdxSelected(idx string) bool {
-	if len(t.idxFilter) == 0 {
-		return true
-	}
-
-	ok, val := t.idxFilter[idx]
-	return ok && val
-}
-
 // Build retrieves all the fields specified in the table and constructs the RTable.
 func (t Table) Build(gs snmpConnection, walk bool, tr Translator) (*RTable, error) {
 	rows := map[string]RTableRow{}
@@ -386,10 +428,6 @@ func (t Table) Build(gs snmpConnection, walk bool, tr Translator) (*RTable, erro
 				rtr.Tags["index"] = idx
 			}
 
-			if !t.isIdxSelected(idx) {
-				continue
-			}
-
 			// don't add an empty string
 			if vs, ok := v.(string); !ok || vs != "" {
 				if f.IsTag {
@@ -424,7 +462,53 @@ func (t Table) Build(gs snmpConnection, walk bool, tr Translator) (*RTable, erro
 		Time: time.Now(), // TODO record time at start
 		Rows: make([]RTableRow, 0, len(rows)),
 	}
+
+	var (
+		err  error
+		expr *govaluate.EvaluableExpression
+	)
+	if len(t.FilterExpression) != 0 {
+		expr, err = govaluate.NewEvaluableExpression(t.FilterExpression)
+		if err != nil {
+			log.Println("filters_expression err:", err)
+		}
+	}
 	for _, r := range rows {
+		if expr == nil {
+			rt.Rows = append(rt.Rows, r)
+			continue
+		}
+		params := make(map[string]interface{})
+		for rk, rv := range t.filtersMap {
+			for k, v := range r.Tags {
+				if strings.HasPrefix(k, rv.key) {
+					if rv.re.MatchString(v) {
+						params[rk] = true
+					} else {
+						params[rk] = false
+					}
+				}
+			}
+
+			for k, v := range r.Fields {
+				if strings.HasPrefix(k, rv.key) {
+					if rv.re.MatchString(fmt.Sprintf("%v", v)) {
+						params[rk] = true
+					} else {
+						params[rk] = false
+					}
+				}
+			}
+		}
+		if len(params) != 0 {
+			result, err := expr.Evaluate(params)
+			if err != nil {
+				log.Println("filter expression err:", err)
+			}
+			if match, ok := result.(bool); ok && !match {
+				continue
+			}
+		}
 		rt.Rows = append(rt.Rows, r)
 	}
 	return &rt, nil