tmp add mib parser, edit readme, related changes

grongor · Jun 3, 2020 · 2a92a86 · 2a92a86
1 parent 94ba153
commit 2a92a86
Show file tree

Hide file tree

Showing 13 changed files with 420 additions and 119 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -7,6 +7,8 @@ jobs:
         steps:
             -   uses: actions/checkout@v2
             -   uses: actions/setup-go@v2
+            -   name: Install dependencies
+                run: sudo apt update && sudo apt install libsnmp-dev
             -   name: Build
                 run: go build ./cmd/snmp-proxy
             -   name: Upload binaries
@@ -21,8 +23,8 @@ jobs:
         steps:
             -   uses: actions/checkout@v2
             -   uses: actions/setup-go@v2
-            -   name: Install pip3
-                run: sudo apt install python3-pip
+            -   name: Install dependencies
+                run: sudo apt update && sudo apt install libsnmp-dev python3-pip snmp-mibs-downloader
             -   name: Install snmpsim
                 run: sudo pip3 install snmpsim
             -   name: Run tests
@@ -34,8 +36,8 @@ jobs:
         steps:
             -   uses: actions/checkout@v2
             -   uses: actions/setup-go@v2
-            -   name: Install pip3
-                run: sudo apt install python3-pip
+            -   name: Install dependencies
+                run: sudo apt update && sudo apt install libsnmp-dev python3-pip snmp-mibs-downloader
             -   name: Install snmpsim
                 run: sudo pip3 install snmpsim
             -   name: Download go-acc

diff --git a/Makefile b/Makefile
@@ -24,5 +24,9 @@ test:
 	timeout 300 go test --race ./...
 	timeout 300 go test --count 100 ./...
 
+.PHONY: clean
+clean:
+	rm -rf bin
+
 $(BIN)/golangci-lint:
 	curl --retry 5 -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.27.0
diff --git a/README.md b/README.md
@@ -41,7 +41,6 @@ The application provides a single HTTP endpoint `/snmp-proxy`, which accepts POS
         ".1.2.3",
         ".4.5.6"
     ],
-    "strip_parent_oid": false,
     "version": "2c",
     "timeout": 10,
     "retries": 3
@@ -78,27 +77,19 @@ Some errors are "standardized" and expected:
 
 The rest of the errors just describe what unexpected happened.
 
-The `stripParentOid` option works only with `RequestType` = `Walk`, and it simply removes the parent OID prefix of the
-OIDs that are returned during the walk. This usually greatly simplifies processing of the data. Example:
-```json
-{
-    "request_type": "walk",
-    "host": "192.168.1.1",
-    "community": "public",
-    "oids": [".1.3.6.1.2.1.31.1.1.1.15"],
-    "strip_parent_oid": true,
-    "version": "2c"
-}
-```
-```json
-{
-    "result": [
-        "1000001",
-        100000,
-        "1000003",
-        60000,
-        "1000005",
-        80000
-    ]
-}
-```
+MIBs
+----
+
+The application will try to find all installed MIBs and parse the DisplayHint information for OctetString types
+so that it knows how to format them. MIB parsing was inspired by
+[Prometheus SNMP exporter generator](https://github.com/prometheus/snmp_exporter/tree/master/generator). Thanks!
+
+In case that OID is of the type OctetString, and it isn't found in the MIBs, then we try to detect whether the string
+is printable (utf8 valid + all characters are printable). If it isn't, it's formatted as `AB C0 D5 D6...`.
+
+Metrics
+-------
+
+If you set Metrics.Listen address in the config, the application will expose Prometheus metrics on given address[:port],
+specifically on `GET /metrics`. These metrics contain all essential information about Go runtime, and a histogram
+of `POST /snmp-proxy` requests (count, durations).
diff --git a/cmd/snmp-proxy/config.go b/cmd/snmp-proxy/config.go
@@ -27,6 +27,9 @@ type Configuration struct {
 	Metrics struct {
 		Listen string
 	}
+	Snmp struct {
+		StrictMibParsing bool
+	}
 	Logger *zap.SugaredLogger
 }
 

diff --git a/cmd/snmp-proxy/snmp-proxy.go b/cmd/snmp-proxy/snmp-proxy.go
@@ -23,7 +23,15 @@ func main() {
 		metrics.Start(config.Logger, config.Metrics.Listen)
 	}
 
-	requester := snmpproxy.NewGosnmpRequester()
+	mibParser := snmpproxy.NewNetsnmpMibParser(config.Logger, config.Snmp.StrictMibParsing)
+
+	displayHints, err := mibParser.Parse()
+	if err != nil {
+		config.Logger.Fatalw("mib parser error: ", zap.Error(err))
+	}
+
+	mibDataProvider := snmpproxy.NewMibDataProvider(displayHints)
+	requester := snmpproxy.NewGosnmpRequester(mibDataProvider)
 
 	apiListener := snmpproxy.NewApiListener(requester, config.Logger, config.Api.Listen)
 	apiListener.Start()

diff --git a/go.mod b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/stretchr/testify v1.6.0
 	go.uber.org/zap v1.15.0
-	golang.org/x/sys v0.0.0-20200523222454-059865788121 // indirect
+	golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 // indirect
 	google.golang.org/protobuf v1.24.0 // indirect
 	gopkg.in/ini.v1 v1.57.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 // indirect

diff --git a/go.sum b/go.sum
@@ -92,8 +92,10 @@ github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -442,8 +444,8 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200523222454-059865788121 h1:rITEj+UZHYC927n8GT97eC3zrpzXdb/voyeOuVKS46o=
-golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y=
+golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=

diff --git a/snmpproxy/data.go b/snmpproxy/data.go
@@ -58,14 +58,13 @@ func (v *SnmpVersion) UnmarshalJSON(data []byte) error {
 }
 
 type Request struct {
-	RequestType    RequestType   `json:"request_type"`
-	Host           string        `json:"host"`
-	Community      string        `json:"community"`
-	StripParentOid bool          `json:"strip_parent_oid"`
-	Version        SnmpVersion   `json:"version"`
-	Retries        uint8         `json:"retries"`
-	Timeout        time.Duration `json:"timeout"`
-	Oids           []string      `json:"oids"`
+	RequestType RequestType   `json:"request_type"`
+	Host        string        `json:"host"`
+	Community   string        `json:"community"`
+	Version     SnmpVersion   `json:"version"`
+	Retries     uint8         `json:"retries"`
+	Timeout     time.Duration `json:"timeout"`
+	Oids        []string      `json:"oids"`
 }
 
 func (r *Request) UnmarshalJSON(data []byte) error {
@@ -97,6 +96,12 @@ func (r *Request) UnmarshalJSON(data []byte) error {
 		return fmt.Errorf("at least one OID must be provided")
 	}
 
+	for _, oid := range t.Oids {
+		if oid[0] != '.' {
+			return fmt.Errorf("all OIDs must begin with a dot")
+		}
+	}
+
 	if t.Version == 0 {
 		var v struct {
 			Version string `json:"version"`

diff --git a/snmpproxy/data_test.go b/snmpproxy/data_test.go
@@ -99,21 +99,19 @@ func TestUnmarshalRequest(t *testing.T) {
         ".1.2.3",
         ".4.5.6"
     ],
-    "strip_parent_oid": true,
     "version": "2c",
     "timeout": 10,
     "retries": 3
 }
 `,
 			expected: snmpproxy.Request{
-				RequestType:    snmpproxy.Get,
-				Host:           "localhost",
-				Community:      "public",
-				Oids:           []string{".1.2.3", ".4.5.6"},
-				StripParentOid: true,
-				Version:        snmpproxy.SnmpVersion(gosnmp.Version2c),
-				Timeout:        10 * time.Second,
-				Retries:        3,
+				RequestType: snmpproxy.Get,
+				Host:        "localhost",
+				Community:   "public",
+				Oids:        []string{".1.2.3", ".4.5.6"},
+				Version:     snmpproxy.SnmpVersion(gosnmp.Version2c),
+				Timeout:     10 * time.Second,
+				Retries:     3,
 			},
 			err: "",
 		},
@@ -131,14 +129,13 @@ func TestUnmarshalRequest(t *testing.T) {
 }
 `,
 			expected: snmpproxy.Request{
-				RequestType:    snmpproxy.Get,
-				Host:           "localhost",
-				Community:      "public",
-				Oids:           []string{".1.2.3", ".4.5.6"},
-				StripParentOid: false,
-				Version:        snmpproxy.SnmpVersion(gosnmp.Version2c),
-				Timeout:        2 * time.Second,
-				Retries:        0,
+				RequestType: snmpproxy.Get,
+				Host:        "localhost",
+				Community:   "public",
+				Oids:        []string{".1.2.3", ".4.5.6"},
+				Version:     snmpproxy.SnmpVersion(gosnmp.Version2c),
+				Timeout:     2 * time.Second,
+				Retries:     0,
 			},
 			err: "",
 		},
@@ -184,6 +181,22 @@ func TestUnmarshalRequest(t *testing.T) {
 			expected: snmpproxy.Request{},
 			err:      "at least one OID must be provided",
 		},
+		{
+			name: "OID without dot prefix",
+			raw: `
+{
+    "request_type": "get",
+    "host": "localhost",
+    "version": "2c",
+	"oids": [
+		".1.2.3",
+		"4.5.6"
+	]
+}
+`,
+			expected: snmpproxy.Request{},
+			err:      "all OIDs must begin with a dot",
+		},
 		{
 			name: "missing version",
 			raw: `