From b6af8fa0b0af5d787e9cc838df4d936f8f652606 Mon Sep 17 00:00:00 2001 From: lmineiro Date: Thu, 15 Oct 2015 17:23:03 +0200 Subject: [PATCH] Added request tracking (flowId) filter Closes #18 --- filters/flowid/filter.go | 48 +++++++++++++++++++++++++ filters/flowid/filter_test.go | 60 +++++++++++++++++++++++++++++++ filters/flowid/hashing.go | 38 ++++++++++++++++++++ filters/flowid/hashing_test.go | 64 ++++++++++++++++++++++++++++++++++ filters/flowid/readme.md | 36 +++++++++++++++++++ proxy/proxy.go | 2 +- 6 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 filters/flowid/filter.go create mode 100644 filters/flowid/filter_test.go create mode 100644 filters/flowid/hashing.go create mode 100644 filters/flowid/hashing_test.go create mode 100644 filters/flowid/readme.md diff --git a/filters/flowid/filter.go b/filters/flowid/filter.go new file mode 100644 index 0000000000..deea31f6aa --- /dev/null +++ b/filters/flowid/filter.go @@ -0,0 +1,48 @@ +package flowid + +import ( + "github.com/zalando/skipper/skipper" +) + +const ( + filterName = "FlowId" + flowIdHeaderName = "X-Flow-Id" +) + +type flowId struct { + id string + reuseExisting bool +} + +func New(id string, allowOverride bool) skipper.Filter { + return &flowId{id, allowOverride} +} + +func (this *flowId) Id() string { return this.id } + +func (this *flowId) Name() string { return filterName } + +func (this *flowId) Request(fc skipper.FilterContext) { + r := fc.Request() + var flowId string + + if this.reuseExisting { + flowId = r.Header.Get(flowIdHeaderName) + } + + var err error + if !isValid(flowId) { + flowId, err = newFlowId(defaultLen) + } + + if err == nil { + fc.Request().Header.Set(flowIdHeaderName, flowId) + } +} + +func (this *flowId) Response(skipper.FilterContext) {} + +func (this *flowId) MakeFilter(id string, fc skipper.FilterConfig) (skipper.Filter, error) { + reuseExisting, _ := fc[0].(bool) + return New(id, reuseExisting), nil +} diff --git a/filters/flowid/filter_test.go b/filters/flowid/filter_test.go new file mode 100644 index 0000000000..17612019f5 --- /dev/null +++ b/filters/flowid/filter_test.go @@ -0,0 +1,60 @@ +package flowid + +import ( + "github.com/zalando/skipper/mock" + "net/http" + "testing" +) + +const testFlowId = "FLOW-ID-FOR-TESTING" + +func TestNewFlowIdGeneration(t *testing.T) { + r, _ := http.NewRequest("GET", "http://example.org", nil) + f := New(filterName, true) + fc := &mock.FilterContext{FRequest: r} + f.Request(fc) + + flowId := fc.Request().Header.Get(flowIdHeaderName) + if !isValid(flowId) { + t.Errorf("'%s' is not a valid flow id", flowId) + } +} + +func TestFlowIdReuseExisting(t *testing.T) { + r, _ := http.NewRequest("GET", "http://example.org", nil) + f := New(filterName, true) + r.Header.Set(flowIdHeaderName, testFlowId) + fc := &mock.FilterContext{FRequest: r} + f.Request(fc) + + flowId := fc.Request().Header.Get(flowIdHeaderName) + if flowId != testFlowId { + t.Errorf("Got wrong flow id. Expected '%s' got '%s'", testFlowId, flowId) + } +} + +func TestFlowIdIgnoreReuseExisting(t *testing.T) { + r, _ := http.NewRequest("GET", "http://example.org", nil) + f := New(filterName, false) + r.Header.Set(flowIdHeaderName, testFlowId) + fc := &mock.FilterContext{FRequest: r} + f.Request(fc) + + flowId := fc.Request().Header.Get(flowIdHeaderName) + if flowId == testFlowId { + t.Errorf("Got wrong flow id. Expected a newly generated flowid but got the test flow id '%s'", flowId) + } +} + +func TestFlowIdRejectInvalidFlowId(t *testing.T) { + r, _ := http.NewRequest("GET", "http://example.org", nil) + f := New(filterName, true) + r.Header.Set(flowIdHeaderName, "[<>] (o) [<>]") + fc := &mock.FilterContext{FRequest: r} + f.Request(fc) + + flowId := fc.Request().Header.Get(flowIdHeaderName) + if flowId == "[<>] (o) [<>]" { + t.Errorf("Got wrong flow id. Expected a newly generated flowid but got the test flow id '%s'", flowId) + } +} diff --git a/filters/flowid/hashing.go b/filters/flowid/hashing.go new file mode 100644 index 0000000000..91fc1f79b4 --- /dev/null +++ b/filters/flowid/hashing.go @@ -0,0 +1,38 @@ +package flowid + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "regexp" + "fmt" +) + +const ( + defaultLen = 16 + maxLength = 255 + minLength = 8 +) + +var ( + ErrInvalidLen = errors.New(fmt.Sprintf("Invalid length. len must be >= %d and < %d", minLength, maxLength)) + flowIdRegex, _ = regexp.Compile(`^[\w+/=\-]+$`) +) + + +func newFlowId(len uint8) (string, error) { + if len < minLength || len%2 != 0 { + return "", ErrInvalidLen + } + + u := make([]byte, hex.DecodedLen(int(len))) + buf := make([]byte, len) + + rand.Read(u) + hex.Encode(buf, u) + return string(buf), nil +} + +func isValid(flowId string) bool { + return len(flowId) >= minLength && len(flowId) <= maxLength && flowIdRegex.MatchString(flowId) +} diff --git a/filters/flowid/hashing_test.go b/filters/flowid/hashing_test.go new file mode 100644 index 0000000000..6ae4795321 --- /dev/null +++ b/filters/flowid/hashing_test.go @@ -0,0 +1,64 @@ +package flowid + +import "testing" + +func TestFlowIdInvalidLength(t *testing.T) { + _, err := newFlowId(0) + if err == nil { + t.Errorf("Request for an invalid flow id length (0) succeeded and it shouldn't") + } + + _, err = newFlowId(15) + if err == nil { + t.Errorf("Request for an invalid flow id length (odd number) succeeded and it shouldn't") + } +} + +func TestFlowIdLength(t *testing.T) { + for expected := minLength; expected <= maxLength; expected += 2 { + flowId, err := newFlowId(uint8(expected)) + if err != nil { + t.Errorf("Failed to generate flowId with len %d", expected) + } + + l := len(flowId) + if l != expected { + t.Errorf("Got wrong flowId len. Requested %d, got %d (%s)", expected, l, flowId) + } + } +} + +func BenchmarkFlowIdLen8(b *testing.B) { + testFlowIdWithLen(b.N, 8) +} + +func BenchmarkFlowIdLen10(b *testing.B) { + testFlowIdWithLen(b.N, 10) +} + +func BenchmarkFlowIdLen12(b *testing.B) { + testFlowIdWithLen(b.N, 12) +} + +func BenchmarkFlowIdLen14(b *testing.B) { + testFlowIdWithLen(b.N, 14) +} + +func BenchmarkFlowIdLen16(b *testing.B) { + testFlowIdWithLen(b.N, 16) +} + +func BenchmarkFlowIdLen32(b *testing.B) { + testFlowIdWithLen(b.N, 32) +} + +func BenchmarkFlowIdLen64(b *testing.B) { + testFlowIdWithLen(b.N, 64) +} + +func testFlowIdWithLen(times int, len uint8) { + for i := 0; i < times; i++ { + newFlowId(len) + } + +} diff --git a/filters/flowid/readme.md b/filters/flowid/readme.md new file mode 100644 index 0000000000..07b6862c85 --- /dev/null +++ b/filters/flowid/readme.md @@ -0,0 +1,36 @@ +# Flow ID Filter + +Flow IDs let you correlate router logs for a given request against the upstream application logs for that same request. +If your upstream application makes other requests to other services it can provide the same Flow ID value so that all +of those logs can be correlated. + +## How it works +Skipper generates a unique Flow ID for every HTTP request that it receives. The Flow ID is then passed to your +upstream application as an HTTP header called X-Flow-Id. + +## Some benchmarks + +To decide upon which hashing mechanism to use we tested some versions of UUID v1 - v4 and some other implementations. +The results are as follow: + + Benchmark_uuidv1-4 5000000 281 ns/op + Benchmark_uuidv2-4 5000000 284 ns/op + Benchmark_uuidv3-4 2000000 605 ns/op + Benchmark_uuidv4-4 1000000 1903 ns/op + BenchmarkRndAndSprintf-4 500000 3312 ns/op + BenchmarkSha1-4 1000000 2188 ns/op + BenchmarkMd5-4 1000000 2076 ns/op + BenchmarkFnv-4 500000 2223 ns/op + +The current implementation just gets len / 2 (hex.DecodedLen) bytes from the PRNG and hex encodes them. +Its performance is only dependent on the length of the generated FlowId, according to the following benchmarks: + + BenchmarkFlowIdLen8-4 1000000 1157 ns/op + BenchmarkFlowIdLen10-4 1000000 1162 ns/op + BenchmarkFlowIdLen12-4 1000000 1163 ns/op + BenchmarkFlowIdLen14-4 1000000 1171 ns/op + BenchmarkFlowIdLen16-4 1000000 1180 ns/op + BenchmarkFlowIdLen32-4 1000000 1957 ns/op + BenchmarkFlowIdLen64-4 300000 3520 ns/op + +As you can see, starting at len = 32 (16 random bytes) the performance starts dropping dramatically. \ No newline at end of file diff --git a/proxy/proxy.go b/proxy/proxy.go index db28ffe4ed..10d6d5b096 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -272,4 +272,4 @@ func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { func addBranding(rs *http.Response) { rs.Header.Set("X-Powered-By", "Skipper") rs.Header.Set("Server", "Skipper") -} \ No newline at end of file +}