SB-10000: Adding app- and pipeline-ids to server

examples/go.mod

@@ -3,22 +3,22 @@ module github.com/NYTimes/gizmo/examples
 replace github.com/NYTimes/gizmo => ../
 
 require (
-	cloud.google.com/go v0.38.0
+	cloud.google.com/go v0.56.0
 	github.com/NYTimes/gizmo v1.2.1
 	github.com/NYTimes/gziphandler v1.1.0
 	github.com/NYTimes/logrotate v1.0.0
 	github.com/NYTimes/sqliface v0.0.0-20180310195202-f8e6c8b78d37
 	github.com/go-kit/kit v0.9.0
-	github.com/go-sql-driver/mysql v1.4.1
-	github.com/golang/protobuf v1.3.2
+	github.com/go-sql-driver/mysql v1.5.0
+	github.com/golang/protobuf v1.3.5
 	github.com/gorilla/context v1.1.1
 	github.com/gorilla/websocket v1.4.0
-	github.com/kelseyhightower/envconfig v1.3.0
-	github.com/pkg/errors v0.8.1
-	github.com/sirupsen/logrus v1.3.0
-	golang.org/x/net v0.0.0-20190628185345-da137c7871d7
-	google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610
-	google.golang.org/grpc v1.22.0
+	github.com/kelseyhightower/envconfig v1.4.0
+	github.com/pkg/errors v0.9.1
+	github.com/sirupsen/logrus v1.5.0
+	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e
+	google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940
+	google.golang.org/grpc v1.28.0
 )
 
 go 1.13

go.sum

@@ -47,6 +47,7 @@ github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20191210083620-6965a1cf
 github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20191210083620-6965a1cfed68/go.mod h1:gMGUEe16aZh0QN941HgDjwrdjU4iTthPoz2/AtDRADE=
 github.com/NYTimes/logrotate v1.0.0 h1:6jFGbon6jOtpy3t3kwZZKS4Gdmf1C/Wv5J4ll4Xn5yk=
 github.com/NYTimes/logrotate v1.0.0/go.mod h1:GxNz1cSw1c6t99PXoZlw+nm90H6cyQyrH66pjVv7x88=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/Shopify/sarama v1.26.1 h1:3jnfWKD7gVwbB1KSy/lE0szA9duPuSFLViK0o/d3DgA=
 github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU=
 github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=
@@ -64,6 +65,7 @@ github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737 h1:rRISKWyXfVx
 github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60=
 github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -72,6 +74,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q=
 github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
@@ -199,6 +202,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ=
 github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
@@ -212,18 +216,23 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v0.9.4 h1:Y8E/JaaPbmFSW2V81Ab/d8yZFYQQGbni1b1jPcG9Y6A=
 github.com/prometheus/client_golang v0.9.4/go.mod h1:oCXIBxdI62A4cR6aTRJCgetEjecSIYzOEaeAn4iYEpM=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ=
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -232,6 +241,7 @@ github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q=
 github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -340,6 +350,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03i
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

server/ids.go

@@ -0,0 +1,99 @@
+package server
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"fmt"
+	"strings"
+
+	uuid "github.com/nu7hatch/gouuid"
+)
+
+// IDer generates a new ID
+type IDer interface {
+	ID() (string, error)
+}
+
+// AppUUID generates IDs based on v4 UUIDs
+type AppUUID struct {
+	formatter func(string) string
+}
+
+// NewAppUUID creates a new AppUUID generator
+func NewAppUUID(appname string) *AppUUID {
+	return &AppUUID{
+		formatter: formatter(appname),
+	}
+}
+
+// ID returns a random (v4) UUID using the prefix supplied to NewAppUUID()
+func (i *AppUUID) ID() (string, error) {
+	u, err := uuid.NewV4()
+	if err != nil {
+		return "", err
+	}
+
+	return i.formatter(u.String()), nil
+}
+
+// RandB64ID is an IDer that returns a random ID based on Base64 encoded
+// bytes.
+type RandB64ID struct {
+	formatter func(string) string
+}
+
+// NewRandB64ID creates a new RandB64ID generator
+func NewRandB64ID(appname string) *RandB64ID {
+	return &RandB64ID{
+		formatter: formatter(appname),
+	}
+}
+
+// ID returns a random B64 encoded ID using the appname prefix supplied to NewRandB64ID()
+func (i *RandB64ID) ID() (string, error) {
+	var buf [12]byte
+	var b64 string
+	for len(b64) < 10 {
+		_, _ = rand.Read(buf[:])
+		b64 = base64.StdEncoding.EncodeToString(buf[:])
+		b64 = strings.NewReplacer("+", "", "/", "").Replace(b64)
+	}
+
+	return i.formatter(b64), nil
+}
+
+// A PipelineID identifies requests all the way through some system by concatenating the
+// IDs generated by each individual app, each separated by a separator string, "|"
+type PipelineID struct {
+	AppIDer IDer
+}
+
+const fullIDerSep = "|"
+
+// ID generates a new pipeline ID based on the current ID and its app IDer
+func (f *PipelineID) ID(current string) (string, error) {
+	next, err := f.AppIDer.ID()
+	if err != nil {
+		return "", err
+	}
+
+	if len(current) == 0 {
+		return next, nil
+	}
+
+	return strings.Join([]string{current, next}, fullIDerSep), nil
+}
+
+func formatter(appname string) func(string) string {
+	if len(appname) > 0 {
+		// ...if an appname is supplied, use that instead of the default
+		format := appname + "-%s"
+		return func(u string) string {
+			return fmt.Sprintf(format, u)
+		}
+	}
+
+	return func(u string) string {
+		return u
+	}
+}

server/ids_test.go

@@ -0,0 +1,102 @@
+package server
+
+import (
+	"regexp"
+	"testing"
+)
+
+const (
+	b64Regex = "\\w{10,20}"
+	// thanks to https://adamscheller.com/regular-expressions/uuid-regex/ for saving me from writing this
+	uuidRegex = "([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}){1}"
+)
+
+func TestAppIDers(t *testing.T) {
+	tests := []struct {
+		desc, regex string
+		iDFunc func() IDer
+	} {
+		{"AppUUIDNoName", uuidRegex, func() IDer {return NewAppUUID("")}},
+		{"AppUUIDWithName", "blapp-"+uuidRegex, func() IDer {return NewAppUUID("blapp")}},
+		{"RandB64NoName", b64Regex, func() IDer {return NewRandB64ID("")}},
+		{"RandB64WithName", "fungus-"+b64Regex, func() IDer {return NewRandB64ID("fungus")}},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			ider := test.iDFunc()
+			id, err := ider.ID()
+			if err != nil {
+				t.Error("failed to get mock app ID", "err", err)
+			}
+
+			match, err := regexp.MatchString(test.regex, id)
+			if err != nil {
+				t.Error("err matching generated ID", "err", err)
+			}
+			if !match {
+				t.Error("ID did not match", "got", id)
+			}
+		})
+	}
+}
+
+type MockIDer struct {
+	sendThis string
+}
+
+func (m *MockIDer) ID() (string, error) {
+	return m.sendThis, nil
+}
+
+func TestPipelineID_ID(t *testing.T) {
+	first := "antleeb"
+	second := "babananab"
+	third := "canola"
+
+	mockIDer := &MockIDer{}
+	pipeIDer := &PipelineID{AppIDer: mockIDer}
+
+	mockIDer.sendThis = first
+	id, err := pipeIDer.ID("")
+	if err != nil {
+		t.Error("failed to get pipeline ID", "err", err)
+	}
+	if first != id {
+		t.Error("frist ID call did not match", "got", id, "expected", first)
+	}
+
+	mockIDer.sendThis = second
+	id, err = pipeIDer.ID(id)
+	exp := first + fullIDerSep + second
+	if err != nil {
+		t.Error("failed to get pipeline ID", "err", err)
+	}
+	if exp != id {
+		t.Error("second ID call did not match", "got", id, "expected", exp)
+	}
+
+	mockIDer.sendThis = third
+	id, err = pipeIDer.ID(id)
+	exp = first + fullIDerSep + second + fullIDerSep + third
+	if err != nil {
+		t.Error("failed to get pipeline ID", "err", err)
+	}
+	if exp != id {
+		t.Error("third ID call did not match", "got", id, "expected", exp)
+	}
+}
+
+func BenchmarkAppUUID_ID(b *testing.B) {
+	iDer := NewAppUUID("something")
+	for i := 0; i < b.N; i++ {
+		_, _ = iDer.ID()
+	}
+}
+
+func BenchmarkRandB64_ID(b *testing.B) {
+	iDer := NewRandB64ID("something")
+	for i := 0; i < b.N; i++ {
+		_, _ = iDer.ID()
+	}
+}

server/kit/kitserver_test.go

@@ -234,5 +234,5 @@ func (s *server) getCatByName(ctx context.Context, _ interface{}) (interface{},
 }
 
 func (s *server) error(ctx context.Context, _ interface{}) (interface{}, error) {
-	return nil, errors.New("doh!")
+	return nil, errors.New("doh")
 }

server/middleware.go

@@ -193,3 +193,42 @@ func WithCloseHandler(h ContextHandler) ContextHandler {
 		h.ServeHTTPContext(ctx, w, r)
 	})
 }
+
+// RequestIDHeader is the request ID key set to http headers
+const RequestIDHeader = "X-Request-Id"
+
+// AppIDHandler is a middleware that, if the $X-Request-ID$ header is not already
+// set on the request, generates an ID using the provided IDer then sets it on the
+// request and response
+func AppIDHandler(next http.Handler, idGen IDer) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var err error
+		var requestID = r.Header.Get(RequestIDHeader)
+		if requestID == "" {
+			requestID, err = idGen.ID()
+			if err != nil {
+				return
+			}
+			r.Header.Set(RequestIDHeader, requestID)
+			w.Header().Set(RequestIDHeader, requestID)
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+// PipelineIDHandler is a middleware that tracks all the IDs generated by the many apps
+// in a system. For each app, it concatenates an app-specific ID to the "pipeline ID",
+// which is then set to the request & response $X-Request-ID$
+func PipelineIDHandler(next http.Handler, pipeGen *PipelineID) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var err error
+		var requestID = r.Header.Get(RequestIDHeader)
+		requestID, err = pipeGen.ID(requestID)
+		if err != nil {
+			return
+		}
+		r.Header.Set(RequestIDHeader, requestID)
+		w.Header().Set(RequestIDHeader, requestID)
+		next.ServeHTTP(w, r)
+	})
+}