Sec ops caching

README.md

@@ -223,7 +223,8 @@ See if we can still get away with an invalid token...
 You should get back a fat 403 with a message along the lines of:
 
     PERMISSION_DENIED:
-    orionadapter-handler.handler.istio-system:unauthorized: invalid JWT data
+    orionadapter-handler.handler.istio-system:
+    unauthorized: invalid consumer JWT data
 
 Like I said earlier, the adapter verifies the JWT you send as part of the IDSA-Header is valid---see
 `deployment/sample_operator_cfg.yaml`. What happens if we send a valid
@@ -476,6 +477,57 @@ and we're ready to try our luck
 If everything went according to plan, you're looking at a `200`
 response on your terminal with the JSON body returned by Orion :-)
 
+##### Caching
+
+Oh my, saving the best for last. So for a [whole bunch of
+reasons](https://github.com/orchestracities/boost/issues/9),
+we wound up with our own caching solution instead of the Mixer's.
+(Lucky us, fun times.) Fingers crossed, our caching should be decent
+enough for most scenarios but time will tell. If you dig deep into the
+adapter logs, you should be able to see that it caches calls to DAPS
+and AuthZ. A DAPS ID token gets cached for the amount of time specified
+in the JWT `exp` field. Likewise, an AuthZ decision gets cached until
+the consumer JWT expires.
+
+For example, if you look at the logs to see what happened while the
+adapter serviced the last call we made to Orion, you should be able
+to spot a message similar to
+
+    AuthZ
+      Request: &{
+        Roles: [role0 role1 role2 role3]
+        ResourceID: b3a4a7d2-ce61-471f-b05d-fb82452ae686
+        ResourcePath: /v2
+        FiwareService: service
+        Action: GET
+      }
+      Decision: Permit
+      Caching: decision not saved to cache
+
+Uh, what's that "decision **not** saved to cache"? Why?! Well when
+we whipped together that `MY_FAT_JWT` earlier we did it a bit too
+quickly. Without an `exp` field, you can't expect calls to get
+cached, can you? If besides the roles, you also add an `exp` field
+with a Unix timestamp in the future, reexport `HEADER_VALUE` and
+repeat the call, the log message should read
+
+    AuthZ
+      Request: ...
+      Decision: Permit
+      Caching: decision saved to cache
+
+Ah, AuthZ's decision got cached. Try the same call again now and
+then you should see a cache hit
+
+    AuthZ
+      Request: ...
+      Decision: Permit (cached)
+
+Just in case you're wondering. Caching takes into account all call
+inputs so if you change the JWT (or HTTP method/path, etc.) just
+slightly the adapter will ask again his old friend AuthZ for permission.
+
+
 ##### Cleaning up
 
 **TODO**

go.mod

@@ -3,6 +3,7 @@ module github.com/orchestracities/boost
 go 1.13
 
 require (
+	github.com/dgraph-io/ristretto v0.0.2
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/gogo/protobuf v1.3.0
 	github.com/google/uuid v1.1.1

go.sum

@@ -61,6 +61,7 @@ github.com/cactus/go-statsd-client v3.1.1+incompatible/go.mod h1:cMRcwZDklk7hXp+
 github.com/cenkalti/backoff v2.0.0+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
@@ -91,8 +92,11 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
 github.com/dchest/siphash v1.1.0/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4=
 github.com/denisenkom/go-mssqldb v0.0.0-20190423183735-731ef375ac02/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM=
+github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po=
+github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
@@ -450,6 +454,8 @@ github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRci
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=

orionadapter/cache/bootstrap.go

@@ -0,0 +1,18 @@
+package cache
+
+// NOTE. Concurrency. Treat this var as a const, you can't touch this :-)
+var cached *store
+
+// Init gets the cache infrastructure ready for use.
+// It must be the first call to this module and must be done in the main
+// function before any other thread can possibly call the functions this
+// module exports.
+func Init() error {
+	s, err := newStore()
+	if err != nil {
+		return err
+	}
+
+	cached = s
+	return nil
+}

orionadapter/cache/keys.go

@@ -0,0 +1,32 @@
+package cache
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/orchestracities/boost/orionadapter/sec/authz"
+	"github.com/orchestracities/boost/orionadapter/sec/consumer"
+)
+
+const adapterConfigKey = "adapterConfigKey"
+const dapsIDTokenKey = "dapsIDTokenKey"
+
+func authZCallKey(idsConsumerHeader string, callParams *authz.Request) (
+	key string, consumerJWT string, err error) {
+	consumerJWT, err = consumer.ReadToken(idsConsumerHeader)
+	if err != nil {
+		return "", "", err
+	}
+	if callParams == nil {
+		return "", "", errors.New("authZCallKey: nil callParams")
+	}
+
+	return makeKey(consumerJWT, callParams), consumerJWT, nil
+}
+
+// NOTE. Fast hashing. Ristretto's default KeyToHash function converts a
+// string to a []byte and then uses xxHash to get the actual hash which
+// is great. So we only need to convert our structs to string.
+func makeKey(consumerJWT string, callParams *authz.Request) string {
+	return fmt.Sprintf("(%v, %v)", consumerJWT, callParams)
+}

orionadapter/cache/keys_test.go

@@ -0,0 +1,70 @@
+package cache
+
+import (
+	"encoding/base64"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/orchestracities/boost/orionadapter/sec/authz"
+)
+
+const clientJSONPayloadTemplate string = `{
+	"securityToken": {
+		"tokenValue": "%s"
+	}
+}`
+
+func toB64(headerValue string) string {
+	v := []byte(headerValue)
+	return base64.StdEncoding.EncodeToString(v)
+}
+
+func clientJSONPayload(token string) string {
+	headerValue := fmt.Sprintf(clientJSONPayloadTemplate, token)
+	return toB64(headerValue)
+}
+
+func callParams(roles []string, path string, service string,
+	action string) *authz.Request {
+	return &authz.Request{
+		Roles:         roles,
+		ResourceID:    "b3a4a7d2-ce61-471f-b05d-fb82452ae686",
+		ResourcePath:  path,
+		FiwareService: service,
+		Action:        action,
+	}
+}
+
+func TestAuthZCallKeyContent(t *testing.T) {
+	header := clientJSONPayload("my.fat.jwt")
+	params := callParams([]string{"r1"}, "/v2", "svc", "GET")
+
+	got, _, err := authZCallKey(header, params)
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+
+	for _, v := range []string{"my.fat.jwt", "r1", "/v2", "GET"} {
+		if !strings.Contains(got, v) {
+			t.Errorf("%v not in %v", v, got)
+		}
+	}
+}
+
+func TestAuthZCallKeyErrorOnNilCallParams(t *testing.T) {
+	header := clientJSONPayload("my.fat.jwt")
+	got, _, err := authZCallKey(header, nil)
+	if err == nil {
+		t.Errorf("want error; got: %v", got)
+	}
+}
+
+func TestAuthZCallKeyErrorOnInvalidHeader(t *testing.T) {
+	header := ""
+	params := callParams([]string{"r1"}, "/v2", "svc", "GET")
+	got, _, err := authZCallKey(header, params)
+	if err == nil {
+		t.Errorf("want error; got: %v", got)
+	}
+}

orionadapter/cache/lookup.go

@@ -0,0 +1,56 @@
+package cache
+
+import (
+	"github.com/orchestracities/boost/orionadapter/codegen/config"
+	"github.com/orchestracities/boost/orionadapter/sec/authz"
+)
+
+func asBool(value interface{}, found bool) (bool, bool) {
+	if found {
+		if converted, ok := value.(bool); ok {
+			return converted, true
+		}
+	}
+	return false, false
+}
+
+func asString(value interface{}, found bool) (string, bool) {
+	if found {
+		if converted, ok := value.(string); ok {
+			return converted, true
+		}
+	}
+	return "", false
+}
+
+func asParams(value interface{}, found bool) (*config.Params, bool) {
+	if found {
+		if converted, ok := value.(*config.Params); ok {
+			return converted, true
+		}
+	}
+	return nil, false
+}
+
+// LookupAdapterConfig gets any cached adapter config. Use the found flag
+// to tell if the lookup was successful.
+func LookupAdapterConfig() (params *config.Params, found bool) {
+	return asParams(cached.Config().lookup(adapterConfigKey))
+}
+
+// LookupDapsIDToken gets any cached DAPS ID token. Use the found flag
+// to tell if the lookup was successful.
+func LookupDapsIDToken() (token string, found bool) {
+	return asString(cached.Daps().lookup(dapsIDTokenKey))
+}
+
+// LookupAuthZDecision gets any cached AuthZ decision for the specified
+// call parameters. Use the found flag to tell if the lookup was successful.
+func LookupAuthZDecision(idsConsumerHeader string, callParams *authz.Request) (
+	authorized bool, found bool) {
+	key, _, err := authZCallKey(idsConsumerHeader, callParams)
+	if err != nil {
+		return false, false
+	}
+	return asBool(cached.AuthZ().lookup(key))
+}

orionadapter/cache/put.go

@@ -0,0 +1,38 @@
+package cache
+
+import (
+	"github.com/orchestracities/boost/orionadapter/codegen/config"
+	"github.com/orchestracities/boost/orionadapter/sec/authz"
+	"github.com/orchestracities/boost/orionadapter/sec/jwt"
+)
+
+// PutAdapterConfig caches adapter config. Use the ok flag to tell if the
+// operation was successful.
+func PutAdapterConfig(params *config.Params) (ok bool) {
+	if params == nil {
+		return false
+	}
+	return cached.Config().keep(adapterConfigKey, params)
+}
+
+// PutDapsIDToken caches a DAPS ID token. Use the ok flag to tell if the
+// operation was successful.
+func PutDapsIDToken(jwtData string) (ok bool) {
+	if len(jwtData) == 0 {
+		return false
+	}
+	ttl := jwt.FromRaw(jwtData).ExpiresIn()
+	return cached.Daps().put(dapsIDTokenKey, jwtData, 1, ttl)
+}
+
+// PutAuthZDecision caches an AuthZ decision. Use the ok flag to tell if the
+// operation was successful.
+func PutAuthZDecision(idsConsumerHeader string, callParams *authz.Request,
+	authorized bool) (ok bool) {
+	key, jwtData, err := authZCallKey(idsConsumerHeader, callParams)
+	if err != nil {
+		return false
+	}
+	ttl := jwt.FromRaw(jwtData).ExpiresIn()
+	return cached.AuthZ().put(key, authorized, 1, ttl)
+}