Merge branch 'master' into configured_clock_skew

.travis.yml

@@ -1,4 +1,4 @@
-sudo: false
+sudo: required
 language: go
 
 notifications:
@@ -14,7 +14,7 @@ addons:
     sources:
     - sourceline: 'ppa:opencpu/jq'
     packages:
-    - python3-dev
+    - python3.5
     - python3-pip
     - libluajit-5.1-dev
     - libjq-dev

api.go

@@ -113,7 +113,16 @@ func applyPoliciesAndSave(keyName string, session *user.SessionState, spec *APIS
 	mw := BaseMiddleware{
 		Spec: spec,
 	}
-	return mw.ApplyPolicies(keyName, session)
+	if err := mw.ApplyPolicies(keyName, session); err != nil {
+		return err
+	}
+
+	lifetime := session.Lifetime(spec.SessionLifetime)
+	if err := spec.SessionManager.UpdateSession(keyName, session, lifetime, false); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func doAddOrUpdate(keyName string, newSession *user.SessionState, dontReset bool) error {
@@ -235,7 +244,7 @@ func handleAddOrUpdate(keyName string, r *http.Request) (interface{}, int) {
 		// Only if it's NEW
 		switch r.Method {
 		case "POST":
-			keyName = newSession.OrgID + keyName
+			keyName = generateToken(newSession.OrgID, keyName)
 			// It's a create, so lets hash the password
 			setSessionPassword(&newSession)
 		case "PUT":
@@ -307,10 +316,27 @@ func handleGetDetail(sessionKey, apiID string, byHash bool) (interface{}, int) {
 	var session user.SessionState
 	var ok bool
 	session, ok = sessionManager.SessionDetail(sessionKey, byHash)
+
 	if !ok {
 		return apiError("Key not found"), http.StatusNotFound
 	}
 
+	quotaKey := QuotaKeyPrefix + storage.HashKey(sessionKey)
+	if byHash {
+		quotaKey = QuotaKeyPrefix + sessionKey
+	}
+
+	if usedQuota, err := sessionManager.Store().GetKey(quotaKey); err == nil {
+		qInt, _ := strconv.Atoi(usedQuota)
+		remaining := session.QuotaMax - int64(qInt)
+
+		if remaining < 0 {
+			session.QuotaRemaining = 0
+		} else {
+			session.QuotaRemaining = remaining
+		}
+	}
+
 	log.WithFields(logrus.Fields{
 		"prefix": "api",
 		"key":    obfuscateKey(sessionKey),
@@ -900,11 +926,7 @@ func createKeyHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if newSession.Certificate != "" {
-		newKey = newSession.OrgID + newSession.Certificate
-
-		if strings.HasPrefix(newSession.Certificate, newSession.OrgID) {
-			newKey = newSession.Certificate
-		}
+		newKey = generateToken(newSession.OrgID, newSession.Certificate)
 	}
 
 	newSession.LastUpdated = strconv.Itoa(int(time.Now().Unix()))
@@ -1513,25 +1535,50 @@ func ctxGetSession(r *http.Request) *user.SessionState {
 	return nil
 }
 
-func ctxSetSession(r *http.Request, s *user.SessionState) {
+func ctxSetSession(r *http.Request, s *user.SessionState, token string, scheduleUpdate bool) {
 	if s == nil {
 		panic("setting a nil context SessionData")
 	}
-	setCtxValue(r, SessionData, s)
+
+	if token == "" {
+		token = ctxGetAuthToken(r)
+	}
+
+	if s.KeyHashEmpty() {
+		s.SetKeyHash(storage.HashKey(token))
+	}
+
+	ctx := r.Context()
+	ctx = context.WithValue(ctx, SessionData, s)
+	ctx = context.WithValue(ctx, AuthToken, token)
+
+	if scheduleUpdate {
+		ctx = context.WithValue(ctx, UpdateSession, true)
+	}
+
+	setContext(r, ctx)
 }
 
-func ctxGetAuthToken(r *http.Request) string {
-	if v := r.Context().Value(AuthHeaderValue); v != nil {
-		return v.(string)
+func ctxScheduleSessionUpdate(r *http.Request) {
+	setCtxValue(r, UpdateSession, true)
+}
+
+func ctxDisableSessionUpdate(r *http.Request) {
+	setCtxValue(r, UpdateSession, false)
+}
+
+func ctxSessionUpdateScheduled(r *http.Request) bool {
+	if v := r.Context().Value(UpdateSession); v != nil {
+		return v.(bool)
 	}
-	return ""
+	return false
 }
 
-func ctxSetAuthToken(r *http.Request, t string) {
-	if t == "" {
-		panic("setting a nil context AuthHeaderValue")
+func ctxGetAuthToken(r *http.Request) string {
+	if v := r.Context().Value(AuthToken); v != nil {
+		return v.(string)
 	}
-	setCtxValue(r, AuthHeaderValue, t)
+	return ""
 }
 
 func ctxGetTrackedPath(r *http.Request) string {

api_definition.go

@@ -693,10 +693,11 @@ func (a APIDefinitionLoader) compileURLRewritesPathSpec(paths []apidef.URLRewrit
 	urlSpec := []URLSpec{}
 
 	for _, stringSpec := range paths {
+		curStringSpec := stringSpec
 		newSpec := URLSpec{}
-		a.generateRegex(stringSpec.Path, &newSpec, stat)
+		a.generateRegex(curStringSpec.Path, &newSpec, stat)
 		// Extend with method actions
-		newSpec.URLRewrite = &stringSpec
+		newSpec.URLRewrite = &curStringSpec
 
 		urlSpec = append(urlSpec, newSpec)
 	}
@@ -1048,6 +1049,10 @@ func (a *APISpec) CheckSpecMatchesStatus(r *http.Request, rxPaths []URLSpec, mod
 }
 
 func (a *APISpec) getVersionFromRequest(r *http.Request) string {
+	if a.VersionData.NotVersioned {
+		return ""
+	}
+
 	switch a.VersionDefinition.Location {
 	case "header":
 		return r.Header.Get(a.VersionDefinition.Key)
@@ -1056,7 +1061,9 @@ func (a *APISpec) getVersionFromRequest(r *http.Request) string {
 		return r.URL.Query().Get(a.VersionDefinition.Key)
 
 	case "url":
-		uPath := r.URL.Path[len(a.Proxy.ListenPath):]
+		uPath := strings.TrimPrefix(r.URL.Path, a.Proxy.ListenPath)
+		uPath = strings.TrimPrefix(uPath, "/"+a.Slug)
+
 		// First non-empty part of the path is the version ID
 		for _, part := range strings.Split(uPath, "/") {
 			if part != "" {

api_definition_test.go

@@ -26,6 +26,60 @@ func createDefinitionFromString(defStr string) *APISpec {
 	return spec
 }
 
+func TestURLRewrites(t *testing.T) {
+	ts := newTykTestServer()
+	defer ts.Close()
+
+	t.Run("Extended Paths with url_rewrites", func(t *testing.T) {
+		buildAndLoadAPI(func(spec *APISpec) {
+			updateAPIVersion(spec, "v1", func(v *apidef.VersionInfo) {
+				json.Unmarshal([]byte(`[
+						{
+                            "path": "/rewrite1",
+                            "method": "GET",
+                            "match_pattern": "/rewrite1",
+                            "rewrite_to": "",
+                            "triggers": [
+                                {
+                                    "on": "all",
+                                    "options": {
+                                        "header_matches": {},
+                                        "query_val_matches": {
+                                            "show_env": {
+                                                "match_rx": "1"
+                                            }
+                                        },
+                                        "path_part_matches": {},
+                                        "session_meta_matches": {},
+                                        "payload_matches": {
+                                            "match_rx": ""
+                                        }
+                                    },
+                                    "rewrite_to": "/get?show_env=2"
+                                }
+                            ],
+                            "MatchRegexp": null
+                        },
+                        {
+                            "path": "/rewrite",
+                            "method": "GET",
+                            "match_pattern": "/rewrite",
+                            "rewrite_to": "/get?just_rewrite",
+                            "triggers": [],
+                            "MatchRegexp": null
+						}
+				]`), &v.ExtendedPaths.URLRewrite)
+			})
+			spec.Proxy.ListenPath = "/"
+		})
+
+		ts.Run(t, []test.TestCase{
+			{Path: "/rewrite1?show_env=1", Code: 200, BodyMatch: `"URI":"/get?show_env=2"`},
+			{Path: "/rewrite", Code: 200, BodyMatch: `"URI":"/get?just_rewrite"`},
+		}...)
+	})
+}
+
 func TestWhitelist(t *testing.T) {
 	ts := newTykTestServer()
 	defer ts.Close()

api_loader.go

@@ -6,10 +6,12 @@ import (
 	"net/url"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/gorilla/mux"
 	"github.com/justinas/alice"
+	"github.com/pmylund/go-cache"
 
 	"github.com/TykTechnologies/tyk/apidef"
 	"github.com/TykTechnologies/tyk/config"
@@ -356,7 +358,7 @@ func processSpec(spec *APISpec, apisByListen map[string]int,
 			mainLog.WithField("api_name", spec.Name).Info("Checking security policy: OAuth")
 		}
 
-		if mwAppendEnabled(&authArray, &BasicAuthKeyIsValid{baseMid}) {
+		if mwAppendEnabled(&authArray, &BasicAuthKeyIsValid{baseMid, cache.New(60*time.Second, 60*time.Minute)}) {
 			mainLog.WithField("api_name", spec.Name).Info("Checking security policy: Basic")
 		}
 

api_test.go

@@ -172,8 +172,6 @@ func TestKeyHandler(t *testing.T) {
 	}
 	withBadPolicyJSON, _ := json.Marshal(withBadPolicy)
 
-	knownKey := createSession()
-
 	t.Run("Create key", func(t *testing.T) {
 		ts.Run(t, []test.TestCase{
 			// Master keys should be disabled by default
@@ -215,6 +213,8 @@ func TestKeyHandler(t *testing.T) {
 		}...)
 	})
 
+	knownKey := createSession()
+
 	t.Run("Get key", func(t *testing.T) {
 		ts.Run(t, []test.TestCase{
 			{Method: "GET", Path: "/tyk/keys/unknown", AdminAuth: true, Code: 404},
@@ -257,9 +257,32 @@ func TestHashKeyHandler(t *testing.T) {
 	// enable hashed keys listing
 	globalConf.EnableHashedKeysListing = true
 	config.SetGlobal(globalConf)
-
 	defer resetTestConfig()
 
+	hashTests := []struct {
+		hashFunction     string
+		expectedHashSize int
+		desc             string
+	}{
+		{"", 8, " Legacy tokens, fallback to murmur32"},
+		{storage.HashMurmur32, 8, ""},
+		{storage.HashMurmur64, 16, ""},
+		{storage.HashMurmur128, 32, ""},
+		{storage.HashSha256, 64, ""},
+		{"wrong", 16, " Should fallback to murmur64 if wrong alg"},
+	}
+
+	for _, tc := range hashTests {
+		globalConf.HashKeyFunction = tc.hashFunction
+		config.SetGlobal(globalConf)
+
+		t.Run(fmt.Sprintf("%sHash fn: %s", tc.desc, tc.hashFunction), func(t *testing.T) {
+			testHashKeyHandlerHelper(t, tc.expectedHashSize)
+		})
+	}
+}
+
+func testHashKeyHandlerHelper(t *testing.T, expectedHashSize int) {
 	ts := newTykTestServer()
 	defer ts.Close()
 
@@ -271,9 +294,13 @@ func TestHashKeyHandler(t *testing.T) {
 	}}
 	withAccessJSON, _ := json.Marshal(withAccess)
 
-	myKey := "my_key_id"
+	myKey := generateToken("", "")
 	myKeyHash := storage.HashKey(myKey)
 
+	if len(myKeyHash) != expectedHashSize {
+		t.Errorf("Expected hash size: %d, got %d. Hash: %s. Key: %s", expectedHashSize, len(myKeyHash), myKeyHash, myKey)
+	}
+
 	t.Run("Create, get and delete key with key hashing", func(t *testing.T) {
 		ts.Run(t, []test.TestCase{
 			// create key
@@ -759,7 +786,7 @@ func TestContextSession(t *testing.T) {
 	if ctxGetSession(r) != nil {
 		t.Fatal("expected ctxGetSession to return nil")
 	}
-	ctxSetSession(r, &user.SessionState{})
+	ctxSetSession(r, &user.SessionState{}, "", false)
 	if ctxGetSession(r) == nil {
 		t.Fatal("expected ctxGetSession to return non-nil")
 	}
@@ -768,7 +795,7 @@ func TestContextSession(t *testing.T) {
 			t.Fatal("expected ctxSetSession of zero val to panic")
 		}
 	}()
-	ctxSetSession(r, nil)
+	ctxSetSession(r, nil, "", false)
 }
 
 func TestApiLoaderLongestPathFirst(t *testing.T) {

apidef/api_definitions.go

@@ -329,8 +329,12 @@ type APIDefinition struct {
 		AllowedAuthorizeTypes  []osin.AuthorizeRequestType `bson:"allowed_authorize_types" json:"allowed_authorize_types"`
 		AuthorizeLoginRedirect string                      `bson:"auth_login_redirect" json:"auth_login_redirect"`
 	} `bson:"oauth_meta" json:"oauth_meta"`
-	Auth                       Auth                 `bson:"auth" json:"auth"`
-	UseBasicAuth               bool                 `bson:"use_basic_auth" json:"use_basic_auth"`
+	Auth         Auth `bson:"auth" json:"auth"`
+	UseBasicAuth bool `bson:"use_basic_auth" json:"use_basic_auth"`
+	BasicAuth    struct {
+		DisableCaching bool `bson:"disable_caching" json:"disable_caching"`
+		CacheTTL       int  `bson:"cache_ttl" json:"cache_ttl"`
+	} `bson:"basic_auth" json:"basic_auth"`
 	UseMutualTLSAuth           bool                 `bson:"use_mutual_tls_auth" json:"use_mutual_tls_auth"`
 	ClientCertificates         []string             `bson:"client_certificates" json:"client_certificates"`
 	UpstreamCertificates       map[string]string    `bson:"upstream_certificates" json:"upstream_certificates"`
@@ -346,6 +350,7 @@ type APIDefinition struct {
 	JWTIssuedAtValidationSkew  uint64               `bson:"jwt_issued_at_validation_skew" json:"jwt_issued_at_validation_skew"`
 	JWTExpiresAtValidationSkew uint64               `bson:"jwt_expires_at_validation_skew" json:"jwt_expires_at_validation_skew"`
 	JWTNotBeforeValidationSkew uint64               `bson:"jwt_not_before_validation_skew" json:"jwt_not_before_validation_skew"`
+	JWTSkipKid                 bool                 `bson:"jwt_skip_kid" json:"jwt_skip_kid"`
 	NotificationsDetails       NotificationsManager `bson:"notifications" json:"notifications"`
 	EnableSignatureChecking    bool                 `bson:"enable_signature_checking" json:"enable_signature_checking"`
 	HmacAllowedClockSkew       float64              `bson:"hmac_allowed_clock_skew" json:"hmac_allowed_clock_skew"`