Basic auth now can extract credentials from body (#2186)

You can specify regexps for username and body.
Regexp should contain one match group, which points to either username or password.

Example:
```
"basic_auth": {
    "extract_from_body": true,
    "body_user_regexp": "<User>(.*)</User>",
    "body_password_regexp": "<Password>(.*)</Password>"
}
```

Fix #1855

api.go

@@ -333,7 +333,7 @@ func handleGetDetail(sessionKey, apiID string, byHash bool) (interface{}, int) {
 		log.WithFields(logrus.Fields{
 			"prefix": "api",
 			"key":    obfuscateKey(sessionKey),
-			"error": err,
+			"error":  err,
 			"status": "ok",
 		}).Info("Can't retrieve key quota")
 	}

api_loader.go

@@ -364,7 +364,7 @@ func processSpec(spec *APISpec, apisByListen map[string]int,
 			mainLog.WithField("api_name", spec.Name).Info("Checking security policy: OAuth")
 		}
 
-		if mwAppendEnabled(&authArray, &BasicAuthKeyIsValid{baseMid, cache.New(60*time.Second, 60*time.Minute)}) {
+		if mwAppendEnabled(&authArray, &BasicAuthKeyIsValid{baseMid, cache.New(60*time.Second, 60*time.Minute), nil, nil}) {
 			mainLog.WithField("api_name", spec.Name).Info("Checking security policy: Basic")
 		}
 

apidef/api_definitions.go

@@ -333,8 +333,11 @@ type APIDefinition struct {
 	Auth         Auth `bson:"auth" json:"auth"`
 	UseBasicAuth bool `bson:"use_basic_auth" json:"use_basic_auth"`
 	BasicAuth    struct {
-		DisableCaching bool `bson:"disable_caching" json:"disable_caching"`
-		CacheTTL       int  `bson:"cache_ttl" json:"cache_ttl"`
+		DisableCaching     bool   `bson:"disable_caching" json:"disable_caching"`
+		CacheTTL           int    `bson:"cache_ttl" json:"cache_ttl"`
+		ExtractFromBody    bool   `bson:"extract_from_body" json:"extract_from_body"`
+		BodyUserRegexp     string `bson:"body_user_regexp" json:"body_user_regexp"`
+		BodyPasswordRegexp string `bson:"body_password_regexp" json:"body_password_regexp"`
 	} `bson:"basic_auth" json:"basic_auth"`
 	UseMutualTLSAuth           bool                 `bson:"use_mutual_tls_auth" json:"use_mutual_tls_auth"`
 	ClientCertificates         []string             `bson:"client_certificates" json:"client_certificates"`

coprocess_helpers.go

@@ -88,11 +88,11 @@ func TykSessionState(session *coprocess.SessionState) *user.SessionState {
 		MetaData:                metadata,
 		Monitor:                 monitor,
 		EnableDetailedRecording: session.EnableDetailedRecording,
-		Tags:                session.Tags,
-		Alias:               session.Alias,
-		LastUpdated:         session.LastUpdated,
-		IdExtractorDeadline: session.IdExtractorDeadline,
-		SessionLifetime:     session.SessionLifetime,
+		Tags:                    session.Tags,
+		Alias:                   session.Alias,
+		LastUpdated:             session.LastUpdated,
+		IdExtractorDeadline:     session.IdExtractorDeadline,
+		SessionLifetime:         session.SessionLifetime,
 	}
 }
 
@@ -174,11 +174,11 @@ func ProtoSessionState(session *user.SessionState) *coprocess.SessionState {
 		Monitor:                 monitor,
 		Metadata:                metadata,
 		EnableDetailedRecording: session.EnableDetailedRecording,
-		Tags:                session.Tags,
-		Alias:               session.Alias,
-		LastUpdated:         session.LastUpdated,
-		IdExtractorDeadline: session.IdExtractorDeadline,
-		SessionLifetime:     session.SessionLifetime,
+		Tags:                    session.Tags,
+		Alias:                   session.Alias,
+		LastUpdated:             session.LastUpdated,
+		IdExtractorDeadline:     session.IdExtractorDeadline,
+		SessionLifetime:         session.SessionLifetime,
 	}
 }
 

gateway_test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -92,7 +93,7 @@ func TestMain(m *testing.M) {
 	if err := config.WriteDefault("", &globalConf); err != nil {
 		panic(err)
 	}
-	globalConf.Storage.Database = 1
+	globalConf.Storage.Database = rand.Intn(15)
 	var err error
 	globalConf.AppPath, err = ioutil.TempDir("", "tyk-test-")
 	if err != nil {
@@ -1348,11 +1349,13 @@ func TestRateLimitForAPIAndRateLimitAndQuotaCheck(t *testing.T) {
 		s.Rate = 1
 		s.Per = 60
 	})
+	defer FallbackKeySesionManager.RemoveSession(sess1token, false)
 
 	sess2token := createSession(func(s *user.SessionState) {
 		s.Rate = 1
 		s.Per = 60
 	})
+	defer FallbackKeySesionManager.RemoveSession(sess2token, false)
 
 	ts.Run(t, []test.TestCase{
 		{Headers: map[string]string{"Authorization": sess1token}, Code: http.StatusOK, Path: "/"},

multiauth_test.go

@@ -78,7 +78,7 @@ func getMultiAuthStandardAndBasicAuthChain(spec *APISpec) http.Handler {
 	chain := alice.New(mwList(
 		&IPWhiteListMiddleware{baseMid},
 		&IPBlackListMiddleware{BaseMiddleware: baseMid},
-		&BasicAuthKeyIsValid{baseMid, cache.New(60*time.Second, 60*time.Minute)},
+		&BasicAuthKeyIsValid{baseMid, cache.New(60*time.Second, 60*time.Minute), nil, nil},
 		&AuthKey{baseMid},
 		&VersionCheck{BaseMiddleware: baseMid},
 		&KeyExpired{baseMid},

mw_basic_auth.go

@@ -1,21 +1,23 @@
 package main
 
 import (
+	"bytes"
 	"encoding/base64"
 	"errors"
+	"io/ioutil"
 	"net/http"
 	"strings"
 	"time"
 
-	"github.com/TykTechnologies/tyk/config"
-	"github.com/TykTechnologies/tyk/storage"
-
 	"github.com/Sirupsen/logrus"
-	"github.com/pmylund/go-cache"
-	"github.com/TykTechnologies/murmur3"
+	cache "github.com/pmylund/go-cache"
 	"golang.org/x/crypto/bcrypt"
 
+	"github.com/TykTechnologies/murmur3"
 	"github.com/TykTechnologies/tyk/apidef"
+	"github.com/TykTechnologies/tyk/config"
+	"github.com/TykTechnologies/tyk/regexp"
+	"github.com/TykTechnologies/tyk/storage"
 	"github.com/TykTechnologies/tyk/user"
 )
 
@@ -25,6 +27,9 @@ const defaultBasicAuthTTL = time.Duration(60) * time.Second
 type BasicAuthKeyIsValid struct {
 	BaseMiddleware
 	cache *cache.Cache
+
+	bodyUserRegexp     *regexp.Regexp
+	bodyPasswordRegexp *regexp.Regexp
 }
 
 func (k *BasicAuthKeyIsValid) Name() string {
@@ -33,7 +38,32 @@ func (k *BasicAuthKeyIsValid) Name() string {
 
 // EnabledForSpec checks if UseBasicAuth is set in the API definition.
 func (k *BasicAuthKeyIsValid) EnabledForSpec() bool {
-	return k.Spec.UseBasicAuth
+	if !k.Spec.UseBasicAuth {
+		return false
+	}
+
+	var err error
+
+	if k.Spec.BasicAuth.ExtractFromBody {
+		if k.Spec.BasicAuth.BodyUserRegexp == "" || k.Spec.BasicAuth.BodyPasswordRegexp == "" {
+			log.Error("Basic Auth configured to extract credentials from body, but regexps are empty")
+			return false
+		}
+
+		k.bodyUserRegexp, err = regexp.Compile(k.Spec.BasicAuth.BodyUserRegexp)
+		if err != nil {
+			log.WithError(err).Error("Invalid user body regexp")
+			return false
+		}
+
+		k.bodyPasswordRegexp, err = regexp.Compile(k.Spec.BasicAuth.BodyPasswordRegexp)
+		if err != nil {
+			log.WithError(err).Error("Invalid user password regexp")
+			return false
+		}
+	}
+
+	return true
 }
 
 // requestForBasicAuth sends error code and message along with WWW-Authenticate header to client.
@@ -44,52 +74,105 @@ func (k *BasicAuthKeyIsValid) requestForBasicAuth(w http.ResponseWriter, msg str
 	return errors.New(msg), http.StatusUnauthorized
 }
 
-// ProcessRequest will run any checks on the request on the way through the system, return an error to have the chain fail
-func (k *BasicAuthKeyIsValid) ProcessRequest(w http.ResponseWriter, r *http.Request, _ interface{}) (error, int) {
+func (k *BasicAuthKeyIsValid) basicAuthHeaderCredentials(w http.ResponseWriter, r *http.Request) (username, password string, err error, code int) {
 	token := r.Header.Get("Authorization")
 	logEntry := getLogEntryForRequest(r, token, nil)
 	if token == "" {
 		// No header value, fail
 		logEntry.Info("Attempted access with malformed header, no auth header found.")
 
-		return k.requestForBasicAuth(w, "Authorization field missing")
+		err, code = k.requestForBasicAuth(w, "Authorization field missing")
+		return
 	}
 
 	bits := strings.Split(token, " ")
 	if len(bits) != 2 {
 		// Header malformed
 		logEntry.Info("Attempted access with malformed header, header not in basic auth format.")
 
-		return errors.New("Attempted access with malformed header, header not in basic auth format"), http.StatusBadRequest
+		err, code = errors.New("Attempted access with malformed header, header not in basic auth format"), http.StatusBadRequest
+		return
 	}
 
 	// Decode the username:password string
 	authvaluesStr, err := base64.StdEncoding.DecodeString(bits[1])
 	if err != nil {
 		logEntry.Info("Base64 Decoding failed of basic auth data: ", err)
 
-		return errors.New("Attempted access with malformed header, auth data not encoded correctly"), http.StatusBadRequest
+		err, code = errors.New("Attempted access with malformed header, auth data not encoded correctly"), http.StatusBadRequest
+		return
 	}
 
 	authValues := strings.Split(string(authvaluesStr), ":")
 	if len(authValues) != 2 {
 		// Header malformed
 		logEntry.Info("Attempted access with malformed header, values not in basic auth format.")
 
-		return errors.New("Attempted access with malformed header, values not in basic auth format"), http.StatusBadRequest
+		err, code = errors.New("Attempted access with malformed header, values not in basic auth format"), http.StatusBadRequest
+		return
+	}
+
+	username, password = authValues[0], authValues[1]
+	return
+}
+
+func (k *BasicAuthKeyIsValid) basicAuthBodyCredentials(w http.ResponseWriter, r *http.Request) (username, password string, err error, code int) {
+	body, _ := ioutil.ReadAll(r.Body)
+	r.Body = ioutil.NopCloser(bytes.NewReader(body))
+
+	userMatch := k.bodyUserRegexp.FindAllSubmatch(body, 1)
+	if len(userMatch) == 0 {
+		err, code = errors.New("Body do not contain username"), http.StatusBadRequest
+		return
+	}
+
+	if len(userMatch[0]) < 2 {
+		err, code = errors.New("username should be inside regexp match group"), http.StatusBadRequest
+		return
+	}
+
+	passMatch := k.bodyPasswordRegexp.FindAllSubmatch(body, 1)
+
+	if len(passMatch) == 0 {
+		err, code = errors.New("Body do not contain password"), http.StatusBadRequest
+		return
+	}
+
+	if len(passMatch[0]) < 2 {
+		err, code = errors.New("password should be inside regexp match group"), http.StatusBadRequest
+		return
+	}
+
+	username, password = string(userMatch[0][1]), string(passMatch[0][1])
+
+	return username, password, nil, 0
+}
+
+// ProcessRequest will run any checks on the request on the way through the system, return an error to have the chain fail
+func (k *BasicAuthKeyIsValid) ProcessRequest(w http.ResponseWriter, r *http.Request, _ interface{}) (error, int) {
+	username, password, err, code := k.basicAuthHeaderCredentials(w, r)
+	token := r.Header.Get("Authorization")
+	if err != nil {
+		if k.Spec.BasicAuth.ExtractFromBody {
+			username, password, err, code = k.basicAuthBodyCredentials(w, r)
+		}
+
+		if err != nil {
+			return err, code
+		}
 	}
 
 	// Check if API key valid
-	keyName := generateToken(k.Spec.OrgID, authValues[0])
-	logEntry = getLogEntryForRequest(r, keyName, nil)
+	keyName := generateToken(k.Spec.OrgID, username)
+	logEntry := getLogEntryForRequest(r, keyName, nil)
 	session, keyExists := k.CheckSessionAndIdentityForValidKey(keyName, r)
 	if !keyExists {
 		if config.Global().HashKeyFunction == "" {
 			logEntry.Warning("Attempted access with non-existent user.")
 			return k.handleAuthFail(w, r, token)
 		} else { // check for key with legacy format "org_id" + "user_name"
 			logEntry.Info("Could not find user, falling back to legacy format key.")
-			legacyKeyName := strings.TrimPrefix(authValues[0], k.Spec.OrgID)
+			legacyKeyName := strings.TrimPrefix(username, k.Spec.OrgID)
 			keyName, _ = storage.GenerateToken(k.Spec.OrgID, legacyKeyName, "")
 			session, keyExists = k.CheckSessionAndIdentityForValidKey(keyName, r)
 			if !keyExists {
@@ -101,14 +184,12 @@ func (k *BasicAuthKeyIsValid) ProcessRequest(w http.ResponseWriter, r *http.Requ
 
 	switch session.BasicAuthData.Hash {
 	case user.HashBCrypt:
-
-		if err := k.compareHashAndPassword(session.BasicAuthData.Password, authValues[1], logEntry); err != nil {
+		if err := k.compareHashAndPassword(session.BasicAuthData.Password, password, logEntry); err != nil {
 			logEntry.Warn("Attempted access with existing user, failed password check.")
 			return k.handleAuthFail(w, r, token)
 		}
 	case user.HashPlainText:
-		if session.BasicAuthData.Password != authValues[1] {
-
+		if session.BasicAuthData.Password != password {
 			logEntry.Warn("Attempted access with existing user, failed password check.")
 			return k.handleAuthFail(w, r, token)
 		}
@@ -154,7 +235,6 @@ func (k *BasicAuthKeyIsValid) compareHashAndPassword(hash string, password strin
 	hashBytes := []byte(hash)
 
 	if !cacheEnabled {
-
 		logEntry.Debug("cache disabled")
 		return bcrypt.CompareHashAndPassword(hashBytes, passwordBytes)
 	}

mw_basic_auth_test.go

@@ -59,6 +59,41 @@ func TestBasicAuth(t *testing.T) {
 	}...)
 }
 
+func TestBasicAuthFromBody(t *testing.T) {
+	ts := newTykTestServer()
+	defer ts.Close()
+
+	session := createStandardSession()
+	session.BasicAuthData.Password = "password"
+	session.AccessRights = map[string]user.AccessDefinition{"test": {APIID: "test", Versions: []string{"v1"}}}
+	session.OrgID = "default"
+
+	buildAndLoadAPI(func(spec *APISpec) {
+		spec.UseBasicAuth = true
+		spec.BasicAuth.ExtractFromBody = true
+		spec.BasicAuth.BodyUserRegexp = `<User>(.*)</User>`
+		spec.BasicAuth.BodyPasswordRegexp = `<Password>(.*)</Password>`
+		spec.UseKeylessAccess = false
+		spec.Proxy.ListenPath = "/"
+		spec.OrgID = "default"
+	})
+
+	validPassword := `<User>user</User><Password>password</Password>`
+	wrongPassword := `<User>user</User><Password>wrong</Password>`
+	withoutPassword := `<User>user</User>`
+	malformed := `<User>User>`
+
+	ts.Run(t, []test.TestCase{
+		// Create base auth based key
+		{Method: "POST", Path: "/tyk/keys/defaultuser", Data: session, AdminAuth: true, Code: 200},
+		{Method: "POST", Path: "/", Code: 400, BodyMatch: `Body do not contain username`},
+		{Method: "POST", Path: "/", Data: validPassword, Code: 200},
+		{Method: "POST", Path: "/", Data: wrongPassword, Code: 401},
+		{Method: "POST", Path: "/", Data: withoutPassword, Code: 400, BodyMatch: `Body do not contain password`},
+		{Method: "GET", Path: "/", Data: malformed, Code: 400, BodyMatch: `Body do not contain username`},
+	}...)
+}
+
 func TestBasicAuthLegacyWithHashFunc(t *testing.T) {
 	globalConf := config.Global()
 

mw_openid.go

@@ -200,7 +200,7 @@ func (k *OpenIDMW) ProcessRequest(w http.ResponseWriter, r *http.Request, _ inte
 	// apply new policy to session if any and update session
 	session.SetPolicies(policyID)
 	if err := k.ApplyPolicies(sessionID, &session); err != nil {
-	    log.WithError(err).Error("Could not apply new policy from OIDC client to session")
+		log.WithError(err).Error("Could not apply new policy from OIDC client to session")
 		return errors.New("Key not authorized: could not apply new policy"), http.StatusForbidden
 	}
 

rpc_test.go

@@ -9,9 +9,9 @@ import (
 	"github.com/gorilla/mux"
 
 	"github.com/TykTechnologies/tyk/cli"
-	"github.com/lonelycode/gorpc"
 	"github.com/TykTechnologies/tyk/config"
 	"github.com/TykTechnologies/tyk/test"
+	"github.com/lonelycode/gorpc"
 )
 
 func startRPCMock(dispatcher *gorpc.Dispatcher) *gorpc.Server {