diff --git a/.travis.yml b/.travis.yml index 58339af6a67..d1b6845b484 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -sudo: false +sudo: required language: go notifications: @@ -14,7 +14,7 @@ addons: sources: - sourceline: 'ppa:opencpu/jq' packages: - - python3-dev + - python3.5 - python3-pip - libluajit-5.1-dev - libjq-dev diff --git a/api.go b/api.go index b552738de84..4c8543a295e 100644 --- a/api.go +++ b/api.go @@ -113,7 +113,16 @@ func applyPoliciesAndSave(keyName string, session *user.SessionState, spec *APIS mw := BaseMiddleware{ Spec: spec, } - return mw.ApplyPolicies(keyName, session) + if err := mw.ApplyPolicies(keyName, session); err != nil { + return err + } + + lifetime := session.Lifetime(spec.SessionLifetime) + if err := spec.SessionManager.UpdateSession(keyName, session, lifetime, false); err != nil { + return err + } + + return nil } func doAddOrUpdate(keyName string, newSession *user.SessionState, dontReset bool) error { @@ -235,7 +244,7 @@ func handleAddOrUpdate(keyName string, r *http.Request) (interface{}, int) { // Only if it's NEW switch r.Method { case "POST": - keyName = newSession.OrgID + keyName + keyName = generateToken(newSession.OrgID, keyName) // It's a create, so lets hash the password setSessionPassword(&newSession) case "PUT": @@ -307,10 +316,27 @@ func handleGetDetail(sessionKey, apiID string, byHash bool) (interface{}, int) { var session user.SessionState var ok bool session, ok = sessionManager.SessionDetail(sessionKey, byHash) + if !ok { return apiError("Key not found"), http.StatusNotFound } + quotaKey := QuotaKeyPrefix + storage.HashKey(sessionKey) + if byHash { + quotaKey = QuotaKeyPrefix + sessionKey + } + + if usedQuota, err := sessionManager.Store().GetKey(quotaKey); err == nil { + qInt, _ := strconv.Atoi(usedQuota) + remaining := session.QuotaMax - int64(qInt) + + if remaining < 0 { + session.QuotaRemaining = 0 + } else { + session.QuotaRemaining = remaining + } + } + log.WithFields(logrus.Fields{ "prefix": "api", "key": obfuscateKey(sessionKey), @@ -900,11 +926,7 @@ func createKeyHandler(w http.ResponseWriter, r *http.Request) { } if newSession.Certificate != "" { - newKey = newSession.OrgID + newSession.Certificate - - if strings.HasPrefix(newSession.Certificate, newSession.OrgID) { - newKey = newSession.Certificate - } + newKey = generateToken(newSession.OrgID, newSession.Certificate) } newSession.LastUpdated = strconv.Itoa(int(time.Now().Unix())) @@ -1513,25 +1535,50 @@ func ctxGetSession(r *http.Request) *user.SessionState { return nil } -func ctxSetSession(r *http.Request, s *user.SessionState) { +func ctxSetSession(r *http.Request, s *user.SessionState, token string, scheduleUpdate bool) { if s == nil { panic("setting a nil context SessionData") } - setCtxValue(r, SessionData, s) + + if token == "" { + token = ctxGetAuthToken(r) + } + + if s.KeyHashEmpty() { + s.SetKeyHash(storage.HashKey(token)) + } + + ctx := r.Context() + ctx = context.WithValue(ctx, SessionData, s) + ctx = context.WithValue(ctx, AuthToken, token) + + if scheduleUpdate { + ctx = context.WithValue(ctx, UpdateSession, true) + } + + setContext(r, ctx) } -func ctxGetAuthToken(r *http.Request) string { - if v := r.Context().Value(AuthHeaderValue); v != nil { - return v.(string) +func ctxScheduleSessionUpdate(r *http.Request) { + setCtxValue(r, UpdateSession, true) +} + +func ctxDisableSessionUpdate(r *http.Request) { + setCtxValue(r, UpdateSession, false) +} + +func ctxSessionUpdateScheduled(r *http.Request) bool { + if v := r.Context().Value(UpdateSession); v != nil { + return v.(bool) } - return "" + return false } -func ctxSetAuthToken(r *http.Request, t string) { - if t == "" { - panic("setting a nil context AuthHeaderValue") +func ctxGetAuthToken(r *http.Request) string { + if v := r.Context().Value(AuthToken); v != nil { + return v.(string) } - setCtxValue(r, AuthHeaderValue, t) + return "" } func ctxGetTrackedPath(r *http.Request) string { diff --git a/api_definition.go b/api_definition.go index fb6ea002629..c5a2b666366 100644 --- a/api_definition.go +++ b/api_definition.go @@ -693,10 +693,11 @@ func (a APIDefinitionLoader) compileURLRewritesPathSpec(paths []apidef.URLRewrit urlSpec := []URLSpec{} for _, stringSpec := range paths { + curStringSpec := stringSpec newSpec := URLSpec{} - a.generateRegex(stringSpec.Path, &newSpec, stat) + a.generateRegex(curStringSpec.Path, &newSpec, stat) // Extend with method actions - newSpec.URLRewrite = &stringSpec + newSpec.URLRewrite = &curStringSpec urlSpec = append(urlSpec, newSpec) } @@ -1048,6 +1049,10 @@ func (a *APISpec) CheckSpecMatchesStatus(r *http.Request, rxPaths []URLSpec, mod } func (a *APISpec) getVersionFromRequest(r *http.Request) string { + if a.VersionData.NotVersioned { + return "" + } + switch a.VersionDefinition.Location { case "header": return r.Header.Get(a.VersionDefinition.Key) @@ -1056,7 +1061,9 @@ func (a *APISpec) getVersionFromRequest(r *http.Request) string { return r.URL.Query().Get(a.VersionDefinition.Key) case "url": - uPath := r.URL.Path[len(a.Proxy.ListenPath):] + uPath := strings.TrimPrefix(r.URL.Path, a.Proxy.ListenPath) + uPath = strings.TrimPrefix(uPath, "/"+a.Slug) + // First non-empty part of the path is the version ID for _, part := range strings.Split(uPath, "/") { if part != "" { diff --git a/api_definition_test.go b/api_definition_test.go index 5aab2efa3ad..f72982cf26c 100644 --- a/api_definition_test.go +++ b/api_definition_test.go @@ -26,6 +26,60 @@ func createDefinitionFromString(defStr string) *APISpec { return spec } +func TestURLRewrites(t *testing.T) { + ts := newTykTestServer() + defer ts.Close() + + t.Run("Extended Paths with url_rewrites", func(t *testing.T) { + buildAndLoadAPI(func(spec *APISpec) { + updateAPIVersion(spec, "v1", func(v *apidef.VersionInfo) { + json.Unmarshal([]byte(`[ + { + "path": "/rewrite1", + "method": "GET", + "match_pattern": "/rewrite1", + "rewrite_to": "", + "triggers": [ + { + "on": "all", + "options": { + "header_matches": {}, + "query_val_matches": { + "show_env": { + "match_rx": "1" + } + }, + "path_part_matches": {}, + "session_meta_matches": {}, + "payload_matches": { + "match_rx": "" + } + }, + "rewrite_to": "/get?show_env=2" + } + ], + "MatchRegexp": null + }, + { + "path": "/rewrite", + "method": "GET", + "match_pattern": "/rewrite", + "rewrite_to": "/get?just_rewrite", + "triggers": [], + "MatchRegexp": null + } + ]`), &v.ExtendedPaths.URLRewrite) + }) + spec.Proxy.ListenPath = "/" + }) + + ts.Run(t, []test.TestCase{ + {Path: "/rewrite1?show_env=1", Code: 200, BodyMatch: `"URI":"/get?show_env=2"`}, + {Path: "/rewrite", Code: 200, BodyMatch: `"URI":"/get?just_rewrite"`}, + }...) + }) +} + func TestWhitelist(t *testing.T) { ts := newTykTestServer() defer ts.Close() diff --git a/api_loader.go b/api_loader.go index 243f6df7cb3..0739b5c4da2 100644 --- a/api_loader.go +++ b/api_loader.go @@ -6,10 +6,12 @@ import ( "net/url" "sort" "strings" + "time" "github.com/Sirupsen/logrus" "github.com/gorilla/mux" "github.com/justinas/alice" + "github.com/pmylund/go-cache" "github.com/TykTechnologies/tyk/apidef" "github.com/TykTechnologies/tyk/config" @@ -356,7 +358,7 @@ func processSpec(spec *APISpec, apisByListen map[string]int, mainLog.WithField("api_name", spec.Name).Info("Checking security policy: OAuth") } - if mwAppendEnabled(&authArray, &BasicAuthKeyIsValid{baseMid}) { + if mwAppendEnabled(&authArray, &BasicAuthKeyIsValid{baseMid, cache.New(60*time.Second, 60*time.Minute)}) { mainLog.WithField("api_name", spec.Name).Info("Checking security policy: Basic") } diff --git a/api_test.go b/api_test.go index 58c15ed37fd..d1bfb1f85d2 100644 --- a/api_test.go +++ b/api_test.go @@ -172,8 +172,6 @@ func TestKeyHandler(t *testing.T) { } withBadPolicyJSON, _ := json.Marshal(withBadPolicy) - knownKey := createSession() - t.Run("Create key", func(t *testing.T) { ts.Run(t, []test.TestCase{ // Master keys should be disabled by default @@ -215,6 +213,8 @@ func TestKeyHandler(t *testing.T) { }...) }) + knownKey := createSession() + t.Run("Get key", func(t *testing.T) { ts.Run(t, []test.TestCase{ {Method: "GET", Path: "/tyk/keys/unknown", AdminAuth: true, Code: 404}, @@ -257,9 +257,32 @@ func TestHashKeyHandler(t *testing.T) { // enable hashed keys listing globalConf.EnableHashedKeysListing = true config.SetGlobal(globalConf) - defer resetTestConfig() + hashTests := []struct { + hashFunction string + expectedHashSize int + desc string + }{ + {"", 8, " Legacy tokens, fallback to murmur32"}, + {storage.HashMurmur32, 8, ""}, + {storage.HashMurmur64, 16, ""}, + {storage.HashMurmur128, 32, ""}, + {storage.HashSha256, 64, ""}, + {"wrong", 16, " Should fallback to murmur64 if wrong alg"}, + } + + for _, tc := range hashTests { + globalConf.HashKeyFunction = tc.hashFunction + config.SetGlobal(globalConf) + + t.Run(fmt.Sprintf("%sHash fn: %s", tc.desc, tc.hashFunction), func(t *testing.T) { + testHashKeyHandlerHelper(t, tc.expectedHashSize) + }) + } +} + +func testHashKeyHandlerHelper(t *testing.T, expectedHashSize int) { ts := newTykTestServer() defer ts.Close() @@ -271,9 +294,13 @@ func TestHashKeyHandler(t *testing.T) { }} withAccessJSON, _ := json.Marshal(withAccess) - myKey := "my_key_id" + myKey := generateToken("", "") myKeyHash := storage.HashKey(myKey) + if len(myKeyHash) != expectedHashSize { + t.Errorf("Expected hash size: %d, got %d. Hash: %s. Key: %s", expectedHashSize, len(myKeyHash), myKeyHash, myKey) + } + t.Run("Create, get and delete key with key hashing", func(t *testing.T) { ts.Run(t, []test.TestCase{ // create key @@ -759,7 +786,7 @@ func TestContextSession(t *testing.T) { if ctxGetSession(r) != nil { t.Fatal("expected ctxGetSession to return nil") } - ctxSetSession(r, &user.SessionState{}) + ctxSetSession(r, &user.SessionState{}, "", false) if ctxGetSession(r) == nil { t.Fatal("expected ctxGetSession to return non-nil") } @@ -768,7 +795,7 @@ func TestContextSession(t *testing.T) { t.Fatal("expected ctxSetSession of zero val to panic") } }() - ctxSetSession(r, nil) + ctxSetSession(r, nil, "", false) } func TestApiLoaderLongestPathFirst(t *testing.T) { diff --git a/apidef/api_definitions.go b/apidef/api_definitions.go index 06331893e3f..d1374398703 100644 --- a/apidef/api_definitions.go +++ b/apidef/api_definitions.go @@ -329,8 +329,12 @@ type APIDefinition struct { AllowedAuthorizeTypes []osin.AuthorizeRequestType `bson:"allowed_authorize_types" json:"allowed_authorize_types"` AuthorizeLoginRedirect string `bson:"auth_login_redirect" json:"auth_login_redirect"` } `bson:"oauth_meta" json:"oauth_meta"` - Auth Auth `bson:"auth" json:"auth"` - UseBasicAuth bool `bson:"use_basic_auth" json:"use_basic_auth"` + Auth Auth `bson:"auth" json:"auth"` + UseBasicAuth bool `bson:"use_basic_auth" json:"use_basic_auth"` + BasicAuth struct { + DisableCaching bool `bson:"disable_caching" json:"disable_caching"` + CacheTTL int `bson:"cache_ttl" json:"cache_ttl"` + } `bson:"basic_auth" json:"basic_auth"` UseMutualTLSAuth bool `bson:"use_mutual_tls_auth" json:"use_mutual_tls_auth"` ClientCertificates []string `bson:"client_certificates" json:"client_certificates"` UpstreamCertificates map[string]string `bson:"upstream_certificates" json:"upstream_certificates"` @@ -346,6 +350,7 @@ type APIDefinition struct { JWTIssuedAtValidationSkew uint64 `bson:"jwt_issued_at_validation_skew" json:"jwt_issued_at_validation_skew"` JWTExpiresAtValidationSkew uint64 `bson:"jwt_expires_at_validation_skew" json:"jwt_expires_at_validation_skew"` JWTNotBeforeValidationSkew uint64 `bson:"jwt_not_before_validation_skew" json:"jwt_not_before_validation_skew"` + JWTSkipKid bool `bson:"jwt_skip_kid" json:"jwt_skip_kid"` NotificationsDetails NotificationsManager `bson:"notifications" json:"notifications"` EnableSignatureChecking bool `bson:"enable_signature_checking" json:"enable_signature_checking"` HmacAllowedClockSkew float64 `bson:"hmac_allowed_clock_skew" json:"hmac_allowed_clock_skew"` diff --git a/auth_manager.go b/auth_manager.go index 60019facab6..e4a0a268c01 100644 --- a/auth_manager.go +++ b/auth_manager.go @@ -44,9 +44,8 @@ type DefaultAuthorisationManager struct { } type DefaultSessionManager struct { - store storage.Handler - asyncWrites bool - disableCacheSessionState bool + store storage.Handler + asyncWrites bool } func (b *DefaultAuthorisationManager) Init(store storage.Handler) { @@ -72,7 +71,6 @@ func (b *DefaultAuthorisationManager) KeyAuthorised(keyName string) (user.Sessio return newSession, false } - newSession.SetFirstSeenHash() return newSession, true } @@ -86,7 +84,6 @@ func (b *DefaultAuthorisationManager) KeyExpired(newSession *user.SessionState) func (b *DefaultSessionManager) Init(store storage.Handler) { b.asyncWrites = config.Global().UseAsyncSessionWrite - b.disableCacheSessionState = config.Global().LocalSessionCache.DisableCacheSessionState b.store = store b.store.Connect() } @@ -96,7 +93,6 @@ func (b *DefaultSessionManager) Store() storage.Handler { } func (b *DefaultSessionManager) ResetQuota(keyName string, session *user.SessionState) { - rawKey := QuotaKeyPrefix + storage.HashKey(keyName) log.WithFields(logrus.Fields{ "prefix": "auth-mgr", @@ -115,11 +111,6 @@ func (b *DefaultSessionManager) ResetQuota(keyName string, session *user.Session // UpdateSession updates the session state in the storage engine func (b *DefaultSessionManager) UpdateSession(keyName string, session *user.SessionState, resetTTLTo int64, hashed bool) error { - if !session.HasChanged() { - log.Debug("Session has not changed, not updating") - return nil - } - v, _ := json.Marshal(session) if hashed { @@ -128,8 +119,6 @@ func (b *DefaultSessionManager) UpdateSession(keyName string, session *user.Sess // async update and return if needed if b.asyncWrites { - b.renewSessionState(keyName, session) - if hashed { go b.store.SetRawKey(keyName, string(v), resetTTLTo) return nil @@ -147,22 +136,9 @@ func (b *DefaultSessionManager) UpdateSession(keyName string, session *user.Sess err = b.store.SetKey(keyName, string(v), resetTTLTo) } - if err == nil { - b.renewSessionState(keyName, session) - } - return err } -func (b *DefaultSessionManager) renewSessionState(keyName string, session *user.SessionState) { - // we have new session state so renew first-seen hash to prevent - session.SetFirstSeenHash() - // delete it from session cache to have it re-populated next time - if !b.disableCacheSessionState { - SessionCache.Delete(keyName) - } -} - // RemoveSession removes session from storage func (b *DefaultSessionManager) RemoveSession(keyName string, hashed bool) { if hashed { @@ -199,8 +175,6 @@ func (b *DefaultSessionManager) SessionDetail(keyName string, hashed bool) (user return session, false } - session.SetFirstSeenHash() - return session, true } @@ -211,11 +185,23 @@ func (b *DefaultSessionManager) Sessions(filter string) []string { type DefaultKeyGenerator struct{} +func generateToken(orgID, keyID string) string { + keyID = strings.TrimPrefix(keyID, orgID) + token, err := storage.GenerateToken(orgID, keyID, config.Global().HashKeyFunction) + + if err != nil { + log.WithFields(logrus.Fields{ + "prefix": "auth-mgr", + "orgID": orgID, + }).WithError(err).Warning("Issue during token generation") + } + + return token +} + // GenerateAuthKey is a utility function for generating new auth keys. Returns the storage key name and the actual key func (DefaultKeyGenerator) GenerateAuthKey(orgID string) string { - u5 := uuid.NewV4() - cleanSting := strings.Replace(u5.String(), "-", "", -1) - return orgID + cleanSting + return generateToken(orgID, "") } // GenerateHMACSecret is a utility function for generating new auth keys. Returns the storage key name and the actual key diff --git a/cert.go b/cert.go index 603ecdcead9..5dffaa9c94c 100644 --- a/cert.go +++ b/cert.go @@ -50,6 +50,8 @@ var cipherSuites = map[string]uint16{ "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305": 0xcca9, } +var certLog = log.WithField("prefix", "certs") + func getUpstreamCertificate(host string, spec *APISpec) (cert *tls.Certificate) { var certID string @@ -108,6 +110,8 @@ func verifyPeerCertificatePinnedCheck(spec *APISpec, tlsConfig *tls.Config) func } return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + certLog.Debug("Checking certificate public key") + for _, rawCert := range rawCerts { cert, _ := x509.ParseCertificate(rawCert) pub, err := x509.MarshalPKIXPublicKey(cert.PublicKey) @@ -148,6 +152,8 @@ func dialTLSPinnedCheck(spec *APISpec, tc *tls.Config) func(network, addr string return c, nil } + certLog.Debug("Checking certificate public key for host:", host) + state := c.ConnectionState() for _, peercert := range state.PeerCertificates { der, err := x509.MarshalPKIXPublicKey(peercert.PublicKey) diff --git a/certs/manager.go b/certs/manager.go index 9b8d344a8b4..3384ea3baf4 100644 --- a/certs/manager.go +++ b/certs/manager.go @@ -314,6 +314,12 @@ func (c *CertificateManager) ListPublicKeys(keyIDs []string) (out []string) { } block, _ := pem.Decode(rawKey) + if block == nil { + c.logger.Error("Can't parse public key:", id) + out = append(out, "") + continue + } + fingerprint := HexSHA256(block.Bytes) c.cache.Set("pub-"+id, fingerprint, cache.DefaultExpiration) out = append(out, fingerprint) diff --git a/cli/cli.go b/cli/cli.go new file mode 100644 index 00000000000..484b59ed956 --- /dev/null +++ b/cli/cli.go @@ -0,0 +1,91 @@ +package cli + +import ( + "fmt" + "os" + + kingpin "gopkg.in/alecthomas/kingpin.v2" + + "github.com/TykTechnologies/tyk/cli/importer" + "github.com/TykTechnologies/tyk/cli/lint" +) + +const ( + appName = "tyk" + appDesc = "Tyk Gateway" +) + +var ( + // Conf specifies the configuration file path. + Conf *string + // Port specifies the listen port. + Port *string + // MemProfile enables memory profiling. + MemProfile *bool + // CPUProfile enables CPU profiling. + CPUProfile *bool + // BlockProfile enables block profiling. + BlockProfile *bool + // MutexProfile enables block profiling. + MutexProfile *bool + // HTTPProfile exposes a HTTP endpoint for accessing profiling data. + HTTPProfile *bool + // DebugMode sets the log level to debug mode. + DebugMode *bool + // LogInstrumentation outputs instrumentation data to stdout. + LogInstrumentation *bool + + app *kingpin.Application +) + +// Init sets all flags and subcommands. +func Init(version string, confPaths []string) { + app = kingpin.New(appName, appDesc) + app.HelpFlag.Short('h') + app.Version(version) + + // Start/default command: + startCmd := app.Command("start", "Starts the Tyk Gateway") + Conf = startCmd.Flag("conf", "load a named configuration file").PlaceHolder("FILE").String() + Port = startCmd.Flag("port", "listen on PORT (overrides config file)").String() + MemProfile = startCmd.Flag("memprofile", "generate a memory profile").Bool() + CPUProfile = startCmd.Flag("cpuprofile", "generate a cpu profile").Bool() + BlockProfile = startCmd.Flag("blockprofile", "generate a block profile").Bool() + MutexProfile = startCmd.Flag("mutexprofile", "generate a mutex profile").Bool() + HTTPProfile = startCmd.Flag("httpprofile", "expose runtime profiling data via HTTP").Bool() + DebugMode = startCmd.Flag("debug", "enable debug mode").Bool() + LogInstrumentation = startCmd.Flag("log-intrumentation", "output intrumentation output to stdout").Bool() + + startCmd.Action(func(ctx *kingpin.ParseContext) error { + return nil + }) + startCmd.Default() + + // Linter: + lintCmd := app.Command("lint", "Runs a linter on Tyk configuration file") + lintCmd.Action(func(c *kingpin.ParseContext) error { + path, lines, err := lint.Run(confPaths) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if len(lines) == 0 { + fmt.Printf("found no issues in %s\n", path) + return nil + } + fmt.Printf("issues found in %s:\n", path) + for _, line := range lines { + fmt.Println(line) + } + os.Exit(1) + return nil + }) + + // Import command: + importer.AddTo(app) +} + +// Parse parses the command-line arguments. +func Parse() { + kingpin.MustParse(app.Parse(os.Args[1:])) +} diff --git a/cli/importer/importer.go b/cli/importer/importer.go new file mode 100644 index 00000000000..ffd908be314 --- /dev/null +++ b/cli/importer/importer.go @@ -0,0 +1,247 @@ +package importer + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "strings" + + log "github.com/Sirupsen/logrus" + + kingpin "gopkg.in/alecthomas/kingpin.v2" + + "github.com/TykTechnologies/tyk/apidef" + "github.com/TykTechnologies/tyk/apidef/importer" +) + +const ( + cmdName = "import" + cmdDesc = "Imports a BluePrint/Swagger file" +) + +var ( + imp *Importer + errUnknownMode = errors.New("Unknown mode") +) + +// Importer wraps the import functionality. +type Importer struct { + input *string + swaggerMode *bool + bluePrintMode *bool + createAPI *bool + orgID *string + upstreamTarget *string + asMock *bool + forAPI *string + asVersion *string +} + +func init() { + imp = &Importer{} +} + +// AddTo initializes an importer object. +func AddTo(app *kingpin.Application) { + cmd := app.Command(cmdName, cmdDesc) + imp.input = cmd.Arg("input file", "e.g. blueprint.json, swagger.json, etc.").String() + imp.swaggerMode = cmd.Flag("swagger", "Use Swagger mode").Bool() + imp.bluePrintMode = cmd.Flag("blueprint", "Use BluePrint mode").Bool() + imp.createAPI = cmd.Flag("create-api", "Creates a new API definition from the blueprint").Bool() + imp.orgID = cmd.Flag("org-id", "assign the API Definition to this org_id (required with create-api").String() + imp.upstreamTarget = cmd.Flag("upstream-target", "set the upstream target for the definition").PlaceHolder("URL").String() + imp.asMock = cmd.Flag("as-mock", "creates the API as a mock based on example fields").Bool() + imp.forAPI = cmd.Flag("for-api", "adds blueprint to existing API Definition as version").PlaceHolder("PATH").String() + imp.asVersion = cmd.Flag("as-version", "the version number to use when inserting").PlaceHolder("VERSION").String() + cmd.Action(imp.Import) +} + +// Import performs the import process. +func (i *Importer) Import(ctx *kingpin.ParseContext) (err error) { + if *i.swaggerMode { + err = i.handleSwaggerMode() + if err != nil { + log.Fatal(err) + os.Exit(1) + } + } else if *i.bluePrintMode { + err = i.handleBluePrintMode() + if err != nil { + log.Fatal(err) + os.Exit(1) + } + } else { + log.Fatal(errUnknownMode) + os.Exit(1) + } + os.Exit(0) + return nil +} + +func (i *Importer) handleBluePrintMode() error { + if !*i.createAPI { + // Different branch, here we need an API Definition to modify + if *i.forAPI == "" { + return fmt.Errorf("If adding to an API, the path to the definition must be listed") + } + + if *i.asVersion == "" { + return fmt.Errorf("No version defined for this import operation, please set an import ID using the --as-version flag") + } + + defFromFile, err := i.apiDefLoadFile(*i.forAPI) + if err != nil { + return fmt.Errorf("failed to load and decode file data for API Definition: %v", err) + } + + bp, err := i.bluePrintLoadFile(*i.input) + if err != nil { + return fmt.Errorf("File load error: %v", err) + } + versionData, err := bp.ConvertIntoApiVersion(*i.asMock) + if err != nil { + return fmt.Errorf("onversion into API Def failed: %v", err) + } + + if err := bp.InsertIntoAPIDefinitionAsVersion(versionData, defFromFile, *i.asVersion); err != nil { + return fmt.Errorf("Insertion failed: %v", err) + } + + i.printDef(defFromFile) + + } + + if *i.upstreamTarget == "" && *i.orgID == "" { + return fmt.Errorf("No upstream target or org ID defined, these are both required") + } + + // Create the API with the blueprint + bp, err := i.bluePrintLoadFile(*i.input) + if err != nil { + return fmt.Errorf("File load error: %v", err) + } + + def, err := bp.ToAPIDefinition(*i.orgID, *i.upstreamTarget, *i.asMock) + if err != nil { + return fmt.Errorf("Failed to create API Definition from file") + } + + i.printDef(def) + return nil +} + +func (i *Importer) handleSwaggerMode() error { + if *i.createAPI { + if *i.upstreamTarget != "" && *i.orgID != "" { + // Create the API with the blueprint + s, err := i.swaggerLoadFile(*i.input) + if err != nil { + return fmt.Errorf("File load error: %v", err) + } + + def, err := s.ToAPIDefinition(*i.orgID, *i.upstreamTarget, *i.asMock) + if err != nil { + return fmt.Errorf("Failed to create API Defintition from file") + } + + i.printDef(def) + return nil + } + + return fmt.Errorf("No upstream target or org ID defined, these are both required") + + } + + // Different branch, here we need an API Definition to modify + if *i.forAPI == "" { + return fmt.Errorf("If adding to an API, the path to the definition must be listed") + } + + if *i.asVersion == "" { + return fmt.Errorf("No version defined for this import operation, please set an import ID using the --as-version flag") + } + + defFromFile, err := i.apiDefLoadFile(*i.forAPI) + if err != nil { + return fmt.Errorf("failed to load and decode file data for API Definition: %v", err) + } + + s, err := i.swaggerLoadFile(*i.input) + if err != nil { + return fmt.Errorf("File load error: %v", err) + } + + versionData, err := s.ConvertIntoApiVersion(*i.asMock) + if err != nil { + return fmt.Errorf("Conversion into API Def failed: %v", err) + } + + if err := s.InsertIntoAPIDefinitionAsVersion(versionData, defFromFile, *i.asVersion); err != nil { + return fmt.Errorf("Insertion failed: %v", err) + } + + i.printDef(defFromFile) + + return nil +} + +func (i *Importer) printDef(def *apidef.APIDefinition) { + asJSON, err := json.MarshalIndent(def, "", " ") + if err != nil { + log.Error("Marshalling failed: ", err) + } + + // The id attribute is for BSON only and breaks the parser if it's empty, cull it here. + fixed := strings.Replace(string(asJSON), ` "id": "",`, "", 1) + fmt.Println(fixed) +} + +func (i *Importer) swaggerLoadFile(path string) (*importer.SwaggerAST, error) { + swagger, err := importer.GetImporterForSource(importer.SwaggerSource) + if err != nil { + return nil, err + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + if err := swagger.LoadFrom(f); err != nil { + return nil, err + } + + return swagger.(*importer.SwaggerAST), nil +} + +func (i *Importer) bluePrintLoadFile(path string) (*importer.BluePrintAST, error) { + blueprint, err := importer.GetImporterForSource(importer.ApiaryBluePrint) + if err != nil { + return nil, err + } + + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + if err := blueprint.LoadFrom(f); err != nil { + return nil, err + } + + return blueprint.(*importer.BluePrintAST), nil +} + +func (i *Importer) apiDefLoadFile(path string) (*apidef.APIDefinition, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + def := &apidef.APIDefinition{} + if err := json.NewDecoder(f).Decode(&def); err != nil { + return nil, err + } + return def, nil +} diff --git a/lint/lint.go b/cli/lint/lint.go similarity index 100% rename from lint/lint.go rename to cli/lint/lint.go diff --git a/lint/lint_test.go b/cli/lint/lint_test.go similarity index 99% rename from lint/lint_test.go rename to cli/lint/lint_test.go index 753977d9632..fb874dd4a8a 100644 --- a/lint/lint_test.go +++ b/cli/lint/lint_test.go @@ -13,7 +13,7 @@ import ( func TestMain(m *testing.M) { // Use the root package, as that's where the directories and // files required to run the gateway are. - os.Chdir("..") + os.Chdir("../..") os.Exit(m.Run()) } diff --git a/lint/schema.go b/cli/lint/schema.go similarity index 99% rename from lint/schema.go rename to cli/lint/schema.go index 40314bd1d5c..64e272f68c4 100644 --- a/lint/schema.go +++ b/cli/lint/schema.go @@ -288,6 +288,10 @@ const confSchema = `{ "hash_keys": { "type": "boolean" }, + "hash_key_function": { + "type": "string", + "enum": ["", "murmur32", "murmur64", "murmur128", "sha256"] + }, "health_check": { "type": ["object", "null"], "additionalProperties": false, @@ -736,12 +740,12 @@ const confSchema = `{ }, "min_token_length": { "type": "integer" - }, + }, "disable_regexp_cache": { "type": "boolean" }, "regexp_cache_expire": { "type": "integer" - } + } } }` diff --git a/command_mode.go b/command_mode.go deleted file mode 100644 index f494e3f172e..00000000000 --- a/command_mode.go +++ /dev/null @@ -1,204 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "os" - "strings" - - "github.com/TykTechnologies/tyk/apidef" - "github.com/TykTechnologies/tyk/apidef/importer" -) - -var commandModeOptions = []interface{}{ - importBlueprint, - importSwagger, - createAPI, - orgID, - upstreamTarget, - asMock, - forAPI, - asVersion, -} - -// ./tyk --import-blueprint=blueprint.json --create-api --org-id= --upstream-target="http://widgets.com/api/"` -func handleCommandModeArgs() { - if *importBlueprint != "" { - if err := handleBluePrintMode(); err != nil { - log.Error(err) - } - } - - if *importSwagger != "" { - if err := handleSwaggerMode(); err != nil { - log.Error(err) - } - } -} - -func handleBluePrintMode() error { - if !*createAPI { - // Different branch, here we need an API Definition to modify - if *forAPI == "" { - return fmt.Errorf("If adding to an API, the path to the definition must be listed") - } - - if *asVersion == "" { - return fmt.Errorf("No version defined for this import operation, please set an import ID using the --as-version flag") - } - - defFromFile, err := apiDefLoadFile(*forAPI) - if err != nil { - return fmt.Errorf("failed to load and decode file data for API Definition: %v", err) - } - - bp, err := bluePrintLoadFile(*importBlueprint) - if err != nil { - return fmt.Errorf("File load error: %v", err) - } - versionData, err := bp.ConvertIntoApiVersion(*asMock) - if err != nil { - return fmt.Errorf("onversion into API Def failed: %v", err) - } - - if err := bp.InsertIntoAPIDefinitionAsVersion(versionData, defFromFile, *asVersion); err != nil { - return fmt.Errorf("Insertion failed: %v", err) - } - - printDef(defFromFile) - - } - - if *upstreamTarget == "" && *orgID == "" { - return fmt.Errorf("No upstream target or org ID defined, these are both required") - } - - // Create the API with the blueprint - bp, err := bluePrintLoadFile(*importBlueprint) - if err != nil { - return fmt.Errorf("File load error: %v", err) - } - - def, err := bp.ToAPIDefinition(*orgID, *upstreamTarget, *asMock) - if err != nil { - return fmt.Errorf("Failed to create API Definition from file") - } - - printDef(def) - return nil -} - -func handleSwaggerMode() error { - if *createAPI { - if *upstreamTarget != "" && *orgID != "" { - // Create the API with the blueprint - s, err := swaggerLoadFile(*importSwagger) - if err != nil { - return fmt.Errorf("File load error: %v", err) - } - - def, err := s.ToAPIDefinition(*orgID, *upstreamTarget, *asMock) - if err != nil { - return fmt.Errorf("Failed to create API Defintition from file") - } - - printDef(def) - return nil - } - - return fmt.Errorf("No upstream target or org ID defined, these are both required") - - } - - // Different branch, here we need an API Definition to modify - if *forAPI == "" { - return fmt.Errorf("If adding to an API, the path to the definition must be listed") - } - - if *asVersion == "" { - return fmt.Errorf("No version defined for this import operation, please set an import ID using the --as-version flag") - } - - defFromFile, err := apiDefLoadFile(*forAPI) - if err != nil { - return fmt.Errorf("failed to load and decode file data for API Definition: %v", err) - } - - s, err := swaggerLoadFile(*importSwagger) - if err != nil { - return fmt.Errorf("File load error: %v", err) - } - - versionData, err := s.ConvertIntoApiVersion(*asMock) - if err != nil { - return fmt.Errorf("Conversion into API Def failed: %v", err) - } - - if err := s.InsertIntoAPIDefinitionAsVersion(versionData, defFromFile, *asVersion); err != nil { - return fmt.Errorf("Insertion failed: %v", err) - } - - printDef(defFromFile) - - return nil -} - -func printDef(def *apidef.APIDefinition) { - asJSON, err := json.MarshalIndent(def, "", " ") - if err != nil { - log.Error("Marshalling failed: ", err) - } - - // The id attribute is for BSON only and breaks the parser if it's empty, cull it here. - fixed := strings.Replace(string(asJSON), ` "id": "",`, "", 1) - fmt.Println(fixed) -} - -func swaggerLoadFile(path string) (*importer.SwaggerAST, error) { - swagger, err := importer.GetImporterForSource(importer.SwaggerSource) - if err != nil { - return nil, err - } - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - - if err := swagger.LoadFrom(f); err != nil { - return nil, err - } - - return swagger.(*importer.SwaggerAST), nil -} - -func bluePrintLoadFile(path string) (*importer.BluePrintAST, error) { - blueprint, err := importer.GetImporterForSource(importer.ApiaryBluePrint) - if err != nil { - return nil, err - } - - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - - if err := blueprint.LoadFrom(f); err != nil { - return nil, err - } - - return blueprint.(*importer.BluePrintAST), nil -} - -func apiDefLoadFile(path string) (*apidef.APIDefinition, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - def := &apidef.APIDefinition{} - if err := json.NewDecoder(f).Decode(&def); err != nil { - return nil, err - } - return def, nil -} diff --git a/config/config.go b/config/config.go index faec0717c86..f7c9fbd1d4c 100644 --- a/config/config.go +++ b/config/config.go @@ -214,6 +214,7 @@ type Config struct { UseAsyncSessionWrite bool `json:"optimisations_use_async_session_write"` AllowMasterKeys bool `json:"allow_master_keys"` HashKeys bool `json:"hash_keys"` + HashKeyFunction string `json:"hash_key_function"` SuppressRedisSignalReload bool `json:"suppress_redis_signal_reload"` SupressDefaultOrgStore bool `json:"suppress_default_org_store"` UseRedisLog bool `json:"use_redis_log"` diff --git a/coprocess.go b/coprocess.go index 549b5b44852..49338d86635 100644 --- a/coprocess.go +++ b/coprocess.go @@ -5,6 +5,7 @@ package main import ( "encoding/json" "strings" + "unicode/utf8" "github.com/Sirupsen/logrus" @@ -67,15 +68,6 @@ type CoProcessor struct { // ObjectFromRequest constructs a CoProcessObject from a given http.Request. func (c *CoProcessor) ObjectFromRequest(r *http.Request) *coprocess.Object { - var body string - if r.Body == nil { - body = "" - } else { - defer r.Body.Close() - originalBody, _ := ioutil.ReadAll(r.Body) - body = string(originalBody) - } - headers := ProtoMap(r.Header) host := r.Host @@ -90,7 +82,6 @@ func (c *CoProcessor) ObjectFromRequest(r *http.Request) *coprocess.Object { Headers: headers, SetHeaders: map[string]string{}, DeleteHeaders: []string{}, - Body: body, Url: r.URL.Path, Params: ProtoMap(r.URL.Query()), AddParams: map[string]string{}, @@ -104,6 +95,14 @@ func (c *CoProcessor) ObjectFromRequest(r *http.Request) *coprocess.Object { Scheme: r.URL.Scheme, } + if r.Body != nil { + defer r.Body.Close() + miniRequestObject.RawBody, _ = ioutil.ReadAll(r.Body) + if utf8.Valid(miniRequestObject.RawBody) { + miniRequestObject.Body = string(miniRequestObject.RawBody) + } + } + object := &coprocess.Object{ Request: miniRequestObject, HookName: c.Middleware.HookName, @@ -136,13 +135,8 @@ func (c *CoProcessor) ObjectFromRequest(r *http.Request) *coprocess.Object { if c.HookType != coprocess.HookType_Pre && c.HookType != coprocess.HookType_CustomKeyCheck { if session := ctxGetSession(r); session != nil { object.Session = ProtoSessionState(session) - // If the session contains metadata, add items to the object's metadata map: - if len(session.MetaData) > 0 { - object.Metadata = make(map[string]string) - for k, v := range session.MetaData { - object.Metadata[k] = v.(string) - } - } + // For compatibility purposes: + object.Metadata = object.Session.Metadata } } @@ -278,7 +272,20 @@ func (m *CoProcessMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Requ coProcessor.ObjectPostProcess(returnObject, r) - token := returnObject.Metadata["token"] + var token string + if returnObject.Session != nil { + // For compatibility purposes, inject coprocess.Object.Metadata fields: + if returnObject.Metadata != nil { + if returnObject.Session.Metadata == nil { + returnObject.Session.Metadata = make(map[string]string) + } + for k, v := range returnObject.Metadata { + returnObject.Session.Metadata[k] = v + } + } + + token = returnObject.Session.Metadata["token"] + } // The CP middleware indicates this is a bad auth: if returnObject.Request.ReturnOverrides.ResponseCode > 400 { @@ -324,15 +331,11 @@ func (m *CoProcessMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Requ for k, v := range returnObject.Metadata { returnedSession.MetaData[k] = string(v) } + if extractor == nil { - sessionLifetime := returnedSession.Lifetime(m.Spec.SessionLifetime) - // This API is not using the ID extractor, but we've got a session: - m.Spec.SessionManager.UpdateSession(token, returnedSession, sessionLifetime, false) - ctxSetSession(r, returnedSession) - ctxSetAuthToken(r, token) + ctxSetSession(r, returnedSession, token, true) } else { - // The CP middleware did setup a session, we should pass it to the ID extractor (caching): - extractor.PostProcess(r, returnedSession, sessionID) + ctxSetSession(r, returnedSession, sessionID, true) } } diff --git a/coprocess/bindings/python/coprocess_mini_request_object_pb2.py b/coprocess/bindings/python/coprocess_mini_request_object_pb2.py index a287f3d27ea..33e3d780155 100644 --- a/coprocess/bindings/python/coprocess_mini_request_object_pb2.py +++ b/coprocess/bindings/python/coprocess_mini_request_object_pb2.py @@ -20,7 +20,7 @@ name='coprocess_mini_request_object.proto', package='coprocess', syntax='proto3', - serialized_pb=_b('\n#coprocess_mini_request_object.proto\x12\tcoprocess\x1a coprocess_return_overrides.proto\"\x88\x06\n\x11MiniRequestObject\x12:\n\x07headers\x18\x01 \x03(\x0b\x32).coprocess.MiniRequestObject.HeadersEntry\x12\x41\n\x0bset_headers\x18\x02 \x03(\x0b\x32,.coprocess.MiniRequestObject.SetHeadersEntry\x12\x16\n\x0e\x64\x65lete_headers\x18\x03 \x03(\t\x12\x0c\n\x04\x62ody\x18\x04 \x01(\t\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x38\n\x06params\x18\x06 \x03(\x0b\x32(.coprocess.MiniRequestObject.ParamsEntry\x12?\n\nadd_params\x18\x07 \x03(\x0b\x32+.coprocess.MiniRequestObject.AddParamsEntry\x12I\n\x0f\x65xtended_params\x18\x08 \x03(\x0b\x32\x30.coprocess.MiniRequestObject.ExtendedParamsEntry\x12\x15\n\rdelete_params\x18\t \x03(\t\x12\x34\n\x10return_overrides\x18\n \x01(\x0b\x32\x1a.coprocess.ReturnOverrides\x12\x0e\n\x06method\x18\x0b \x01(\t\x12\x13\n\x0brequest_uri\x18\x0c \x01(\t\x12\x0e\n\x06scheme\x18\r \x01(\t\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x31\n\x0fSetHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a-\n\x0bParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x30\n\x0e\x41\x64\x64ParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x35\n\x13\x45xtendedParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x62\x06proto3') + serialized_pb=_b('\n#coprocess_mini_request_object.proto\x12\tcoprocess\x1a coprocess_return_overrides.proto\"\x9a\x06\n\x11MiniRequestObject\x12:\n\x07headers\x18\x01 \x03(\x0b\x32).coprocess.MiniRequestObject.HeadersEntry\x12\x41\n\x0bset_headers\x18\x02 \x03(\x0b\x32,.coprocess.MiniRequestObject.SetHeadersEntry\x12\x16\n\x0e\x64\x65lete_headers\x18\x03 \x03(\t\x12\x0c\n\x04\x62ody\x18\x04 \x01(\t\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x38\n\x06params\x18\x06 \x03(\x0b\x32(.coprocess.MiniRequestObject.ParamsEntry\x12?\n\nadd_params\x18\x07 \x03(\x0b\x32+.coprocess.MiniRequestObject.AddParamsEntry\x12I\n\x0f\x65xtended_params\x18\x08 \x03(\x0b\x32\x30.coprocess.MiniRequestObject.ExtendedParamsEntry\x12\x15\n\rdelete_params\x18\t \x03(\t\x12\x34\n\x10return_overrides\x18\n \x01(\x0b\x32\x1a.coprocess.ReturnOverrides\x12\x0e\n\x06method\x18\x0b \x01(\t\x12\x13\n\x0brequest_uri\x18\x0c \x01(\t\x12\x0e\n\x06scheme\x18\r \x01(\t\x12\x10\n\x08raw_body\x18\x0e \x01(\x0c\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x31\n\x0fSetHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a-\n\x0bParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x30\n\x0e\x41\x64\x64ParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x35\n\x13\x45xtendedParamsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x62\x06proto3') , dependencies=[coprocess__return__overrides__pb2.DESCRIPTOR,]) @@ -60,8 +60,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=612, - serialized_end=658, + serialized_start=630, + serialized_end=676, ) _MINIREQUESTOBJECT_SETHEADERSENTRY = _descriptor.Descriptor( @@ -97,8 +97,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=660, - serialized_end=709, + serialized_start=678, + serialized_end=727, ) _MINIREQUESTOBJECT_PARAMSENTRY = _descriptor.Descriptor( @@ -134,8 +134,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=711, - serialized_end=756, + serialized_start=729, + serialized_end=774, ) _MINIREQUESTOBJECT_ADDPARAMSENTRY = _descriptor.Descriptor( @@ -171,8 +171,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=758, - serialized_end=806, + serialized_start=776, + serialized_end=824, ) _MINIREQUESTOBJECT_EXTENDEDPARAMSENTRY = _descriptor.Descriptor( @@ -208,8 +208,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=808, - serialized_end=861, + serialized_start=826, + serialized_end=879, ) _MINIREQUESTOBJECT = _descriptor.Descriptor( @@ -310,6 +310,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='raw_body', full_name='coprocess.MiniRequestObject.raw_body', index=13, + number=14, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -323,7 +330,7 @@ oneofs=[ ], serialized_start=85, - serialized_end=861, + serialized_end=879, ) _MINIREQUESTOBJECT_HEADERSENTRY.containing_type = _MINIREQUESTOBJECT diff --git a/coprocess/bindings/python/coprocess_session_state_pb2.py b/coprocess/bindings/python/coprocess_session_state_pb2.py index b313d51753c..c342655627b 100644 --- a/coprocess/bindings/python/coprocess_session_state_pb2.py +++ b/coprocess/bindings/python/coprocess_session_state_pb2.py @@ -19,7 +19,7 @@ name='coprocess_session_state.proto', package='coprocess', syntax='proto3', - serialized_pb=_b('\n\x1d\x63oprocess_session_state.proto\x12\tcoprocess\"*\n\nAccessSpec\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x0f\n\x07methods\x18\x02 \x03(\t\"s\n\x10\x41\x63\x63\x65ssDefinition\x12\x10\n\x08\x61pi_name\x18\x01 \x01(\t\x12\x0e\n\x06\x61pi_id\x18\x02 \x01(\t\x12\x10\n\x08versions\x18\x03 \x03(\t\x12+\n\x0c\x61llowed_urls\x18\x04 \x03(\x0b\x32\x15.coprocess.AccessSpec\"/\n\rBasicAuthData\x12\x10\n\x08password\x18\x01 \x01(\t\x12\x0c\n\x04hash\x18\x02 \x01(\t\"\x19\n\x07JWTData\x12\x0e\n\x06secret\x18\x01 \x01(\t\"!\n\x07Monitor\x12\x16\n\x0etrigger_limits\x18\x01 \x03(\x01\"\xa5\x07\n\x0cSessionState\x12\x12\n\nlast_check\x18\x01 \x01(\x03\x12\x11\n\tallowance\x18\x02 \x01(\x01\x12\x0c\n\x04rate\x18\x03 \x01(\x01\x12\x0b\n\x03per\x18\x04 \x01(\x01\x12\x0f\n\x07\x65xpires\x18\x05 \x01(\x03\x12\x11\n\tquota_max\x18\x06 \x01(\x03\x12\x14\n\x0cquota_renews\x18\x07 \x01(\x03\x12\x17\n\x0fquota_remaining\x18\x08 \x01(\x03\x12\x1a\n\x12quota_renewal_rate\x18\t \x01(\x03\x12@\n\raccess_rights\x18\n \x03(\x0b\x32).coprocess.SessionState.AccessRightsEntry\x12\x0e\n\x06org_id\x18\x0b \x01(\t\x12\x17\n\x0foauth_client_id\x18\x0c \x01(\t\x12:\n\noauth_keys\x18\r \x03(\x0b\x32&.coprocess.SessionState.OauthKeysEntry\x12\x31\n\x0f\x62\x61sic_auth_data\x18\x0e \x01(\x0b\x32\x18.coprocess.BasicAuthData\x12$\n\x08jwt_data\x18\x0f \x01(\x0b\x32\x12.coprocess.JWTData\x12\x14\n\x0chmac_enabled\x18\x10 \x01(\x08\x12\x13\n\x0bhmac_secret\x18\x11 \x01(\t\x12\x13\n\x0bis_inactive\x18\x12 \x01(\x08\x12\x17\n\x0f\x61pply_policy_id\x18\x13 \x01(\t\x12\x14\n\x0c\x64\x61ta_expires\x18\x14 \x01(\x03\x12#\n\x07monitor\x18\x15 \x01(\x0b\x32\x12.coprocess.Monitor\x12!\n\x19\x65nable_detailed_recording\x18\x16 \x01(\x08\x12\x10\n\x08metadata\x18\x17 \x01(\t\x12\x0c\n\x04tags\x18\x18 \x03(\t\x12\r\n\x05\x61lias\x18\x19 \x01(\t\x12\x14\n\x0clast_updated\x18\x1a \x01(\t\x12\x1d\n\x15id_extractor_deadline\x18\x1b \x01(\x03\x12\x18\n\x10session_lifetime\x18\x1c \x01(\x03\x12\x16\n\x0e\x61pply_policies\x18\x1d \x03(\t\x12\x13\n\x0b\x63\x65rtificate\x18\x1e \x01(\t\x1aP\n\x11\x41\x63\x63\x65ssRightsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.coprocess.AccessDefinition:\x02\x38\x01\x1a\x30\n\x0eOauthKeysEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x62\x06proto3') + serialized_pb=_b('\n\x1d\x63oprocess_session_state.proto\x12\tcoprocess\"*\n\nAccessSpec\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x0f\n\x07methods\x18\x02 \x03(\t\"s\n\x10\x41\x63\x63\x65ssDefinition\x12\x10\n\x08\x61pi_name\x18\x01 \x01(\t\x12\x0e\n\x06\x61pi_id\x18\x02 \x01(\t\x12\x10\n\x08versions\x18\x03 \x03(\t\x12+\n\x0c\x61llowed_urls\x18\x04 \x03(\x0b\x32\x15.coprocess.AccessSpec\"/\n\rBasicAuthData\x12\x10\n\x08password\x18\x01 \x01(\t\x12\x0c\n\x04hash\x18\x02 \x01(\t\"\x19\n\x07JWTData\x12\x0e\n\x06secret\x18\x01 \x01(\t\"!\n\x07Monitor\x12\x16\n\x0etrigger_limits\x18\x01 \x03(\x01\"\xfd\x07\n\x0cSessionState\x12\x12\n\nlast_check\x18\x01 \x01(\x03\x12\x11\n\tallowance\x18\x02 \x01(\x01\x12\x0c\n\x04rate\x18\x03 \x01(\x01\x12\x0b\n\x03per\x18\x04 \x01(\x01\x12\x0f\n\x07\x65xpires\x18\x05 \x01(\x03\x12\x11\n\tquota_max\x18\x06 \x01(\x03\x12\x14\n\x0cquota_renews\x18\x07 \x01(\x03\x12\x17\n\x0fquota_remaining\x18\x08 \x01(\x03\x12\x1a\n\x12quota_renewal_rate\x18\t \x01(\x03\x12@\n\raccess_rights\x18\n \x03(\x0b\x32).coprocess.SessionState.AccessRightsEntry\x12\x0e\n\x06org_id\x18\x0b \x01(\t\x12\x17\n\x0foauth_client_id\x18\x0c \x01(\t\x12:\n\noauth_keys\x18\r \x03(\x0b\x32&.coprocess.SessionState.OauthKeysEntry\x12\x31\n\x0f\x62\x61sic_auth_data\x18\x0e \x01(\x0b\x32\x18.coprocess.BasicAuthData\x12$\n\x08jwt_data\x18\x0f \x01(\x0b\x32\x12.coprocess.JWTData\x12\x14\n\x0chmac_enabled\x18\x10 \x01(\x08\x12\x13\n\x0bhmac_secret\x18\x11 \x01(\t\x12\x13\n\x0bis_inactive\x18\x12 \x01(\x08\x12\x17\n\x0f\x61pply_policy_id\x18\x13 \x01(\t\x12\x14\n\x0c\x64\x61ta_expires\x18\x14 \x01(\x03\x12#\n\x07monitor\x18\x15 \x01(\x0b\x32\x12.coprocess.Monitor\x12!\n\x19\x65nable_detailed_recording\x18\x16 \x01(\x08\x12\x37\n\x08metadata\x18\x17 \x03(\x0b\x32%.coprocess.SessionState.MetadataEntry\x12\x0c\n\x04tags\x18\x18 \x03(\t\x12\r\n\x05\x61lias\x18\x19 \x01(\t\x12\x14\n\x0clast_updated\x18\x1a \x01(\t\x12\x1d\n\x15id_extractor_deadline\x18\x1b \x01(\x03\x12\x18\n\x10session_lifetime\x18\x1c \x01(\x03\x12\x16\n\x0e\x61pply_policies\x18\x1d \x03(\t\x12\x13\n\x0b\x63\x65rtificate\x18\x1e \x01(\t\x1aP\n\x11\x41\x63\x63\x65ssRightsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.coprocess.AccessDefinition:\x02\x38\x01\x1a\x30\n\x0eOauthKeysEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x62\x06proto3') ) @@ -248,8 +248,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1120, - serialized_end=1200, + serialized_start=1159, + serialized_end=1239, ) _SESSIONSTATE_OAUTHKEYSENTRY = _descriptor.Descriptor( @@ -285,8 +285,45 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1202, - serialized_end=1250, + serialized_start=1241, + serialized_end=1289, +) + +_SESSIONSTATE_METADATAENTRY = _descriptor.Descriptor( + name='MetadataEntry', + full_name='coprocess.SessionState.MetadataEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='coprocess.SessionState.MetadataEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', full_name='coprocess.SessionState.MetadataEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1291, + serialized_end=1338, ) _SESSIONSTATE = _descriptor.Descriptor( @@ -452,8 +489,8 @@ options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='metadata', full_name='coprocess.SessionState.metadata', index=22, - number=23, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + number=23, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), @@ -509,7 +546,7 @@ ], extensions=[ ], - nested_types=[_SESSIONSTATE_ACCESSRIGHTSENTRY, _SESSIONSTATE_OAUTHKEYSENTRY, ], + nested_types=[_SESSIONSTATE_ACCESSRIGHTSENTRY, _SESSIONSTATE_OAUTHKEYSENTRY, _SESSIONSTATE_METADATAENTRY, ], enum_types=[ ], options=None, @@ -519,18 +556,20 @@ oneofs=[ ], serialized_start=317, - serialized_end=1250, + serialized_end=1338, ) _ACCESSDEFINITION.fields_by_name['allowed_urls'].message_type = _ACCESSSPEC _SESSIONSTATE_ACCESSRIGHTSENTRY.fields_by_name['value'].message_type = _ACCESSDEFINITION _SESSIONSTATE_ACCESSRIGHTSENTRY.containing_type = _SESSIONSTATE _SESSIONSTATE_OAUTHKEYSENTRY.containing_type = _SESSIONSTATE +_SESSIONSTATE_METADATAENTRY.containing_type = _SESSIONSTATE _SESSIONSTATE.fields_by_name['access_rights'].message_type = _SESSIONSTATE_ACCESSRIGHTSENTRY _SESSIONSTATE.fields_by_name['oauth_keys'].message_type = _SESSIONSTATE_OAUTHKEYSENTRY _SESSIONSTATE.fields_by_name['basic_auth_data'].message_type = _BASICAUTHDATA _SESSIONSTATE.fields_by_name['jwt_data'].message_type = _JWTDATA _SESSIONSTATE.fields_by_name['monitor'].message_type = _MONITOR +_SESSIONSTATE.fields_by_name['metadata'].message_type = _SESSIONSTATE_METADATAENTRY DESCRIPTOR.message_types_by_name['AccessSpec'] = _ACCESSSPEC DESCRIPTOR.message_types_by_name['AccessDefinition'] = _ACCESSDEFINITION DESCRIPTOR.message_types_by_name['BasicAuthData'] = _BASICAUTHDATA @@ -589,6 +628,13 @@ # @@protoc_insertion_point(class_scope:coprocess.SessionState.OauthKeysEntry) )) , + + MetadataEntry = _reflection.GeneratedProtocolMessageType('MetadataEntry', (_message.Message,), dict( + DESCRIPTOR = _SESSIONSTATE_METADATAENTRY, + __module__ = 'coprocess_session_state_pb2' + # @@protoc_insertion_point(class_scope:coprocess.SessionState.MetadataEntry) + )) + , DESCRIPTOR = _SESSIONSTATE, __module__ = 'coprocess_session_state_pb2' # @@protoc_insertion_point(class_scope:coprocess.SessionState) @@ -596,10 +642,13 @@ _sym_db.RegisterMessage(SessionState) _sym_db.RegisterMessage(SessionState.AccessRightsEntry) _sym_db.RegisterMessage(SessionState.OauthKeysEntry) +_sym_db.RegisterMessage(SessionState.MetadataEntry) _SESSIONSTATE_ACCESSRIGHTSENTRY.has_options = True _SESSIONSTATE_ACCESSRIGHTSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) _SESSIONSTATE_OAUTHKEYSENTRY.has_options = True _SESSIONSTATE_OAUTHKEYSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_SESSIONSTATE_METADATAENTRY.has_options = True +_SESSIONSTATE_METADATAENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) # @@protoc_insertion_point(module_scope) diff --git a/coprocess/bindings/ruby/coprocess_mini_request_object_pb.rb b/coprocess/bindings/ruby/coprocess_mini_request_object_pb.rb index afb69231ab6..6dac208c0d3 100644 --- a/coprocess/bindings/ruby/coprocess_mini_request_object_pb.rb +++ b/coprocess/bindings/ruby/coprocess_mini_request_object_pb.rb @@ -19,6 +19,7 @@ optional :method, :string, 11 optional :request_uri, :string, 12 optional :scheme, :string, 13 + optional :raw_body, :bytes, 14 end end diff --git a/coprocess/bindings/ruby/coprocess_session_state_pb.rb b/coprocess/bindings/ruby/coprocess_session_state_pb.rb index d9a611c4c29..7f638c3e097 100644 --- a/coprocess/bindings/ruby/coprocess_session_state_pb.rb +++ b/coprocess/bindings/ruby/coprocess_session_state_pb.rb @@ -47,7 +47,7 @@ optional :data_expires, :int64, 20 optional :monitor, :message, 21, "coprocess.Monitor" optional :enable_detailed_recording, :bool, 22 - optional :metadata, :string, 23 + map :metadata, :string, :string, 23 repeated :tags, :string, 24 optional :alias, :string, 25 optional :last_updated, :string, 26 diff --git a/coprocess/coprocess_mini_request_object.pb.go b/coprocess/coprocess_mini_request_object.pb.go index b0dd906e1ab..7c10ed664c5 100644 --- a/coprocess/coprocess_mini_request_object.pb.go +++ b/coprocess/coprocess_mini_request_object.pb.go @@ -26,6 +26,7 @@ type MiniRequestObject struct { Method string `protobuf:"bytes,11,opt,name=method" json:"method,omitempty"` RequestUri string `protobuf:"bytes,12,opt,name=request_uri,json=requestUri" json:"request_uri,omitempty"` Scheme string `protobuf:"bytes,13,opt,name=scheme" json:"scheme,omitempty"` + RawBody []byte `protobuf:"bytes,14,opt,name=raw_body,json=rawBody,proto3" json:"raw_body,omitempty"` } func (m *MiniRequestObject) Reset() { *m = MiniRequestObject{} } @@ -124,6 +125,13 @@ func (m *MiniRequestObject) GetScheme() string { return "" } +func (m *MiniRequestObject) GetRawBody() []byte { + if m != nil { + return m.RawBody + } + return nil +} + func init() { proto.RegisterType((*MiniRequestObject)(nil), "coprocess.MiniRequestObject") } @@ -131,32 +139,34 @@ func init() { func init() { proto.RegisterFile("coprocess_mini_request_object.proto", fileDescriptor1) } var fileDescriptor1 = []byte{ - // 431 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x93, 0xd1, 0x6b, 0xd4, 0x40, - 0x10, 0xc6, 0xc9, 0xa5, 0xcd, 0x99, 0xc9, 0xf5, 0xae, 0xae, 0x22, 0x4b, 0x5e, 0x0c, 0x16, 0x21, - 0xa2, 0x04, 0xa9, 0x2f, 0x5a, 0x14, 0x2c, 0x72, 0x20, 0x42, 0xa9, 0x44, 0x7c, 0xf0, 0x29, 0xe4, - 0xb2, 0x03, 0x17, 0x7b, 0xc9, 0x9e, 0xbb, 0x9b, 0x62, 0xfe, 0x4c, 0xff, 0x23, 0xb9, 0xdd, 0x24, - 0x26, 0xb5, 0x04, 0xf2, 0xb6, 0xf3, 0xe5, 0xfb, 0x7e, 0x0c, 0x33, 0x13, 0x38, 0xcb, 0xf8, 0x5e, - 0xf0, 0x0c, 0xa5, 0x4c, 0x8a, 0xbc, 0xcc, 0x13, 0x81, 0xbf, 0x2a, 0x94, 0x2a, 0xe1, 0x9b, 0x9f, - 0x98, 0xa9, 0x68, 0x2f, 0xb8, 0xe2, 0xc4, 0xed, 0x4c, 0x7e, 0xf0, 0xcf, 0x2f, 0x50, 0x55, 0xa2, - 0x4c, 0xf8, 0x2d, 0x0a, 0x91, 0x33, 0x94, 0xc6, 0xfc, 0xec, 0xcf, 0x1c, 0x1e, 0x5e, 0xe5, 0x65, - 0x1e, 0x1b, 0xd2, 0xb5, 0x06, 0x91, 0x4f, 0x30, 0xdf, 0x62, 0xca, 0x50, 0x48, 0x6a, 0x05, 0x76, - 0xe8, 0x9d, 0xbf, 0x88, 0x3a, 0x52, 0xf4, 0x9f, 0x3d, 0xfa, 0x6c, 0xbc, 0xeb, 0x52, 0x89, 0x3a, - 0x6e, 0x93, 0xe4, 0x0a, 0x3c, 0x89, 0x2a, 0x69, 0x41, 0x33, 0x0d, 0x7a, 0x35, 0x0a, 0xfa, 0x86, - 0x6a, 0xc0, 0x02, 0xd9, 0x09, 0xe4, 0x39, 0x2c, 0x19, 0xee, 0x50, 0x61, 0x47, 0xb4, 0x03, 0x3b, - 0x74, 0xe3, 0x13, 0xa3, 0xb6, 0x36, 0x02, 0x47, 0x1b, 0xce, 0x6a, 0x7a, 0x14, 0x58, 0xa1, 0x1b, - 0xeb, 0x37, 0x39, 0x05, 0xbb, 0x12, 0x3b, 0x7a, 0xac, 0xa5, 0xc3, 0x93, 0x7c, 0x04, 0x67, 0x9f, - 0x8a, 0xb4, 0x90, 0xd4, 0xd1, 0x6d, 0x85, 0xa3, 0x6d, 0x7d, 0xd5, 0x56, 0xd3, 0x52, 0x93, 0x23, - 0x5f, 0x00, 0x52, 0xc6, 0x92, 0x86, 0x32, 0xd7, 0x94, 0x97, 0xa3, 0x94, 0x4b, 0xc6, 0xfa, 0x20, - 0x37, 0x6d, 0x6b, 0xf2, 0x03, 0x56, 0xf8, 0x5b, 0x61, 0xc9, 0xb0, 0x03, 0x3e, 0xd0, 0xc0, 0xd7, - 0xa3, 0xc0, 0x75, 0x93, 0xe9, 0x53, 0x97, 0x38, 0x10, 0xc9, 0x19, 0x34, 0xf3, 0x69, 0xc1, 0xae, - 0x1e, 0xda, 0xc2, 0x88, 0x8d, 0x69, 0x0d, 0xa7, 0x77, 0xcf, 0x83, 0x42, 0x60, 0x85, 0xde, 0xb9, - 0xdf, 0x6b, 0x20, 0xd6, 0x96, 0xeb, 0xd6, 0x11, 0xaf, 0xc4, 0x50, 0x20, 0x4f, 0xc0, 0x29, 0x50, - 0x6d, 0x39, 0xa3, 0x9e, 0x9e, 0x74, 0x53, 0x91, 0xa7, 0xe0, 0xb5, 0x87, 0x5a, 0x89, 0x9c, 0x2e, - 0xf4, 0x47, 0x68, 0xa4, 0xef, 0x22, 0x3f, 0x04, 0x65, 0xb6, 0xc5, 0x02, 0xe9, 0x89, 0x09, 0x9a, - 0xca, 0xbf, 0x80, 0x45, 0xff, 0x1c, 0x0e, 0x7b, 0xbc, 0xc1, 0x9a, 0x5a, 0x66, 0x8f, 0x37, 0x58, - 0x93, 0xc7, 0x70, 0x7c, 0x9b, 0xee, 0x2a, 0xa4, 0x33, 0xad, 0x99, 0xe2, 0x62, 0xf6, 0xd6, 0xf2, - 0x3f, 0xc0, 0xea, 0xce, 0x35, 0x4d, 0x8a, 0xbf, 0x03, 0xaf, 0x37, 0xd6, 0x49, 0xd1, 0xf7, 0xb0, - 0x1c, 0xae, 0x7a, 0x52, 0xfa, 0x12, 0x1e, 0xdd, 0xb3, 0xd7, 0x29, 0x88, 0x8d, 0xa3, 0x7f, 0xed, - 0x37, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x7e, 0x7f, 0x64, 0x62, 0x2e, 0x04, 0x00, 0x00, + // 452 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x93, 0xd1, 0x8b, 0xd3, 0x40, + 0x10, 0xc6, 0x49, 0x7b, 0xd7, 0x5e, 0x26, 0xbd, 0xf6, 0x5c, 0x45, 0xd6, 0xbe, 0x18, 0x3c, 0x84, + 0x88, 0x52, 0xe4, 0x7c, 0xd1, 0x43, 0xc1, 0x53, 0x0a, 0x22, 0x1c, 0x27, 0x11, 0x1f, 0x7c, 0x0a, + 0xdb, 0xee, 0x40, 0xe3, 0xb5, 0xd9, 0x3a, 0xbb, 0xb9, 0x33, 0xff, 0xb3, 0x7f, 0x84, 0x74, 0x37, + 0x89, 0xe9, 0x29, 0x81, 0xbc, 0xed, 0x7c, 0xf9, 0xbe, 0x1f, 0xc3, 0xcc, 0x04, 0x4e, 0x97, 0x6a, + 0x4b, 0x6a, 0x89, 0x5a, 0x27, 0x9b, 0x34, 0x4b, 0x13, 0xc2, 0x9f, 0x39, 0x6a, 0x93, 0xa8, 0xc5, + 0x0f, 0x5c, 0x9a, 0xd9, 0x96, 0x94, 0x51, 0xcc, 0xaf, 0x4d, 0xd3, 0xf0, 0xaf, 0x9f, 0xd0, 0xe4, + 0x94, 0x25, 0xea, 0x06, 0x89, 0x52, 0x89, 0xda, 0x99, 0x9f, 0xfc, 0x1e, 0xc2, 0xbd, 0xcb, 0x34, + 0x4b, 0x63, 0x47, 0xba, 0xb2, 0x20, 0xf6, 0x11, 0x86, 0x2b, 0x14, 0x12, 0x49, 0x73, 0x2f, 0xec, + 0x47, 0xc1, 0xd9, 0xb3, 0x59, 0x4d, 0x9a, 0xfd, 0x63, 0x9f, 0x7d, 0x72, 0xde, 0x79, 0x66, 0xa8, + 0x88, 0xab, 0x24, 0xbb, 0x84, 0x40, 0xa3, 0x49, 0x2a, 0x50, 0xcf, 0x82, 0x5e, 0xb4, 0x82, 0xbe, + 0xa2, 0xd9, 0x63, 0x81, 0xae, 0x05, 0xf6, 0x14, 0xc6, 0x12, 0xd7, 0x68, 0xb0, 0x26, 0xf6, 0xc3, + 0x7e, 0xe4, 0xc7, 0xc7, 0x4e, 0xad, 0x6c, 0x0c, 0x0e, 0x16, 0x4a, 0x16, 0xfc, 0x20, 0xf4, 0x22, + 0x3f, 0xb6, 0x6f, 0x76, 0x02, 0xfd, 0x9c, 0xd6, 0xfc, 0xd0, 0x4a, 0xbb, 0x27, 0x7b, 0x0f, 0x83, + 0xad, 0x20, 0xb1, 0xd1, 0x7c, 0x60, 0xdb, 0x8a, 0x5a, 0xdb, 0xfa, 0x62, 0xad, 0xae, 0xa5, 0x32, + 0xc7, 0x3e, 0x03, 0x08, 0x29, 0x93, 0x92, 0x32, 0xb4, 0x94, 0xe7, 0xad, 0x94, 0x0b, 0x29, 0x9b, + 0x20, 0x5f, 0x54, 0x35, 0xfb, 0x0e, 0x13, 0xfc, 0x65, 0x30, 0x93, 0x58, 0x03, 0x8f, 0x2c, 0xf0, + 0x65, 0x2b, 0x70, 0x5e, 0x66, 0x9a, 0xd4, 0x31, 0xee, 0x89, 0xec, 0x14, 0xca, 0xf9, 0x54, 0x60, + 0xdf, 0x0e, 0x6d, 0xe4, 0xc4, 0xd2, 0x34, 0x87, 0x93, 0xbb, 0xe7, 0xc1, 0x21, 0xf4, 0xa2, 0xe0, + 0x6c, 0xda, 0x68, 0x20, 0xb6, 0x96, 0xab, 0xca, 0x11, 0x4f, 0x68, 0x5f, 0x60, 0x0f, 0x61, 0xb0, + 0x41, 0xb3, 0x52, 0x92, 0x07, 0x76, 0xd2, 0x65, 0xc5, 0x1e, 0x43, 0x50, 0x1d, 0x6a, 0x4e, 0x29, + 0x1f, 0xd9, 0x8f, 0x50, 0x4a, 0xdf, 0x28, 0xdd, 0x05, 0xf5, 0x72, 0x85, 0x1b, 0xe4, 0xc7, 0x2e, + 0xe8, 0x2a, 0xf6, 0x08, 0x8e, 0x48, 0xdc, 0x26, 0x76, 0x9f, 0xe3, 0xd0, 0x8b, 0x46, 0xf1, 0x90, + 0xc4, 0xed, 0x07, 0x25, 0x8b, 0xe9, 0x39, 0x8c, 0x9a, 0x97, 0xb2, 0x5b, 0xf1, 0x35, 0x16, 0xdc, + 0x73, 0x2b, 0xbe, 0xc6, 0x82, 0x3d, 0x80, 0xc3, 0x1b, 0xb1, 0xce, 0x91, 0xf7, 0xac, 0xe6, 0x8a, + 0xf3, 0xde, 0x6b, 0x6f, 0xfa, 0x0e, 0x26, 0x77, 0x0e, 0xad, 0x53, 0xfc, 0x0d, 0x04, 0x8d, 0x89, + 0x77, 0x8a, 0xbe, 0x85, 0xf1, 0xfe, 0x15, 0x74, 0x4a, 0x5f, 0xc0, 0xfd, 0xff, 0xac, 0xbc, 0x0b, + 0x62, 0x31, 0xb0, 0x7f, 0xfd, 0xab, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xa0, 0xe0, 0x90, 0xb7, + 0x49, 0x04, 0x00, 0x00, } diff --git a/coprocess/coprocess_session_state.pb.go b/coprocess/coprocess_session_state.pb.go index c69efce4210..6149fa78cad 100644 --- a/coprocess/coprocess_session_state.pb.go +++ b/coprocess/coprocess_session_state.pb.go @@ -155,7 +155,7 @@ type SessionState struct { DataExpires int64 `protobuf:"varint,20,opt,name=data_expires,json=dataExpires" json:"data_expires,omitempty"` Monitor *Monitor `protobuf:"bytes,21,opt,name=monitor" json:"monitor,omitempty"` EnableDetailedRecording bool `protobuf:"varint,22,opt,name=enable_detailed_recording,json=enableDetailedRecording" json:"enable_detailed_recording,omitempty"` - Metadata string `protobuf:"bytes,23,opt,name=metadata" json:"metadata,omitempty"` + Metadata map[string]string `protobuf:"bytes,23,rep,name=metadata" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` Tags []string `protobuf:"bytes,24,rep,name=tags" json:"tags,omitempty"` Alias string `protobuf:"bytes,25,opt,name=alias" json:"alias,omitempty"` LastUpdated string `protobuf:"bytes,26,opt,name=last_updated,json=lastUpdated" json:"last_updated,omitempty"` @@ -324,11 +324,11 @@ func (m *SessionState) GetEnableDetailedRecording() bool { return false } -func (m *SessionState) GetMetadata() string { +func (m *SessionState) GetMetadata() map[string]string { if m != nil { return m.Metadata } - return "" + return nil } func (m *SessionState) GetTags() []string { @@ -392,63 +392,64 @@ func init() { func init() { proto.RegisterFile("coprocess_session_state.proto", fileDescriptor4) } var fileDescriptor4 = []byte{ - // 914 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x55, 0x6f, 0x4f, 0x1b, 0xc7, - 0x13, 0x96, 0x31, 0x60, 0x7b, 0x6c, 0x03, 0xd9, 0x40, 0xb2, 0x40, 0xf8, 0xfd, 0x8c, 0xa5, 0xa6, - 0x8e, 0x94, 0xa2, 0x96, 0xbe, 0x41, 0x51, 0xa5, 0x36, 0x0d, 0xbc, 0xa0, 0x4d, 0xd2, 0xea, 0x68, - 0xd4, 0x37, 0x95, 0x56, 0xc3, 0xdd, 0x60, 0x6f, 0xb8, 0x7f, 0xdd, 0x5d, 0x63, 0xfc, 0x55, 0xfa, - 0x29, 0xfa, 0x11, 0xab, 0x9d, 0xdb, 0x83, 0x43, 0xb4, 0xef, 0x76, 0x9e, 0xe7, 0xd9, 0xb9, 0x67, - 0x77, 0x67, 0xe6, 0xe0, 0x20, 0x2e, 0x4a, 0x53, 0xc4, 0x64, 0xad, 0xb2, 0x64, 0xad, 0x2e, 0x72, - 0x65, 0x1d, 0x3a, 0x3a, 0x2a, 0x4d, 0xe1, 0x0a, 0xd1, 0xbb, 0xa3, 0xc7, 0x27, 0x00, 0x6f, 0x63, - 0xbf, 0xba, 0x28, 0x29, 0x16, 0x5b, 0xd0, 0x9e, 0x9b, 0x54, 0xb6, 0x46, 0xad, 0x49, 0x2f, 0xf2, - 0x4b, 0x21, 0xa1, 0x93, 0x91, 0x9b, 0x15, 0x89, 0x95, 0x2b, 0xa3, 0xf6, 0xa4, 0x17, 0xd5, 0xe1, - 0xf8, 0xaf, 0x16, 0x6c, 0x55, 0x5b, 0x4f, 0xe9, 0x4a, 0xe7, 0xda, 0xe9, 0x22, 0x17, 0xbb, 0xd0, - 0xc5, 0x52, 0xab, 0x1c, 0x33, 0x0a, 0x59, 0x3a, 0x58, 0xea, 0x8f, 0x98, 0x91, 0xd8, 0x81, 0x75, - 0x4f, 0xe9, 0x44, 0xae, 0x30, 0xb1, 0x86, 0xa5, 0x3e, 0x4f, 0xc4, 0x1e, 0x74, 0x6f, 0xc8, 0x78, - 0x8b, 0x56, 0xb6, 0xf9, 0x0b, 0x77, 0xb1, 0x38, 0x81, 0x01, 0xa6, 0x69, 0xb1, 0xa0, 0x44, 0xcd, - 0x4d, 0x6a, 0xe5, 0xea, 0xa8, 0x3d, 0xe9, 0x1f, 0xef, 0x1c, 0xdd, 0xd9, 0x3f, 0xba, 0xf7, 0x1e, - 0xf5, 0x83, 0xf4, 0x93, 0x49, 0xed, 0xf8, 0x7b, 0x18, 0xfe, 0x88, 0x56, 0xc7, 0x6f, 0xe7, 0x6e, - 0x76, 0x8a, 0x0e, 0xfd, 0x67, 0x4a, 0xb4, 0x76, 0x51, 0x98, 0x24, 0x18, 0xbb, 0x8b, 0x85, 0x80, - 0xd5, 0x19, 0xda, 0x59, 0xf0, 0xc5, 0xeb, 0xf1, 0x21, 0x74, 0x7e, 0xfa, 0xfd, 0x37, 0xde, 0xfa, - 0x0c, 0xd6, 0x2d, 0xc5, 0x86, 0x5c, 0xd8, 0x18, 0xa2, 0xf1, 0xd7, 0xd0, 0xf9, 0x50, 0xe4, 0xda, - 0x15, 0x46, 0x7c, 0x01, 0x1b, 0xce, 0xe8, 0xe9, 0x94, 0x8c, 0x4a, 0x75, 0xa6, 0x9d, 0x95, 0xad, - 0x51, 0x7b, 0xd2, 0x8a, 0x86, 0x01, 0x7d, 0xcf, 0xe0, 0xf8, 0x6f, 0x80, 0xc1, 0x45, 0xf5, 0x1e, - 0x17, 0xfe, 0x39, 0xc4, 0x01, 0x40, 0x8a, 0xd6, 0xa9, 0x78, 0x46, 0xf1, 0x35, 0xa7, 0x6f, 0x47, - 0x3d, 0x8f, 0xbc, 0xf3, 0x80, 0x78, 0x01, 0x3d, 0x3e, 0x14, 0xe6, 0x31, 0xb1, 0xbb, 0x56, 0x74, - 0x0f, 0x78, 0xdb, 0x06, 0x1d, 0xc9, 0x36, 0x13, 0xbc, 0xf6, 0x0f, 0x58, 0x92, 0x91, 0xab, 0x0c, - 0xf9, 0xa5, 0x7f, 0x40, 0xba, 0x2d, 0xb5, 0x21, 0x2b, 0xd7, 0x38, 0x7f, 0x1d, 0x8a, 0x7d, 0xe8, - 0xfd, 0x39, 0x2f, 0x1c, 0xaa, 0x0c, 0x6f, 0xe5, 0x3a, 0x73, 0x5d, 0x06, 0x3e, 0xe0, 0xad, 0x38, - 0x84, 0x41, 0x45, 0x1a, 0xca, 0x69, 0x61, 0x65, 0x87, 0xf9, 0x3e, 0x63, 0x11, 0x43, 0xe2, 0x4b, - 0xd8, 0xac, 0x25, 0x19, 0xea, 0x5c, 0xe7, 0x53, 0xd9, 0x65, 0xd5, 0x46, 0x50, 0x05, 0x54, 0xbc, - 0x06, 0xd1, 0xc8, 0x85, 0xa9, 0x62, 0xdb, 0x3d, 0xd6, 0x6e, 0xdd, 0x67, 0xc4, 0x34, 0xf2, 0x47, - 0xf8, 0x08, 0x43, 0xe4, 0x57, 0x55, 0x46, 0x4f, 0x67, 0xce, 0x4a, 0xe0, 0x57, 0x7f, 0xd5, 0x78, - 0xf5, 0xe6, 0x1d, 0x86, 0x12, 0x88, 0x58, 0x7b, 0x96, 0x3b, 0xb3, 0x8c, 0x06, 0xd8, 0x80, 0x7c, - 0xdd, 0x15, 0x66, 0xea, 0xeb, 0xae, 0x5f, 0xd5, 0x5d, 0x61, 0xa6, 0xe7, 0x89, 0x78, 0x09, 0x9b, - 0x05, 0xce, 0xdd, 0x4c, 0xc5, 0xa9, 0xa6, 0xdc, 0x79, 0x7e, 0xc0, 0xfc, 0x90, 0xe1, 0x77, 0x8c, - 0x9e, 0x27, 0xe2, 0x0c, 0xa0, 0xd2, 0x5d, 0xd3, 0xd2, 0xca, 0x21, 0x7b, 0x79, 0xf9, 0x5f, 0x5e, - 0x7e, 0xf1, 0xca, 0x9f, 0x69, 0x19, 0x8c, 0xf4, 0x8a, 0x3a, 0x16, 0x3f, 0xc0, 0xe6, 0xa5, 0x2f, - 0x48, 0xc5, 0xb9, 0x12, 0x74, 0x28, 0x37, 0x46, 0xad, 0x49, 0xff, 0x58, 0x36, 0x72, 0x3d, 0x28, - 0xd9, 0x68, 0x78, 0xf9, 0xa0, 0x82, 0xbf, 0x82, 0xee, 0xe7, 0x85, 0xab, 0xb6, 0x6e, 0xf2, 0x56, - 0xd1, 0xd8, 0x1a, 0x8a, 0x35, 0xea, 0x7c, 0x5e, 0x38, 0x96, 0x1f, 0xc2, 0x60, 0x96, 0x61, 0xac, - 0x28, 0xc7, 0xcb, 0x94, 0x12, 0xb9, 0x35, 0x6a, 0x4d, 0xba, 0x51, 0xdf, 0x63, 0x67, 0x15, 0x24, - 0xfe, 0x0f, 0x1c, 0xaa, 0x50, 0xdd, 0x4f, 0xf8, 0xf8, 0xe0, 0xa1, 0x0b, 0x46, 0xbc, 0x40, 0x5b, - 0xa5, 0x73, 0x8c, 0x9d, 0xbe, 0x21, 0x29, 0x38, 0x05, 0x68, 0x7b, 0x1e, 0x10, 0x7f, 0x89, 0x58, - 0x96, 0xe9, 0x52, 0x95, 0x45, 0xaa, 0xe3, 0xa5, 0xbf, 0xc4, 0xa7, 0xd5, 0x25, 0x32, 0xfc, 0x2b, - 0xa3, 0xe7, 0x89, 0x37, 0xe3, 0x7d, 0xab, 0xba, 0x12, 0xb7, 0xab, 0x6a, 0xf2, 0xd8, 0x59, 0xa8, - 0xc6, 0xd7, 0xd0, 0xc9, 0xaa, 0x6e, 0x92, 0x3b, 0x8f, 0x4e, 0x17, 0xfa, 0x2c, 0xaa, 0x25, 0xe2, - 0x0d, 0xec, 0x56, 0x07, 0x53, 0x09, 0x39, 0xd4, 0x29, 0x25, 0xca, 0x50, 0x5c, 0x98, 0xc4, 0x57, - 0xe1, 0x33, 0xf6, 0xf9, 0xbc, 0x12, 0x9c, 0x06, 0x3e, 0xaa, 0x69, 0x3f, 0x0a, 0x32, 0x72, 0xc8, - 0x17, 0xf9, 0xbc, 0x1a, 0x05, 0x75, 0xec, 0x7b, 0xca, 0xe1, 0xd4, 0x4a, 0xc9, 0x93, 0x88, 0xd7, - 0x62, 0x1b, 0xd6, 0x30, 0xd5, 0x68, 0xe5, 0x6e, 0x98, 0x5b, 0x3e, 0xf0, 0x47, 0xe2, 0xd6, 0x9d, - 0x97, 0x09, 0x3a, 0x4a, 0xe4, 0x1e, 0x93, 0x7d, 0x8f, 0x7d, 0xaa, 0x20, 0x71, 0x0c, 0x3b, 0x3a, - 0x51, 0x74, 0xeb, 0x0c, 0xc6, 0xae, 0x30, 0x2a, 0x21, 0x4c, 0x52, 0x9d, 0x93, 0xdc, 0xe7, 0xe3, - 0x3f, 0xd5, 0xc9, 0x59, 0xcd, 0x9d, 0x06, 0x4a, 0xbc, 0x82, 0xad, 0x7a, 0x62, 0xa7, 0xfa, 0x8a, - 0x9c, 0xce, 0x48, 0xbe, 0x60, 0xf9, 0x66, 0xc0, 0xdf, 0x07, 0xd8, 0x0f, 0x9d, 0xc6, 0xe5, 0x6b, - 0xb2, 0xf2, 0x80, 0x5d, 0x37, 0xee, 0x5e, 0x93, 0x15, 0x23, 0xe8, 0xc7, 0x64, 0x9c, 0xbe, 0xd2, - 0xb1, 0x6f, 0xbb, 0xff, 0x55, 0x3e, 0x1b, 0xd0, 0xde, 0x1f, 0xf0, 0xe4, 0x51, 0x13, 0xf9, 0x49, - 0x72, 0x4d, 0xcb, 0xfa, 0x57, 0x70, 0x4d, 0x4b, 0xf1, 0x0d, 0xac, 0xdd, 0x60, 0x3a, 0xaf, 0x26, - 0x51, 0xff, 0x78, 0xff, 0xd1, 0x18, 0xbe, 0xff, 0x0f, 0x44, 0x95, 0xf2, 0xcd, 0xca, 0x49, 0x6b, - 0xef, 0x3b, 0xd8, 0x78, 0xd8, 0x16, 0xff, 0x92, 0x7a, 0xbb, 0x99, 0xba, 0xd7, 0xd8, 0x7d, 0xb9, - 0xce, 0x7f, 0xac, 0x6f, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0x93, 0xd4, 0x16, 0x3a, 0xd2, 0x06, - 0x00, 0x00, + // 933 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0x4f, 0x6f, 0x1b, 0xb7, + 0x13, 0x85, 0x2c, 0xdb, 0x92, 0x66, 0x25, 0x5b, 0x61, 0xec, 0x84, 0xb6, 0xe3, 0xdf, 0x4f, 0x16, + 0x90, 0x54, 0x01, 0x52, 0xa3, 0x75, 0x2f, 0x46, 0x5a, 0xa0, 0x75, 0x63, 0x1d, 0xd4, 0xc6, 0x69, + 0xb1, 0x6e, 0xd0, 0x4b, 0x01, 0x82, 0xde, 0x1d, 0x4b, 0x8c, 0xf7, 0x5f, 0x49, 0xca, 0xb2, 0xbe, + 0x47, 0x4f, 0xfd, 0xb4, 0x05, 0x67, 0xb9, 0xf2, 0x1a, 0x6e, 0x0e, 0xbd, 0x71, 0xde, 0x7b, 0x33, + 0xfb, 0xc8, 0x19, 0x72, 0xe1, 0x30, 0xca, 0x0b, 0x9d, 0x47, 0x68, 0x8c, 0x30, 0x68, 0x8c, 0xca, + 0x33, 0x61, 0xac, 0xb4, 0x78, 0x5c, 0xe8, 0xdc, 0xe6, 0xac, 0xb3, 0xa2, 0x87, 0xa7, 0x00, 0x67, + 0x91, 0x5b, 0x5d, 0x16, 0x18, 0xb1, 0x3e, 0x34, 0xe7, 0x3a, 0xe1, 0x8d, 0x41, 0x63, 0xd4, 0x09, + 0xdd, 0x92, 0x71, 0x68, 0xa5, 0x68, 0x67, 0x79, 0x6c, 0xf8, 0xda, 0xa0, 0x39, 0xea, 0x84, 0x55, + 0x38, 0xfc, 0xbb, 0x01, 0xfd, 0x32, 0xf5, 0x1c, 0xaf, 0x55, 0xa6, 0xac, 0xca, 0x33, 0xb6, 0x07, + 0x6d, 0x59, 0x28, 0x91, 0xc9, 0x14, 0x7d, 0x95, 0x96, 0x2c, 0xd4, 0x07, 0x99, 0x22, 0xdb, 0x85, + 0x4d, 0x47, 0xa9, 0x98, 0xaf, 0x11, 0xb1, 0x21, 0x0b, 0x35, 0x89, 0xd9, 0x3e, 0xb4, 0x6f, 0x51, + 0x3b, 0x8b, 0x86, 0x37, 0xe9, 0x0b, 0xab, 0x98, 0x9d, 0x42, 0x57, 0x26, 0x49, 0xbe, 0xc0, 0x58, + 0xcc, 0x75, 0x62, 0xf8, 0xfa, 0xa0, 0x39, 0x0a, 0x4e, 0x76, 0x8f, 0x57, 0xf6, 0x8f, 0xef, 0xbd, + 0x87, 0x81, 0x97, 0x7e, 0xd4, 0x89, 0x19, 0x7e, 0x0f, 0xbd, 0x1f, 0xa5, 0x51, 0xd1, 0xd9, 0xdc, + 0xce, 0xce, 0xa5, 0x95, 0xee, 0x33, 0x85, 0x34, 0x66, 0x91, 0xeb, 0xd8, 0x1b, 0x5b, 0xc5, 0x8c, + 0xc1, 0xfa, 0x4c, 0x9a, 0x99, 0xf7, 0x45, 0xeb, 0xe1, 0x11, 0xb4, 0x7e, 0xfa, 0xfd, 0x37, 0x4a, + 0x7d, 0x06, 0x9b, 0x06, 0x23, 0x8d, 0xd6, 0x27, 0xfa, 0x68, 0xf8, 0x15, 0xb4, 0x2e, 0xf2, 0x4c, + 0xd9, 0x5c, 0xb3, 0x97, 0xb0, 0x65, 0xb5, 0x9a, 0x4e, 0x51, 0x8b, 0x44, 0xa5, 0xca, 0x1a, 0xde, + 0x18, 0x34, 0x47, 0x8d, 0xb0, 0xe7, 0xd1, 0xf7, 0x04, 0x0e, 0xff, 0x0a, 0xa0, 0x7b, 0x59, 0xf6, + 0xe3, 0xd2, 0xb5, 0x83, 0x1d, 0x02, 0x24, 0xd2, 0x58, 0x11, 0xcd, 0x30, 0xba, 0xa1, 0xf2, 0xcd, + 0xb0, 0xe3, 0x90, 0x77, 0x0e, 0x60, 0x2f, 0xa0, 0x43, 0x9b, 0x92, 0x59, 0x84, 0xe4, 0xae, 0x11, + 0xde, 0x03, 0xce, 0xb6, 0x96, 0x16, 0x79, 0x93, 0x08, 0x5a, 0xbb, 0x06, 0x16, 0xa8, 0xf9, 0x3a, + 0x41, 0x6e, 0xe9, 0x1a, 0x88, 0x77, 0x85, 0xd2, 0x68, 0xf8, 0x06, 0xd5, 0xaf, 0x42, 0x76, 0x00, + 0x9d, 0x3f, 0xe7, 0xb9, 0x95, 0x22, 0x95, 0x77, 0x7c, 0x93, 0xb8, 0x36, 0x01, 0x17, 0xf2, 0x8e, + 0x1d, 0x41, 0xb7, 0x24, 0x35, 0x66, 0xb8, 0x30, 0xbc, 0x45, 0x7c, 0x40, 0x58, 0x48, 0x10, 0xfb, + 0x02, 0xb6, 0x2b, 0x49, 0x2a, 0x55, 0xa6, 0xb2, 0x29, 0x6f, 0x93, 0x6a, 0xcb, 0xab, 0x3c, 0xca, + 0xde, 0x00, 0xab, 0xd5, 0x92, 0x89, 0x20, 0xdb, 0x1d, 0xd2, 0xf6, 0xef, 0x2b, 0xca, 0x24, 0x74, + 0x5b, 0xf8, 0x00, 0x3d, 0x49, 0x5d, 0x15, 0x5a, 0x4d, 0x67, 0xd6, 0x70, 0xa0, 0xae, 0xbf, 0xae, + 0x75, 0xbd, 0x7e, 0x86, 0x7e, 0x04, 0x42, 0xd2, 0x8e, 0x33, 0xab, 0x97, 0x61, 0x57, 0xd6, 0x20, + 0x37, 0x77, 0xb9, 0x9e, 0xba, 0xb9, 0x0b, 0xca, 0xb9, 0xcb, 0xf5, 0x74, 0x12, 0xb3, 0x57, 0xb0, + 0x9d, 0xcb, 0xb9, 0x9d, 0x89, 0x28, 0x51, 0x98, 0x59, 0xc7, 0x77, 0x89, 0xef, 0x11, 0xfc, 0x8e, + 0xd0, 0x49, 0xcc, 0xc6, 0x00, 0xa5, 0xee, 0x06, 0x97, 0x86, 0xf7, 0xc8, 0xcb, 0xab, 0xcf, 0x79, + 0xf9, 0xc5, 0x29, 0x7f, 0xc6, 0xa5, 0x37, 0xd2, 0xc9, 0xab, 0x98, 0xfd, 0x00, 0xdb, 0x57, 0x6e, + 0x20, 0x05, 0xd5, 0x8a, 0xa5, 0x95, 0x7c, 0x6b, 0xd0, 0x18, 0x05, 0x27, 0xbc, 0x56, 0xeb, 0xc1, + 0xc8, 0x86, 0xbd, 0xab, 0x07, 0x13, 0xfc, 0x25, 0xb4, 0x3f, 0x2d, 0x6c, 0x99, 0xba, 0x4d, 0xa9, + 0xac, 0x96, 0xea, 0x87, 0x35, 0x6c, 0x7d, 0x5a, 0x58, 0x92, 0x1f, 0x41, 0x77, 0x96, 0xca, 0x48, + 0x60, 0x26, 0xaf, 0x12, 0x8c, 0x79, 0x7f, 0xd0, 0x18, 0xb5, 0xc3, 0xc0, 0x61, 0xe3, 0x12, 0x62, + 0xff, 0x07, 0x0a, 0x85, 0x9f, 0xee, 0x27, 0xb4, 0x7d, 0x70, 0xd0, 0x25, 0x21, 0x4e, 0xa0, 0x8c, + 0x50, 0x99, 0x8c, 0xac, 0xba, 0x45, 0xce, 0xa8, 0x04, 0x28, 0x33, 0xf1, 0x88, 0x3b, 0x44, 0x59, + 0x14, 0xc9, 0x52, 0x14, 0x79, 0xa2, 0xa2, 0xa5, 0x3b, 0xc4, 0xa7, 0xe5, 0x21, 0x12, 0xfc, 0x2b, + 0xa1, 0x93, 0xd8, 0x99, 0x71, 0xbe, 0x45, 0x35, 0x89, 0x3b, 0xe5, 0x34, 0x39, 0x6c, 0xec, 0xa7, + 0xf1, 0x0d, 0xb4, 0xd2, 0xf2, 0x36, 0xf1, 0xdd, 0x47, 0xbb, 0xf3, 0xf7, 0x2c, 0xac, 0x24, 0xec, + 0x2d, 0xec, 0x95, 0x1b, 0x13, 0x31, 0x5a, 0xa9, 0x12, 0x8c, 0x85, 0xc6, 0x28, 0xd7, 0xb1, 0x9b, + 0xc2, 0x67, 0xe4, 0xf3, 0x79, 0x29, 0x38, 0xf7, 0x7c, 0x58, 0xd1, 0xec, 0x0c, 0xda, 0x29, 0x5a, + 0x49, 0x07, 0xf9, 0x9c, 0xfa, 0xf9, 0xf2, 0x73, 0xfd, 0xbc, 0xf0, 0xba, 0xb2, 0x9d, 0xab, 0x34, + 0x77, 0xf5, 0xac, 0x9c, 0x1a, 0xce, 0xe9, 0xc1, 0xa2, 0x35, 0xdb, 0x81, 0x0d, 0x99, 0x28, 0x69, + 0xf8, 0x9e, 0x7f, 0xde, 0x5c, 0xe0, 0x76, 0x4e, 0x37, 0x7c, 0x5e, 0xc4, 0xd2, 0x62, 0xcc, 0xf7, + 0x89, 0x0c, 0x1c, 0xf6, 0xb1, 0x84, 0xd8, 0x09, 0xec, 0xaa, 0x58, 0xe0, 0x9d, 0xd5, 0x32, 0xb2, + 0xb9, 0x16, 0x31, 0xca, 0x38, 0x51, 0x19, 0xf2, 0x03, 0x3a, 0xa5, 0xa7, 0x2a, 0x1e, 0x57, 0xdc, + 0xb9, 0xa7, 0xd8, 0x6b, 0xe8, 0x57, 0x0f, 0x7b, 0xa2, 0xae, 0xd1, 0xaa, 0x14, 0xf9, 0x0b, 0x92, + 0x6f, 0x7b, 0xfc, 0xbd, 0x87, 0xdd, 0xdb, 0x54, 0xeb, 0x91, 0x42, 0xc3, 0x0f, 0xc9, 0x75, 0xad, + 0x45, 0x0a, 0x0d, 0x1b, 0x40, 0x10, 0xa1, 0xb6, 0xea, 0x5a, 0x45, 0xee, 0x76, 0xfe, 0xaf, 0xf4, + 0x59, 0x83, 0xf6, 0xff, 0x80, 0x27, 0x8f, 0xee, 0x9a, 0x7b, 0x70, 0x6e, 0x70, 0x59, 0xfd, 0x31, + 0x6e, 0x70, 0xc9, 0xbe, 0x86, 0x8d, 0x5b, 0x99, 0xcc, 0xcb, 0x07, 0x2b, 0x38, 0x39, 0x78, 0xf4, + 0x5a, 0xdf, 0xff, 0x2e, 0xc2, 0x52, 0xf9, 0x76, 0xed, 0xb4, 0xb1, 0xff, 0x1d, 0x6c, 0x3d, 0xbc, + 0x3d, 0xff, 0x52, 0x7a, 0xa7, 0x5e, 0xba, 0x53, 0xcf, 0xfe, 0x16, 0x7a, 0x0f, 0x7a, 0xf5, 0x5f, + 0x92, 0xaf, 0x36, 0xe9, 0xaf, 0xf8, 0xcd, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xdc, 0xad, 0x0c, + 0x63, 0x36, 0x07, 0x00, 0x00, } diff --git a/coprocess/proto/coprocess_mini_request_object.proto b/coprocess/proto/coprocess_mini_request_object.proto index c9f18ab63e0..bbf4c0b7ecd 100644 --- a/coprocess/proto/coprocess_mini_request_object.proto +++ b/coprocess/proto/coprocess_mini_request_object.proto @@ -19,4 +19,5 @@ message MiniRequestObject { string method = 11; string request_uri = 12; string scheme = 13; + bytes raw_body = 14; } diff --git a/coprocess/proto/coprocess_session_state.proto b/coprocess/proto/coprocess_session_state.proto index 36f080127e9..c045bfd10c6 100644 --- a/coprocess/proto/coprocess_session_state.proto +++ b/coprocess/proto/coprocess_session_state.proto @@ -60,7 +60,7 @@ message SessionState { bool enable_detailed_recording = 22; - string metadata = 23; + map metadata = 23; repeated string tags = 24; string alias = 25; diff --git a/coprocess/python/tyk/middleware.py b/coprocess/python/tyk/middleware.py index 44b84ec39be..5ad0cf5ec16 100644 --- a/coprocess/python/tyk/middleware.py +++ b/coprocess/python/tyk/middleware.py @@ -78,7 +78,9 @@ def process(self, handler, object): handler(object, object.spec) return elif handler.arg_count == 4: - object.request, object.session, object.metadata = handler(object.request, object.session, object.metadata, object.spec) + md = object.session.metadata + object.request, object.session, md = handler(object.request, object.session, md, object.spec) + object.session.metadata.MergeFrom(md) elif handler.arg_count == 3: object.request, object.session = handler(object.request, object.session, object.spec) return object diff --git a/coprocess_grpc_test.go b/coprocess_grpc_test.go index 135d4abb493..67ab30661a0 100644 --- a/coprocess_grpc_test.go +++ b/coprocess_grpc_test.go @@ -4,10 +4,13 @@ package main import ( + "bytes" "encoding/json" "io/ioutil" + "mime/multipart" "net" "net/http" + "strings" "testing" "context" @@ -18,6 +21,7 @@ import ( "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/coprocess" "github.com/TykTechnologies/tyk/test" + "github.com/TykTechnologies/tyk/user" ) const ( @@ -30,12 +34,77 @@ const ( type dispatcher struct{} +func (d *dispatcher) grpcError(object *coprocess.Object, errorMsg string) (*coprocess.Object, error) { + object.Request.ReturnOverrides.ResponseError = errorMsg + object.Request.ReturnOverrides.ResponseCode = 400 + return object, nil +} + func (d *dispatcher) Dispatch(ctx context.Context, object *coprocess.Object) (*coprocess.Object, error) { switch object.HookName { case "testPreHook1": object.Request.SetHeaders = map[string]string{ testHeaderName: testHeaderValue, } + case "testPreHook2": + contentType, found := object.Request.Headers["Content-Type"] + if !found { + return d.grpcError(object, "Content Type field not found") + } + if strings.Contains(contentType, "json") { + if len(object.Request.Body) == 0 { + return d.grpcError(object, "Body field is empty") + } + if len(object.Request.RawBody) == 0 { + return d.grpcError(object, "Raw body field is empty") + } + if strings.Compare(object.Request.Body, string(object.Request.Body)) != 0 { + return d.grpcError(object, "Raw body and body fields don't match") + } + } else if strings.Contains(contentType, "multipart") { + if len(object.Request.Body) != 0 { + return d.grpcError(object, "Body field isn't empty") + } + if len(object.Request.RawBody) == 0 { + return d.grpcError(object, "Raw body field is empty") + } + } else { + return d.grpcError(object, "Request content type should be either JSON or multipart") + } + case "testPostHook1": + testKeyValue, ok := object.Session.Metadata["testkey"] + if !ok { + return d.grpcError(object, "'testkey' not found in session metadata") + } + jsonObject := make(map[string]string) + if err := json.Unmarshal([]byte(testKeyValue), &jsonObject); err != nil { + return d.grpcError(object, "couldn't decode 'testkey' nested value") + } + nestedKeyValue, ok := jsonObject["nestedkey"] + if !ok { + return d.grpcError(object, "'nestedkey' not found in JSON object") + } + if nestedKeyValue != "nestedvalue" { + return d.grpcError(object, "'nestedvalue' value doesn't match") + } + testKey2Value, ok := object.Session.Metadata["testkey2"] + if !ok { + return d.grpcError(object, "'testkey' not found in session metadata") + } + if testKey2Value != "testvalue" { + return d.grpcError(object, "'testkey2' value doesn't match") + } + + // Check for compatibility (object.Metadata should contain the same keys as object.Session.Metadata) + for k, v := range object.Metadata { + sessionKeyValue, ok := object.Session.Metadata[k] + if !ok { + return d.grpcError(object, k+" not found in object.Session.Metadata") + } + if strings.Compare(sessionKeyValue, v) != 0 { + return d.grpcError(object, k+" doesn't match value in object.Session.Metadata") + } + } } return object, nil } @@ -50,13 +119,14 @@ func newTestGRPCServer() (s *grpc.Server) { return s } -func loadTestGRPCSpec() *APISpec { - spec := buildAndLoadAPI(func(spec *APISpec) { - spec.APIID = "999999" - spec.OrgID = "default" +func loadTestGRPCAPIs() { + buildAndLoadAPI(func(spec *APISpec) { + spec.APIID = "1" + spec.OrgID = mockOrgID spec.Auth = apidef.Auth{ AuthHeaderName: "authorization", } + spec.UseKeylessAccess = false spec.VersionData = struct { NotVersioned bool `bson:"not_versioned" json:"not_versioned"` DefaultVersion string `bson:"default_version" json:"default_version"` @@ -77,9 +147,61 @@ func loadTestGRPCSpec() *APISpec { }, Driver: apidef.GrpcDriver, } - })[0] - - return spec + }, func(spec *APISpec) { + spec.APIID = "2" + spec.OrgID = mockOrgID + spec.Auth = apidef.Auth{ + AuthHeaderName: "authorization", + } + spec.UseKeylessAccess = true + spec.VersionData = struct { + NotVersioned bool `bson:"not_versioned" json:"not_versioned"` + DefaultVersion string `bson:"default_version" json:"default_version"` + Versions map[string]apidef.VersionInfo `bson:"versions" json:"versions"` + }{ + NotVersioned: true, + Versions: map[string]apidef.VersionInfo{ + "v1": { + Name: "v1", + }, + }, + } + spec.Proxy.ListenPath = "/grpc-test-api-2/" + spec.Proxy.StripListenPath = true + spec.CustomMiddleware = apidef.MiddlewareSection{ + Pre: []apidef.MiddlewareDefinition{ + {Name: "testPreHook2"}, + }, + Driver: apidef.GrpcDriver, + } + }, func(spec *APISpec) { + spec.APIID = "3" + spec.OrgID = "default" + spec.Auth = apidef.Auth{ + AuthHeaderName: "authorization", + } + spec.UseKeylessAccess = false + spec.VersionData = struct { + NotVersioned bool `bson:"not_versioned" json:"not_versioned"` + DefaultVersion string `bson:"default_version" json:"default_version"` + Versions map[string]apidef.VersionInfo `bson:"versions" json:"versions"` + }{ + NotVersioned: true, + Versions: map[string]apidef.VersionInfo{ + "v1": { + Name: "v1", + }, + }, + } + spec.Proxy.ListenPath = "/grpc-test-api-3/" + spec.Proxy.StripListenPath = true + spec.CustomMiddleware = apidef.MiddlewareSection{ + Post: []apidef.MiddlewareDefinition{ + {Name: "testPostHook1"}, + }, + Driver: apidef.GrpcDriver, + } + }) } func startTykWithGRPC() (*tykTestServer, *grpc.Server) { @@ -95,8 +217,8 @@ func startTykWithGRPC() (*tykTestServer, *grpc.Server) { } ts := newTykTestServer(tykTestServerConfig{coprocessConfig: cfg}) - // Load a test API: - loadTestGRPCSpec() + // Load test APIs: + loadTestGRPCAPIs() return &ts, grpcServer } @@ -105,11 +227,20 @@ func TestGRPCDispatch(t *testing.T) { defer ts.Close() defer grpcServer.Stop() + keyID := createSession(func(s *user.SessionState) { + s.MetaData = map[string]interface{}{ + "testkey": map[string]interface{}{"nestedkey": "nestedvalue"}, + "testkey2": "testvalue", + } + }) + headers := map[string]string{"authorization": keyID} + t.Run("Pre Hook with SetHeaders", func(t *testing.T) { res, err := ts.Run(t, test.TestCase{ - Path: "/grpc-test-api/", - Method: http.MethodGet, - Code: http.StatusOK, + Path: "/grpc-test-api/", + Method: http.MethodGet, + Code: http.StatusOK, + Headers: headers, }) if err != nil { t.Fatalf("Request failed: %s", err.Error()) @@ -132,6 +263,46 @@ func TestGRPCDispatch(t *testing.T) { } }) + t.Run("Pre Hook with UTF-8/non-UTF-8 request data", func(t *testing.T) { + fileData := generateTestBinaryData() + var buf bytes.Buffer + multipartWriter := multipart.NewWriter(&buf) + file, err := multipartWriter.CreateFormFile("file", "test.bin") + if err != nil { + t.Fatalf("Couldn't use multipart writer: %s", err.Error()) + } + _, err = fileData.WriteTo(file) + if err != nil { + t.Fatalf("Couldn't write to multipart file: %s", err.Error()) + } + field, err := multipartWriter.CreateFormField("testfield") + if err != nil { + t.Fatalf("Couldn't use multipart writer: %s", err.Error()) + } + _, err = field.Write([]byte("testvalue")) + if err != nil { + t.Fatalf("Couldn't write to form field: %s", err.Error()) + } + err = multipartWriter.Close() + if err != nil { + t.Fatalf("Couldn't close multipart writer: %s", err.Error()) + } + + ts.Run(t, []test.TestCase{ + {Path: "/grpc-test-api-2/", Code: 200, Data: &buf, Headers: map[string]string{"Content-Type": multipartWriter.FormDataContentType()}}, + {Path: "/grpc-test-api-2/", Code: 200, Data: "{}", Headers: map[string]string{"Content-Type": "application/json"}}, + }...) + }) + + t.Run("Post Hook with metadata", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Path: "/grpc-test-api-3/", + Method: http.MethodGet, + Code: http.StatusOK, + Headers: headers, + }) + }) + } func BenchmarkGRPCDispatch(b *testing.B) { @@ -139,14 +310,18 @@ func BenchmarkGRPCDispatch(b *testing.B) { defer ts.Close() defer grpcServer.Stop() + keyID := createSession(func(s *user.SessionState) {}) + headers := map[string]string{"authorization": keyID} + b.Run("Pre Hook with SetHeaders", func(b *testing.B) { path := "/grpc-test-api/" b.ReportAllocs() for i := 0; i < b.N; i++ { ts.Run(b, test.TestCase{ - Path: path, - Method: http.MethodGet, - Code: http.StatusOK, + Path: path, + Method: http.MethodGet, + Code: http.StatusOK, + Headers: headers, }) } }) diff --git a/coprocess_helpers.go b/coprocess_helpers.go index 64b8d9a3cd3..4361ec10a0b 100644 --- a/coprocess_helpers.go +++ b/coprocess_helpers.go @@ -5,6 +5,8 @@ package main import ( "encoding/json" + "github.com/Sirupsen/logrus" + "github.com/TykTechnologies/tyk/coprocess" "github.com/TykTechnologies/tyk/user" ) @@ -54,10 +56,9 @@ func TykSessionState(session *coprocess.SessionState) *user.SessionState { } metadata := make(map[string]interface{}) - if session.Metadata != "" { - err := json.Unmarshal([]byte(session.Metadata), &metadata) - if err != nil { - log.Error("Error interpreting metadata: ", err) + if session.Metadata != nil { + for k, v := range session.Metadata { + metadata[k] = v } } @@ -129,6 +130,25 @@ func ProtoSessionState(session *user.SessionState) *coprocess.SessionState { TriggerLimits: session.Monitor.TriggerLimits, } + metadata := make(map[string]string) + if len(session.MetaData) > 0 { + for k, v := range session.MetaData { + switch v.(type) { + case string: + metadata[k] = v.(string) + default: + jsonValue, err := json.Marshal(v) + if err != nil { + log.WithFields(logrus.Fields{ + "prefix": "coprocess", + }).WithError(err).Error("Couldn't encode session metadata") + continue + } + metadata[k] = string(jsonValue) + } + } + } + return &coprocess.SessionState{ LastCheck: session.LastCheck, Allowance: session.Allowance, @@ -152,6 +172,7 @@ func ProtoSessionState(session *user.SessionState) *coprocess.SessionState { ApplyPolicies: session.ApplyPolicies, DataExpires: session.DataExpires, Monitor: monitor, + Metadata: metadata, EnableDetailedRecording: session.EnableDetailedRecording, Tags: session.Tags, Alias: session.Alias, diff --git a/coprocess_id_extractor.go b/coprocess_id_extractor.go index 03e2347487b..5f85fb64128 100644 --- a/coprocess_id_extractor.go +++ b/coprocess_id_extractor.go @@ -15,13 +15,11 @@ import ( "github.com/TykTechnologies/tyk/apidef" "github.com/TykTechnologies/tyk/regexp" - "github.com/TykTechnologies/tyk/user" ) // IdExtractor is the base interface for an ID extractor. type IdExtractor interface { ExtractAndCheck(*http.Request) (string, ReturnOverrides) - PostProcess(*http.Request, *user.SessionState, string) GenerateSessionID(string, BaseMiddleware) string } @@ -40,15 +38,6 @@ func (e *BaseExtractor) ExtractAndCheck(r *http.Request) (sessionID string, retu return "", ReturnOverrides{ResponseCode: 403, ResponseError: "Key not authorised"} } -// PostProcess sets context variables and updates the storage. -func (e *BaseExtractor) PostProcess(r *http.Request, session *user.SessionState, sessionID string) { - sessionLifetime := session.Lifetime(e.Spec.SessionLifetime) - e.Spec.SessionManager.UpdateSession(sessionID, session, sessionLifetime, false) - - ctxSetSession(r, session) - ctxSetAuthToken(r, sessionID) -} - // ExtractHeader is used when a HeaderSource is specified. func (e *BaseExtractor) ExtractHeader(r *http.Request) (headerValue string, err error) { headerName := e.Config.ExtractorConfig["header_name"].(string) @@ -101,7 +90,7 @@ func (e *BaseExtractor) Error(r *http.Request, err error, message string) (retur func (e *BaseExtractor) GenerateSessionID(input string, mw BaseMiddleware) (sessionID string) { data := []byte(input) tokenID := fmt.Sprintf("%x", md5.Sum(data)) - sessionID = mw.Spec.OrgID + tokenID + sessionID = generateToken(mw.Spec.OrgID, tokenID) return sessionID } @@ -146,11 +135,11 @@ func (e *ValueExtractor) ExtractAndCheck(r *http.Request) (sessionID string, ret sessionID = e.GenerateSessionID(extractorOutput, e.BaseMid) - previousSession, keyExists := e.BaseMid.CheckSessionAndIdentityForValidKey(sessionID) + previousSession, keyExists := e.BaseMid.CheckSessionAndIdentityForValidKey(sessionID, r) if keyExists { if previousSession.IdExtractorDeadline > time.Now().Unix() { - e.PostProcess(r, &previousSession, sessionID) + ctxSetSession(r, &previousSession, sessionID, true) returnOverrides = ReturnOverrides{ ResponseCode: 200, } @@ -219,11 +208,11 @@ func (e *RegexExtractor) ExtractAndCheck(r *http.Request) (SessionID string, ret } SessionID = e.GenerateSessionID(regexOutput[e.cfg.RegexMatchIndex], e.BaseMid) - previousSession, keyExists := e.BaseMid.CheckSessionAndIdentityForValidKey(SessionID) + previousSession, keyExists := e.BaseMid.CheckSessionAndIdentityForValidKey(SessionID, r) if keyExists { if previousSession.IdExtractorDeadline > time.Now().Unix() { - e.PostProcess(r, &previousSession, SessionID) + ctxSetSession(r, &previousSession, SessionID, true) returnOverrides = ReturnOverrides{ ResponseCode: 200, } @@ -295,10 +284,10 @@ func (e *XPathExtractor) ExtractAndCheck(r *http.Request) (SessionID string, ret SessionID = e.GenerateSessionID(output, e.BaseMid) - previousSession, keyExists := e.BaseMid.CheckSessionAndIdentityForValidKey(SessionID) + previousSession, keyExists := e.BaseMid.CheckSessionAndIdentityForValidKey(SessionID, r) if keyExists { if previousSession.IdExtractorDeadline > time.Now().Unix() { - e.PostProcess(r, &previousSession, SessionID) + ctxSetSession(r, &previousSession, SessionID, true) returnOverrides = ReturnOverrides{ ResponseCode: 200, } diff --git a/coprocess_id_extractor_test.go b/coprocess_id_extractor_test.go index 707861d208c..c23b69830eb 100644 --- a/coprocess_id_extractor_test.go +++ b/coprocess_id_extractor_test.go @@ -13,8 +13,6 @@ import ( ) const ( - extractorTestOrgID = "testorg" - extractorValueInput = "testkey" extractorRegexExpr = "prefix-(.*)" @@ -41,7 +39,7 @@ func createSpecTestFrom(t testing.TB, def *apidef.APIDefinition) *APISpec { func prepareExtractor(t testing.TB, extractorSource apidef.IdExtractorSource, extractorType apidef.IdExtractorType, config map[string]interface{}) (IdExtractor, *APISpec) { def := &apidef.APIDefinition{ - OrgID: extractorTestOrgID, + OrgID: mockOrgID, CustomMiddleware: apidef.MiddlewareSection{ IdExtractor: apidef.MiddlewareIdExtractor{ ExtractFrom: extractorSource, @@ -87,7 +85,7 @@ func prepareExtractorFormRequest(values map[string]string) *http.Request { func generateSessionID(input string) string { data := []byte(input) tokenID := fmt.Sprintf("%x", md5.Sum(data)) - return extractorTestOrgID + tokenID + return generateToken(mockOrgID, tokenID) } func TestValueExtractor(t *testing.T) { @@ -112,7 +110,7 @@ func TestValueExtractor(t *testing.T) { if sessionID != testSessionID { t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) } - if !strings.HasPrefix(sessionID, spec.OrgID) { + if storage.TokenOrg(sessionID) != spec.OrgID { t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) } if overrides.ResponseCode != 0 { @@ -139,7 +137,7 @@ func TestValueExtractor(t *testing.T) { if sessionID != testSessionID { t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) } - if !strings.HasPrefix(sessionID, spec.OrgID) { + if storage.TokenOrg(sessionID) != spec.OrgID { t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) } if overrides.ResponseCode != 0 { @@ -172,7 +170,7 @@ func TestRegexExtractor(t *testing.T) { if sessionID != testSessionID { t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) } - if !strings.HasPrefix(sessionID, spec.OrgID) { + if storage.TokenOrg(sessionID) != spec.OrgID { t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) } if overrides.ResponseCode != 0 { @@ -200,7 +198,7 @@ func TestRegexExtractor(t *testing.T) { if sessionID != testSessionID { t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) } - if !strings.HasPrefix(sessionID, spec.OrgID) { + if storage.TokenOrg(sessionID) != spec.OrgID { t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) } if overrides.ResponseCode != 0 { @@ -229,7 +227,7 @@ func TestRegexExtractor(t *testing.T) { if sessionID != testSessionID { t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) } - if !strings.HasPrefix(sessionID, spec.OrgID) { + if storage.TokenOrg(sessionID) != spec.OrgID { t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) } if overrides.ResponseCode != 0 { @@ -261,7 +259,7 @@ func TestXPathExtractor(t *testing.T) { if sessionID != testSessionID { t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) } - if !strings.HasPrefix(sessionID, spec.OrgID) { + if storage.TokenOrg(sessionID) != spec.OrgID { t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) } if overrides.ResponseCode != 0 { @@ -288,7 +286,7 @@ func TestXPathExtractor(t *testing.T) { if sessionID != testSessionID { t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) } - if !strings.HasPrefix(sessionID, spec.OrgID) { + if storage.TokenOrg(sessionID) != spec.OrgID { t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) } if overrides.ResponseCode != 0 { @@ -316,7 +314,7 @@ func TestXPathExtractor(t *testing.T) { if sessionID != testSessionID { t.Fatalf("session ID doesn't match, expected %s, got %s", testSessionID, sessionID) } - if !strings.HasPrefix(sessionID, spec.OrgID) { + if storage.TokenOrg(sessionID) != spec.OrgID { t.Fatalf("session ID doesn't contain the org ID, got %s", sessionID) } if overrides.ResponseCode != 0 { diff --git a/coprocess_python_test.go b/coprocess_python_test.go index 86bbcd5674c..3db06979c32 100644 --- a/coprocess_python_test.go +++ b/coprocess_python_test.go @@ -4,11 +4,14 @@ package main import ( + "bytes" + "mime/multipart" "testing" "time" "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/test" + "github.com/TykTechnologies/tyk/user" ) var pythonBundleWithAuthCheck = map[string]string{ @@ -31,7 +34,6 @@ from gateway import TykGateway as tyk @Hook def MyAuthHook(request, session, metadata, spec): - print("MyAuthHook is called") auth_header = request.get_header('Authorization') if auth_header == 'valid_token': session.rate = 1000.0 @@ -42,6 +44,92 @@ def MyAuthHook(request, session, metadata, spec): `, } +var pythonBundleWithPostHook = map[string]string{ + "manifest.json": ` + { + "file_list": [ + "middleware.py" + ], + "custom_middleware": { + "driver": "python", + "post": [{ + "name": "MyPostHook" + }] + } + } + `, + "middleware.py": ` +from tyk.decorators import * +from gateway import TykGateway as tyk +import json + +@Hook +def MyPostHook(request, session, spec): + if "testkey" not in session.metadata.keys(): + request.object.return_overrides.response_code = 400 + request.object.return_overrides.response_error = "'testkey' not found in metadata" + return request, session + nested_data = json.loads(session.metadata["testkey"]) + if "nestedkey" not in nested_data: + request.object.return_overrides.response_code = 400 + request.object.return_overrides.response_error = "'nestedkey' not found in nested metadata" + return request, session + if "stringkey" not in session.metadata.keys(): + request.object.return_overrides.response_code = 400 + request.object.return_overrides.response_error = "'stringkey' not found in metadata" + return request, session + stringkey = session.metadata["stringkey"] + if stringkey != "testvalue": + request.object.return_overrides.response_code = 400 + request.object.return_overrides.response_error = "'stringkey' value doesn't match" + return request, session + return request, session + +`, +} + +var pythonBundleWithPreHook = map[string]string{ + "manifest.json": ` + { + "file_list": [ + "middleware.py" + ], + "custom_middleware": { + "driver": "python", + "pre": [{ + "name": "MyPreHook" + }] + } + } + `, + "middleware.py": ` +from tyk.decorators import * +from gateway import TykGateway as tyk + +@Hook +def MyPreHook(request, session, metadata, spec): + content_type = request.get_header("Content-Type") + if "json" in content_type: + if len(request.object.raw_body) <= 0: + request.object.return_overrides.response_code = 400 + request.object.return_overrides.response_error = "Raw body field is empty" + return request, session, metadata + if "{}" not in request.object.body: + request.object.return_overrides.response_code = 400 + request.object.return_overrides.response_error = "Body field doesn't match" + return request, session, metadata + if "multipart" in content_type: + if len(request.object.body) != 0: + request.object.return_overrides.response_code = 400 + request.object.return_overrides.response_error = "Body field isn't empty" + if len(request.object.raw_body) <= 0: + request.object.return_overrides.response_code = 400 + request.object.return_overrides.response_error = "Raw body field is empty" + return request, session, metadata + +`, +} + func TestPythonBundles(t *testing.T) { ts := newTykTestServer(tykTestServerConfig{ coprocessConfig: config.CoProcessConfig{ @@ -49,14 +137,16 @@ func TestPythonBundles(t *testing.T) { }}) defer ts.Close() - bundleID := registerBundle("python_with_auth_check", pythonBundleWithAuthCheck) + authCheckBundle := registerBundle("python_with_auth_check", pythonBundleWithAuthCheck) + postHookBundle := registerBundle("python_with_post_hook", pythonBundleWithPostHook) + preHookBundle := registerBundle("python_with_pre_hook", pythonBundleWithPreHook) t.Run("Single-file bundle with authentication hook", func(t *testing.T) { buildAndLoadAPI(func(spec *APISpec) { spec.Proxy.ListenPath = "/test-api/" spec.UseKeylessAccess = false spec.EnableCoProcessAuth = true - spec.CustomMiddlewareBundle = bundleID + spec.CustomMiddlewareBundle = authCheckBundle spec.VersionData.NotVersioned = true }) @@ -70,4 +160,71 @@ func TestPythonBundles(t *testing.T) { {Path: "/test-api/", Code: 403, Headers: invalidAuth}, }...) }) + + t.Run("Single-file bundle with post hook", func(t *testing.T) { + + keyID := createSession(func(s *user.SessionState) { + s.MetaData = map[string]interface{}{ + "testkey": map[string]interface{}{"nestedkey": "nestedvalue"}, + "stringkey": "testvalue", + } + }) + + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/test-api-2/" + spec.UseKeylessAccess = false + spec.EnableCoProcessAuth = false + spec.CustomMiddlewareBundle = postHookBundle + spec.VersionData.NotVersioned = true + }) + + time.Sleep(1 * time.Second) + + auth := map[string]string{"Authorization": keyID} + + ts.Run(t, []test.TestCase{ + {Path: "/test-api-2/", Code: 200, Headers: auth}, + }...) + }) + + t.Run("Single-file bundle with pre hook and UTF-8/non-UTF-8 request data", func(t *testing.T) { + buildAndLoadAPI(func(spec *APISpec) { + spec.Proxy.ListenPath = "/test-api-2/" + spec.UseKeylessAccess = true + spec.EnableCoProcessAuth = false + spec.CustomMiddlewareBundle = preHookBundle + spec.VersionData.NotVersioned = true + }) + + time.Sleep(1 * time.Second) + + fileData := generateTestBinaryData() + var buf bytes.Buffer + multipartWriter := multipart.NewWriter(&buf) + file, err := multipartWriter.CreateFormFile("file", "test.bin") + if err != nil { + t.Fatalf("Couldn't use multipart writer: %s", err.Error()) + } + _, err = fileData.WriteTo(file) + if err != nil { + t.Fatalf("Couldn't write to multipart file: %s", err.Error()) + } + field, err := multipartWriter.CreateFormField("testfield") + if err != nil { + t.Fatalf("Couldn't use multipart writer: %s", err.Error()) + } + _, err = field.Write([]byte("testvalue")) + if err != nil { + t.Fatalf("Couldn't write to form field: %s", err.Error()) + } + err = multipartWriter.Close() + if err != nil { + t.Fatalf("Couldn't close multipart writer: %s", err.Error()) + } + + ts.Run(t, []test.TestCase{ + {Path: "/test-api-2/", Code: 200, Data: &buf, Headers: map[string]string{"Content-Type": multipartWriter.FormDataContentType()}}, + {Path: "/test-api-2/", Code: 200, Data: "{}", Headers: map[string]string{"Content-Type": "application/json"}}, + }...) + }) } diff --git a/gateway_test.go b/gateway_test.go index 66606c990ae..7301cffd71b 100644 --- a/gateway_test.go +++ b/gateway_test.go @@ -24,6 +24,7 @@ import ( "gopkg.in/vmihailenco/msgpack.v2" "github.com/TykTechnologies/tyk/apidef" + "github.com/TykTechnologies/tyk/cli" "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/storage" "github.com/TykTechnologies/tyk/test" @@ -46,6 +47,7 @@ var ( ) const defaultListenPort = 8080 +const mockOrgID = "507f1f77bcf86cd799439011" var defaultTestConfig config.Config var testServerRouter *mux.Router @@ -130,6 +132,8 @@ func TestMain(m *testing.M) { panic(err) } + cli.Init(VERSION, confPaths) + initialiseSystem() // Small part of start() loadAPIEndpoints(mainRouter) @@ -214,8 +218,8 @@ func createSpecTest(t testing.TB, def string) *APISpec { return spec } -func testKey(t testing.TB, name string) string { - return fmt.Sprintf("%s-%s", t.Name(), name) +func testKey(testName string, name string) string { + return fmt.Sprintf("%s-%s", testName, name) } func testReqBody(t testing.TB, body interface{}) io.Reader { @@ -712,8 +716,8 @@ func TestControlListener(t *testing.T) { } func TestHttpPprof(t *testing.T) { - old := httpProfile - defer func() { httpProfile = old }() + old := cli.HTTPProfile + defer func() { cli.HTTPProfile = old }() ts := newTykTestServer(tykTestServerConfig{ sepatateControlAPI: true, @@ -725,7 +729,7 @@ func TestHttpPprof(t *testing.T) { }...) ts.Close() - *httpProfile = true + *cli.HTTPProfile = true ts.Start() ts.Run(t, []test.TestCase{ @@ -1231,7 +1235,6 @@ func TestKeepAliveConns(t *testing.T) { // for the API. Meaning that a single token cannot reduce service availability for other tokens by simply going over the // API's global rate limit. func TestRateLimitForAPIAndRateLimitAndQuotaCheck(t *testing.T) { - globalCfg := config.Global() globalCfg.EnableNonTransactionalRateLimiter = false globalCfg.EnableSentinelRateLImiter = true diff --git a/handler_error.go b/handler_error.go index 69ca1ba4b81..c503099a8e9 100644 --- a/handler_error.go +++ b/handler_error.go @@ -31,6 +31,7 @@ type ErrorHandler struct { // HandleError is the actual error handler and will store the error details in analytics if analytics processing is enabled. func (e *ErrorHandler) HandleError(w http.ResponseWriter, r *http.Request, errMsg string, errCode int) { + defer e.Base().UpdateRequestSession(r) var templateExtension string var contentType string diff --git a/handler_success.go b/handler_success.go index ec1eebaa99b..50ad855d686 100644 --- a/handler_success.go +++ b/handler_success.go @@ -22,7 +22,9 @@ import ( // these to be implemented and is lifted pretty much from docs const ( SessionData = iota - AuthHeaderValue + UpdateSession + AuthToken + HashedAuthToken VersionData VersionDefault OrgSessionContext @@ -240,9 +242,10 @@ func recordDetail(r *http.Request, globalConf config.Config) bool { // Spec states the path is Ignored func (s *SuccessHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) *http.Response { log.Debug("Started proxy") + defer s.Base().UpdateRequestSession(r) versionDef := s.Spec.VersionDefinition - if versionDef.Location == "url" && versionDef.StripPath { + if !s.Spec.VersionData.NotVersioned && versionDef.Location == "url" && versionDef.StripPath { part := s.Spec.getVersionFromRequest(r) log.Info("Stripping version from url: ", part) @@ -291,7 +294,7 @@ func (s *SuccessHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) *http func (s *SuccessHandler) ServeHTTPWithCache(w http.ResponseWriter, r *http.Request) *http.Response { versionDef := s.Spec.VersionDefinition - if versionDef.Location == "url" && versionDef.StripPath { + if !s.Spec.VersionData.NotVersioned && versionDef.Location == "url" && versionDef.StripPath { part := s.Spec.getVersionFromRequest(r) log.Info("Stripping version from url: ", part) diff --git a/helpers_test.go b/helpers_test.go index 11f50511b9f..ff0f77f74b8 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -2,12 +2,15 @@ package main import ( "archive/zip" + "bytes" "context" "crypto/tls" + "encoding/binary" "encoding/json" "fmt" "io" "io/ioutil" + "math/rand" "net" "net/http" "net/url" @@ -29,6 +32,7 @@ import ( "github.com/TykTechnologies/tyk/apidef" "github.com/TykTechnologies/tyk/config" + "github.com/TykTechnologies/tyk/storage" "github.com/TykTechnologies/tyk/test" "github.com/TykTechnologies/tyk/user" ) @@ -71,6 +75,7 @@ func bundleHandleFunc(w http.ResponseWriter, r *http.Request) { type testHttpResponse struct { Method string + URI string Url string Body string Headers map[string]string @@ -134,6 +139,7 @@ func testHttpHandler() *mux.Router { err := json.NewEncoder(w).Encode(testHttpResponse{ Method: r.Method, + URI: r.RequestURI, Url: r.URL.String(), Headers: firstVals(r.Header), Form: firstVals(r.Form), @@ -189,16 +195,16 @@ func withAuth(r *http.Request) *http.Request { // TODO: replace with /tyk/keys/create call func createSession(sGen ...func(s *user.SessionState)) string { - key := keyGen.GenerateAuthKey("") + key := generateToken("", "") session := createStandardSession() if len(sGen) > 0 { sGen[0](session) } if session.Certificate != "" { - key = session.Certificate + key = generateToken("", session.Certificate) } - FallbackKeySesionManager.UpdateSession(key, session, 60, false) + FallbackKeySesionManager.UpdateSession(storage.HashKey(key), session, 60, config.Global().HashKeys) return key } @@ -407,7 +413,7 @@ func (s *tykTestServer) Run(t testing.TB, testCases ...test.TestCase) (*http.Res respCopy := copyResponse(lastResponse) if lastError = test.AssertResponse(respCopy, tc); lastError != nil { - t.Errorf("[%d] %s. %s", ti, lastError.Error(), string(tcJSON)) + t.Errorf("[%d] %s. %s\n", ti, lastError.Error(), string(tcJSON)) } delay := tc.Delay @@ -728,3 +734,17 @@ func initProxy(proto string, tlsConfig *tls.Config) *httpProxyHandler { return proxy } + +func generateTestBinaryData() (buf *bytes.Buffer) { + buf = new(bytes.Buffer) + type testData struct { + a float32 + b float64 + c uint32 + } + for i := 0; i < 10; i++ { + s := &testData{rand.Float32(), rand.Float64(), rand.Uint32()} + binary.Write(buf, binary.BigEndian, s) + } + return buf +} diff --git a/install/data/tyk.self_contained.conf b/install/data/tyk.self_contained.conf index e5658799425..f315094def5 100644 --- a/install/data/tyk.self_contained.conf +++ b/install/data/tyk.self_contained.conf @@ -20,10 +20,10 @@ "type": "", "ignored_ips": [], "normalise_urls": { - "enabled": true, - "normalise_uuids": true, - "normalise_numbers": true, - "custom_patterns": [] + "enabled": true, + "normalise_uuids": true, + "normalise_numbers": true, + "custom_patterns": [] } }, "health_check": { @@ -33,12 +33,12 @@ "optimisations_use_async_session_write": true, "allow_master_keys": false, "policies": { - "policy_source": "file", - "policy_record_name": "policies" - }, + "policy_source": "file", + "policy_record_name": "policies" + }, "hash_keys": true, "suppress_redis_signal_reload": false, - "close_connections": true, + "close_connections": false, "enable_non_transactional_rate_limiter": true, "enable_sentinel_rate_limiter": false, "local_session_cache": { @@ -54,7 +54,7 @@ } }, "http_server_options": { - "enable_websockets": true + "enable_websockets": true }, "hostname": "", "enable_custom_domains": true, @@ -73,5 +73,5 @@ "bundle_base_url": "", "global_session_lifetime": 100, "force_global_session_lifetime": false, - "max_idle_connections_per_host": 100 + "max_idle_connections_per_host": 500 } diff --git a/install/data/tyk.with_dash.conf b/install/data/tyk.with_dash.conf index c7f25e580c0..9ff3e0b9b4e 100644 --- a/install/data/tyk.with_dash.conf +++ b/install/data/tyk.with_dash.conf @@ -5,9 +5,9 @@ "template_path": "/opt/tyk-gateway/templates", "use_db_app_configs": true, "db_app_conf_options": { - "connection_string": "", - "node_is_segmented": false, - "tags": [] + "connection_string": "", + "node_is_segmented": false, + "tags": [] }, "disable_dashboard_zeroconf": false, "app_path": "/opt/tyk-gateway/apps", @@ -30,11 +30,11 @@ "enable_geo_ip": false, "geo_ip_db_path": "", "normalise_urls": { - "enabled": true, - "normalise_uuids": true, - "normalise_numbers": true, - "custom_patterns": [] - } + "enabled": true, + "normalise_uuids": true, + "normalise_numbers": true, + "custom_patterns": [] + } }, "health_check": { "enable_health_checks": false, @@ -51,7 +51,7 @@ "hash_keys": true, "suppress_redis_signal_reload": false, "use_redis_log": true, - "close_connections": true, + "close_connections": false, "enable_non_transactional_rate_limiter": true, "enable_sentinel_rate_limiter": false, "experimental_process_org_off_thread": false, @@ -59,7 +59,7 @@ "disable_cached_session_state": false }, "http_server_options": { - "enable_websockets": true + "enable_websockets": true }, "uptime_tests": { "disable": false, @@ -87,5 +87,5 @@ "bundle_base_url": "", "global_session_lifetime": 100, "force_global_session_lifetime": false, - "max_idle_connections_per_host": 100 + "max_idle_connections_per_host": 500 } diff --git a/instrumentation_handlers.go b/instrumentation_handlers.go index c96ba501d97..0c441dccd9f 100644 --- a/instrumentation_handlers.go +++ b/instrumentation_handlers.go @@ -9,6 +9,7 @@ import ( "github.com/gocraft/health" + "github.com/TykTechnologies/tyk/cli" "github.com/TykTechnologies/tyk/request" "github.com/TykTechnologies/tyk/config" @@ -20,7 +21,7 @@ var instrument = health.NewStream() // setupInstrumentation handles all the intialisation of the instrumentation handler func setupInstrumentation() { switch { - case *logInstrumentation: + case *cli.LogInstrumentation: case os.Getenv("TYK_INSTRUMENTATION") == "1": default: return diff --git a/main.go b/main.go index 54a8f14c7fe..d56ca3a60c5 100644 --- a/main.go +++ b/main.go @@ -35,14 +35,13 @@ import ( "github.com/lonelycode/osin" "github.com/rs/cors" "github.com/satori/go.uuid" - "gopkg.in/alecthomas/kingpin.v2" "rsc.io/letsencrypt" "github.com/TykTechnologies/goagain" "github.com/TykTechnologies/tyk/apidef" "github.com/TykTechnologies/tyk/certs" + cli "github.com/TykTechnologies/tyk/cli" "github.com/TykTechnologies/tyk/config" - "github.com/TykTechnologies/tyk/lint" logger "github.com/TykTechnologies/tyk/log" "github.com/TykTechnologies/tyk/regexp" "github.com/TykTechnologies/tyk/storage" @@ -86,27 +85,6 @@ var ( runningTests = false - version = kingpin.Version(VERSION) - help = kingpin.CommandLine.HelpFlag.Short('h') - conf = kingpin.Flag("conf", "load a named configuration file").PlaceHolder("FILE").String() - port = kingpin.Flag("port", "listen on PORT (overrides config file)").String() - memProfile = kingpin.Flag("memprofile", "generate a memory profile").Bool() - cpuProfile = kingpin.Flag("cpuprofile", "generate a cpu profile").Bool() - blockProfile = kingpin.Flag("blockprofile", "generate a block profile").Bool() - mutexProfile = kingpin.Flag("mutexprofile", "generate a mutex profile").Bool() - httpProfile = kingpin.Flag("httpprofile", "expose runtime profiling data via HTTP").Bool() - debugMode = kingpin.Flag("debug", "enable debug mode").Bool() - importBlueprint = kingpin.Flag("import-blueprint", "import an API Blueprint file").PlaceHolder("FILE").String() - importSwagger = kingpin.Flag("import-swagger", "import a Swagger file").PlaceHolder("FILE").String() - createAPI = kingpin.Flag("create-api", "creates a new API definition from the blueprint").Bool() - orgID = kingpin.Flag("org-id", "assign the API Definition to this org_id (required with create-api").String() - upstreamTarget = kingpin.Flag("upstream-target", "set the upstream target for the definition").PlaceHolder("URL").String() - asMock = kingpin.Flag("as-mock", "creates the API as a mock based on example fields").Bool() - forAPI = kingpin.Flag("for-api", "adds blueprint to existing API Definition as version").PlaceHolder("PATH").String() - asVersion = kingpin.Flag("as-version", "the version number to use when inserting").PlaceHolder("VERSION").String() - logInstrumentation = kingpin.Flag("log-intrumentation", "output intrumentation output to stdout").Bool() - subcmd = kingpin.Arg("subcmd", "run a Tyk subcommand i.e. lint").String() - // confPaths is the series of paths to try to use as config files. The // first one to exist will be used. If none exists, a default config // will be written to the first path in the list. @@ -375,7 +353,7 @@ func loadAPIEndpoints(muxer *mux.Router) { mainLog.Info("Control API hostname set: ", hostname) } - if *httpProfile { + if *cli.HTTPProfile { muxer.HandleFunc("/debug/pprof/profile", pprof_http.Profile) muxer.HandleFunc("/debug/pprof/{_:.*}", pprof_http.Index) } @@ -777,26 +755,6 @@ func setupLogger() { } func initialiseSystem() error { - - // Enable command mode - for _, opt := range commandModeOptions { - switch x := opt.(type) { - case *string: - if *x == "" { - continue - } - case *bool: - if !*x { - continue - } - default: - panic("unexpected type") - } - handleCommandModeArgs() - os.Exit(0) - - } - if runningTests && os.Getenv("TYK_LOGLEVEL") == "" { // `go test` without TYK_LOGLEVEL set defaults to no log // output @@ -804,14 +762,14 @@ func initialiseSystem() error { log.Out = ioutil.Discard gorpc.SetErrorLogger(func(string, ...interface{}) {}) stdlog.SetOutput(ioutil.Discard) - } else if *debugMode { + } else if *cli.DebugMode { log.Level = logrus.DebugLevel mainLog.Debug("Enabling debug-level output") } - if *conf != "" { - mainLog.Debugf("Using %s for configuration", *conf) - confPaths = []string{*conf} + if *cli.Conf != "" { + mainLog.Debugf("Using %s for configuration", *cli.Conf) + confPaths = []string{*cli.Conf} } else { mainLog.Debug("No configuration file defined, will try to use default (tyk.conf)") } @@ -828,7 +786,7 @@ func initialiseSystem() error { config.SetGlobal(globalConf) } - if os.Getenv("TYK_LOGLEVEL") == "" && !*debugMode { + if os.Getenv("TYK_LOGLEVEL") == "" && !*cli.DebugMode { level := strings.ToLower(config.Global().LogLevel) switch level { case "", "info": @@ -850,8 +808,8 @@ func initialiseSystem() error { setupGlobals() - if *port != "" { - portNum, err := strconv.Atoi(*port) + if *cli.Port != "" { + portNum, err := strconv.Atoi(*cli.Port) if err != nil { mainLog.Error("Port specified in flags must be a number: ", err) } else { @@ -926,24 +884,8 @@ func getGlobalStorageHandler(keyPrefix string, hashKeys bool) storage.Handler { } func main() { - kingpin.Parse() - - if *subcmd == "lint" { - path, lines, err := lint.Run(confPaths) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - if len(lines) == 0 { - fmt.Printf("found no issues in %s\n", path) - return - } - fmt.Printf("issues found in %s:\n", path) - for _, line := range lines { - fmt.Println(line) - } - os.Exit(1) - } + cli.Init(VERSION, confPaths) + cli.Parse() NodeID = "solo-" + uuid.NewV4().String() @@ -996,7 +938,7 @@ func main() { } mainLog.Info("Redis connection pools are ready") - if *memProfile { + if *cli.MemProfile { mainLog.Debug("Memory profiling active") var err error if memProfFile, err = os.Create("tyk.mprof"); err != nil { @@ -1004,7 +946,7 @@ func main() { } defer memProfFile.Close() } - if *cpuProfile { + if *cli.CPUProfile { mainLog.Info("Cpu profiling active") cpuProfFile, err := os.Create("tyk.prof") if err != nil { @@ -1013,11 +955,11 @@ func main() { pprof.StartCPUProfile(cpuProfFile) defer pprof.StopCPUProfile() } - if *blockProfile { + if *cli.BlockProfile { mainLog.Info("Block profiling active") runtime.SetBlockProfileRate(1) } - if *mutexProfile { + if *cli.MutexProfile { mainLog.Info("Mutex profiling active") runtime.SetMutexProfileFraction(1) } @@ -1073,7 +1015,7 @@ func main() { } func writeProfiles() { - if *blockProfile { + if *cli.BlockProfile { f, err := os.Create("tyk.blockprof") if err != nil { panic(err) @@ -1083,7 +1025,7 @@ func writeProfiles() { } f.Close() } - if *mutexProfile { + if *cli.MutexProfile { f, err := os.Create("tyk.mutexprof") if err != nil { panic(err) diff --git a/middleware.go b/middleware.go index 2b324034646..77c7b114653 100644 --- a/middleware.go +++ b/middleware.go @@ -99,6 +99,8 @@ func createMiddleware(mw TykMiddleware) func(http.Handler) http.Handler { // No error, carry on... meta["bypass"] = "1" h.ServeHTTP(w, r) + } else { + mw.Base().UpdateRequestSession(r) } job.TimingKv("exec_time", time.Since(startTime).Nanoseconds(), meta) @@ -149,6 +151,8 @@ func (t BaseMiddleware) OrgSession(key string) (user.SessionState, bool) { log.Debug("Setting data expiry: ", session.OrgID) go t.SetOrgExpiry(session.OrgID, session.DataExpires) } + + session.SetKeyHash(storage.HashKey(key)) return session, found } @@ -168,6 +172,35 @@ func (t BaseMiddleware) OrgSessionExpiry(orgid string) int64 { return cachedVal.(int64) } +func (t BaseMiddleware) UpdateRequestSession(r *http.Request) bool { + session := ctxGetSession(r) + token := ctxGetAuthToken(r) + + if session == nil || token == "" { + return false + } + + if !ctxSessionUpdateScheduled(r) { + return false + } + + lifetime := session.Lifetime(t.Spec.SessionLifetime) + if err := t.Spec.SessionManager.UpdateSession(token, session, lifetime, false); err != nil { + log.WithError(err).Error("Can't update session") + return false + } + + // Set context state back + // Useful for benchmarks when request object stays same + ctxDisableSessionUpdate(r) + + if !t.Spec.GlobalConfig.LocalSessionCache.DisableCacheSessionState { + SessionCache.Set(session.KeyHash(), *session, cache.DefaultExpiration) + } + + return true +} + // ApplyPolicies will check if any policies are loaded. If any are, it // will overwrite the session state to use the policy values. func (t BaseMiddleware) ApplyPolicies(key string, session *user.SessionState) error { @@ -279,13 +312,13 @@ func (t BaseMiddleware) ApplyPolicies(key string, session *user.SessionState) er } session.AccessRights = rights - // Update the session in the session manager in case it gets called again - return t.Spec.SessionManager.UpdateSession(key, session, session.Lifetime(t.Spec.SessionLifetime), false) + + return nil } // CheckSessionAndIdentityForValidKey will check first the Session store for a valid key, if not found, it will try // the Auth Handler, if not found it will fail -func (t BaseMiddleware) CheckSessionAndIdentityForValidKey(key string) (user.SessionState, bool) { +func (t BaseMiddleware) CheckSessionAndIdentityForValidKey(key string, r *http.Request) (user.SessionState, bool) { minLength := t.Spec.GlobalConfig.MinTokenLength if minLength == 0 { // See https://github.com/TykTechnologies/tyk/issues/1681 @@ -302,6 +335,7 @@ func (t BaseMiddleware) CheckSessionAndIdentityForValidKey(key string) (user.Ses if t.Spec.GlobalConfig.HashKeys { cacheKey = storage.HashStr(key) } + // Check in-memory cache if !t.Spec.GlobalConfig.LocalSessionCache.DisableCacheSessionState { cachedVal, found := SessionCache.Get(cacheKey) @@ -319,6 +353,7 @@ func (t BaseMiddleware) CheckSessionAndIdentityForValidKey(key string) (user.Ses log.Debug("Querying keystore") session, found := t.Spec.SessionManager.SessionDetail(key, false) if found { + session.SetKeyHash(cacheKey) // If exists, assume it has been authorized and pass on // cache it if !t.Spec.GlobalConfig.LocalSessionCache.DisableCacheSessionState { @@ -337,6 +372,7 @@ func (t BaseMiddleware) CheckSessionAndIdentityForValidKey(key string) (user.Ses // 2. If not there, get it from the AuthorizationHandler session, found = t.Spec.AuthManager.KeyAuthorised(key) if found { + session.SetKeyHash(cacheKey) // If not in Session, and got it from AuthHandler, create a session with a new TTL log.Info("Recreating session for key: ", key) @@ -351,9 +387,7 @@ func (t BaseMiddleware) CheckSessionAndIdentityForValidKey(key string) (user.Ses } log.Debug("Lifetime is: ", session.Lifetime(t.Spec.SessionLifetime)) - // Need to set this in order for the write to work! - session.LastUpdated = time.Now().String() - t.Spec.SessionManager.UpdateSession(key, &session, session.Lifetime(t.Spec.SessionLifetime), false) + ctxScheduleSessionUpdate(r) } return session, found diff --git a/multiauth_test.go b/multiauth_test.go index 1dfb8dc4cb7..dab548e2533 100644 --- a/multiauth_test.go +++ b/multiauth_test.go @@ -12,6 +12,7 @@ import ( "github.com/justinas/alice" "github.com/lonelycode/go-uuid/uuid" + "github.com/pmylund/go-cache" "github.com/TykTechnologies/tyk/user" ) @@ -77,7 +78,7 @@ func getMultiAuthStandardAndBasicAuthChain(spec *APISpec) http.Handler { chain := alice.New(mwList( &IPWhiteListMiddleware{baseMid}, &IPBlackListMiddleware{BaseMiddleware: baseMid}, - &BasicAuthKeyIsValid{baseMid}, + &BasicAuthKeyIsValid{baseMid, cache.New(60*time.Second, 60*time.Minute)}, &AuthKey{baseMid}, &VersionCheck{BaseMiddleware: baseMid}, &KeyExpired{baseMid}, @@ -99,8 +100,9 @@ func testPrepareMultiSessionBA(t testing.TB, isBench bool) (*APISpec, *http.Requ username = "0987876" } password := "TEST" + keyName := generateToken("default", username) // Basic auth sessions are stored as {org-id}{username}, so we need to append it here when we create the session. - spec.SessionManager.UpdateSession("default"+username, baSession, 60, false) + spec.SessionManager.UpdateSession(keyName, baSession, 60, false) // Create key session := createMultiAuthKeyAuthSession(isBench) diff --git a/mw_api_rate_limit.go b/mw_api_rate_limit.go index d12c1f4fd04..a312a3e177d 100644 --- a/mw_api_rate_limit.go +++ b/mw_api_rate_limit.go @@ -8,6 +8,7 @@ import ( "time" "github.com/TykTechnologies/tyk/request" + "github.com/TykTechnologies/tyk/storage" "github.com/TykTechnologies/tyk/user" ) @@ -37,6 +38,7 @@ func (k *RateLimitForAPI) EnabledForSpec() bool { Per: k.Spec.GlobalRateLimit.Per, LastUpdated: strconv.Itoa(int(time.Now().UnixNano())), } + k.apiSess.SetKeyHash(storage.HashKey(k.keyName)) return true } @@ -62,12 +64,12 @@ func (k *RateLimitForAPI) handleRateLimitFailure(r *http.Request, token string) // ProcessRequest will run any checks on the request on the way through the system, return an error to have the chain fail func (k *RateLimitForAPI) ProcessRequest(w http.ResponseWriter, r *http.Request, _ interface{}) (error, int) { storeRef := k.Spec.SessionManager.Store() - reason := sessionLimiter.ForwardMessage(k.apiSess, + reason := sessionLimiter.ForwardMessage(r, k.apiSess, k.keyName, storeRef, true, false, - k.Spec.GlobalConfig, + &k.Spec.GlobalConfig, ) if reason == sessionFailRateLimit { diff --git a/mw_auth_key.go b/mw_auth_key.go index 283a72d1760..fdecd8b917c 100644 --- a/mw_auth_key.go +++ b/mw_auth_key.go @@ -70,7 +70,7 @@ func (k *AuthKey) ProcessRequest(w http.ResponseWriter, r *http.Request, _ inter // If key not provided in header or cookie and client certificate is provided, try to find certificate based key if config.UseCertificate && key == "" && r.TLS != nil && len(r.TLS.PeerCertificates) > 0 { - key = k.Spec.OrgID + certs.HexSHA256(r.TLS.PeerCertificates[0].Raw) + key = generateToken(k.Spec.OrgID, certs.HexSHA256(r.TLS.PeerCertificates[0].Raw)) } if key == "" { @@ -85,7 +85,7 @@ func (k *AuthKey) ProcessRequest(w http.ResponseWriter, r *http.Request, _ inter key = stripBearer(key) // Check if API key valid - session, keyExists := k.CheckSessionAndIdentityForValidKey(key) + session, keyExists := k.CheckSessionAndIdentityForValidKey(key, r) if !keyExists { logEntry := getLogEntryForRequest(r, key, nil) logEntry.Info("Attempted access with non-existent key.") @@ -96,14 +96,13 @@ func (k *AuthKey) ProcessRequest(w http.ResponseWriter, r *http.Request, _ inter // Report in health check reportHealthValue(k.Spec, KeyFailure, "1") - return errors.New("Key not authorised"), http.StatusForbidden + return errors.New("Access to this API has been disallowed"), http.StatusForbidden } // Set session state on context, we will need it later switch k.Spec.BaseIdentityProvidedBy { case apidef.AuthToken, apidef.UnsetAuth: - ctxSetSession(r, &session) - ctxSetAuthToken(r, key) + ctxSetSession(r, &session, key, false) k.setContextVars(r, key) } diff --git a/mw_auth_key_test.go b/mw_auth_key_test.go index 804c74f11bf..4784cd27f88 100644 --- a/mw_auth_key_test.go +++ b/mw_auth_key_test.go @@ -11,9 +11,96 @@ import ( "github.com/justinas/alice" "github.com/lonelycode/go-uuid/uuid" + "github.com/TykTechnologies/tyk/config" + "github.com/TykTechnologies/tyk/test" "github.com/TykTechnologies/tyk/user" ) +func TestMurmur3CharBug(t *testing.T) { + defer resetTestConfig() + ts := newTykTestServer() + defer ts.Close() + + api := buildAPI(func(spec *APISpec) { + spec.UseKeylessAccess = false + spec.Proxy.ListenPath = "/" + })[0] + + genTestCase := func(key string, status int) test.TestCase { + return test.TestCase{Path: "/", Headers: map[string]string{"Authorization": key}, Code: status} + } + + t.Run("Without hashing", func(t *testing.T) { + globalConf := config.Global() + globalConf.HashKeys = false + config.SetGlobal(globalConf) + + loadAPI(api) + + key := createSession() + + ts.Run(t, []test.TestCase{ + genTestCase("wrong", 403), + genTestCase(key+"abc", 403), + genTestCase(key, 200), + }...) + }) + + t.Run("murmur32 hashing, legacy", func(t *testing.T) { + globalConf := config.Global() + globalConf.HashKeys = true + globalConf.HashKeyFunction = "" + config.SetGlobal(globalConf) + + loadAPI(api) + + key := createSession() + + ts.Run(t, []test.TestCase{ + genTestCase("wrong", 403), + // Should reject instead, just to show bug + genTestCase(key+"abc", 200), + genTestCase(key, 200), + }...) + }) + + t.Run("murmur32 hashing, json keys", func(t *testing.T) { + globalConf := config.Global() + globalConf.HashKeys = true + globalConf.HashKeyFunction = "murmur32" + config.SetGlobal(globalConf) + + loadAPI(api) + + key := createSession() + + ts.Run(t, []test.TestCase{ + genTestCase("wrong", 403), + // Should reject instead, just to show bug + genTestCase(key+"abc", 200), + genTestCase(key, 200), + }...) + }) + + t.Run("murmur64 hashing", func(t *testing.T) { + globalConf := config.Global() + globalConf.HashKeys = true + globalConf.HashKeyFunction = "murmur64" + config.SetGlobal(globalConf) + + loadAPI(api) + + key := createSession() + + ts.Run(t, []test.TestCase{ + genTestCase("wrong", 403), + // New hashing fixes the bug + genTestCase(key+"abc", 403), + genTestCase(key, 200), + }...) + }) +} + func createAuthKeyAuthSession(isBench bool) *user.SessionState { session := new(user.SessionState) // essentially non-throttled diff --git a/mw_basic_auth.go b/mw_basic_auth.go index df3c6256953..c44b1b3548c 100644 --- a/mw_basic_auth.go +++ b/mw_basic_auth.go @@ -5,22 +5,30 @@ import ( "errors" "net/http" "strings" + "time" + "github.com/Sirupsen/logrus" + "github.com/pmylund/go-cache" + "github.com/spaolacci/murmur3" "golang.org/x/crypto/bcrypt" "github.com/TykTechnologies/tyk/apidef" "github.com/TykTechnologies/tyk/user" ) +const defaultBasicAuthTTL = time.Duration(60) * time.Second + // BasicAuthKeyIsValid uses a username instead of type BasicAuthKeyIsValid struct { BaseMiddleware + cache *cache.Cache } func (k *BasicAuthKeyIsValid) Name() string { return "BasicAuthKeyIsValid" } +// EnabledForSpec checks if UseBasicAuth is set in the API definition. func (k *BasicAuthKeyIsValid) EnabledForSpec() bool { return k.Spec.UseBasicAuth } @@ -69,54 +77,95 @@ func (k *BasicAuthKeyIsValid) ProcessRequest(w http.ResponseWriter, r *http.Requ } // Check if API key valid - keyName := k.Spec.OrgID + authValues[0] + keyName := generateToken(k.Spec.OrgID, authValues[0]) logEntry = getLogEntryForRequest(r, keyName, nil) - session, keyExists := k.CheckSessionAndIdentityForValidKey(keyName) + session, keyExists := k.CheckSessionAndIdentityForValidKey(keyName, r) if !keyExists { logEntry.Info("Attempted access with non-existent user.") - // Fire Authfailed Event - AuthFailed(k, r, token) - - // Report in health check - reportHealthValue(k.Spec, KeyFailure, "-1") - - return k.requestForBasicAuth(w, "User not authorised") + return k.handleAuthFail(w, r, token) } - // Ensure that the username and password match up - var passMatch bool switch session.BasicAuthData.Hash { case user.HashBCrypt: - err := bcrypt.CompareHashAndPassword([]byte(session.BasicAuthData.Password), []byte(authValues[1])) - if err == nil { - passMatch = true + + if err := k.compareHashAndPassword(session.BasicAuthData.Password, authValues[1], logEntry); err != nil { + logEntry.Warn("Attempted access with existing user, failed password check.") + return k.handleAuthFail(w, r, token) } case user.HashPlainText: - if session.BasicAuthData.Password == authValues[1] { - passMatch = true + if session.BasicAuthData.Password != authValues[1] { + + logEntry.Warn("Attempted access with existing user, failed password check.") + return k.handleAuthFail(w, r, token) } } - if !passMatch { - logEntry.Info("Attempted access with existing user but failed password check.") + // Set session state on context, we will need it later + switch k.Spec.BaseIdentityProvidedBy { + case apidef.BasicAuthUser, apidef.UnsetAuth: + ctxSetSession(r, &session, keyName, false) + } - // Fire Authfailed Event - AuthFailed(k, r, token) + return nil, http.StatusOK +} + +func (k *BasicAuthKeyIsValid) handleAuthFail(w http.ResponseWriter, r *http.Request, token string) (error, int) { + + // Fire Authfailed Event + AuthFailed(k, r, token) - // Report in health check - reportHealthValue(k.Spec, KeyFailure, "-1") + // Report in health check + reportHealthValue(k.Spec, KeyFailure, "-1") - return k.requestForBasicAuth(w, "User not authorised") + return k.requestForBasicAuth(w, "User not authorised") +} + +func (k *BasicAuthKeyIsValid) doBcryptWithCache(cacheDuration time.Duration, hashedPassword []byte, password []byte) error { + if err := bcrypt.CompareHashAndPassword(hashedPassword, password); err != nil { + + return err } - // Set session state on context, we will need it later - switch k.Spec.BaseIdentityProvidedBy { - case apidef.BasicAuthUser, apidef.UnsetAuth: - ctxSetSession(r, &session) - ctxSetAuthToken(r, keyName) + hasher := murmur3.New32() + hasher.Write(password) + k.cache.Set(string(hashedPassword), string(hasher.Sum(nil)), cacheDuration) + + return nil +} + +func (k *BasicAuthKeyIsValid) compareHashAndPassword(hash string, password string, logEntry *logrus.Entry) error { + + cacheEnabled := !k.Spec.BasicAuth.DisableCaching + passwordBytes := []byte(password) + hashBytes := []byte(hash) + + if !cacheEnabled { + + logEntry.Debug("cache disabled") + return bcrypt.CompareHashAndPassword(hashBytes, passwordBytes) } - // Request is valid, carry on - return nil, http.StatusOK + cacheTTL := defaultBasicAuthTTL // set a default TTL, then override based on BasicAuth.CacheTTL + if k.Spec.BasicAuth.CacheTTL > 0 { + cacheTTL = time.Duration(k.Spec.BasicAuth.CacheTTL) * time.Second + } + + cachedPass, inCache := k.cache.Get(hash) + if !inCache { + + logEntry.Debug("cache enabled: miss: bcrypt") + return k.doBcryptWithCache(cacheTTL, hashBytes, passwordBytes) + } + + hasher := murmur3.New32() + hasher.Write(passwordBytes) + if cachedPass.(string) != string(hasher.Sum(nil)) { + + logEntry.Warn("cache enabled: hit: failed auth: bcrypt") + return bcrypt.CompareHashAndPassword(hashBytes, passwordBytes) + } + + logEntry.Debug("cache enabled: hit: success") + return nil } diff --git a/mw_basic_auth_test.go b/mw_basic_auth_test.go index 6923e1c8305..07e9d772827 100644 --- a/mw_basic_auth_test.go +++ b/mw_basic_auth_test.go @@ -16,13 +16,15 @@ func genAuthHeader(username, password string) string { return fmt.Sprintf("Basic %s", encodedPass) } -func testPrepareBasicAuth() *user.SessionState { +func testPrepareBasicAuth(cacheDisabled bool) *user.SessionState { session := createStandardSession() session.BasicAuthData.Password = "password" session.AccessRights = map[string]user.AccessDefinition{"test": {APIID: "test", Versions: []string{"v1"}}} + session.OrgID = "default" buildAndLoadAPI(func(spec *APISpec) { spec.UseBasicAuth = true + spec.BasicAuth.DisableCaching = cacheDisabled spec.UseKeylessAccess = false spec.Proxy.ListenPath = "/" spec.OrgID = "default" @@ -35,7 +37,7 @@ func TestBasicAuth(t *testing.T) { ts := newTykTestServer() defer ts.Close() - session := testPrepareBasicAuth() + session := testPrepareBasicAuth(false) validPassword := map[string]string{"Authorization": genAuthHeader("user", "password")} wrongPassword := map[string]string{"Authorization": genAuthHeader("user", "wrong")} @@ -59,7 +61,7 @@ func BenchmarkBasicAuth(b *testing.B) { ts := newTykTestServer() defer ts.Close() - session := testPrepareBasicAuth() + session := testPrepareBasicAuth(false) validPassword := map[string]string{"Authorization": genAuthHeader("user", "password")} wrongPassword := map[string]string{"Authorization": genAuthHeader("user", "wrong")} @@ -85,3 +87,55 @@ func BenchmarkBasicAuth(b *testing.B) { }...) } } + +func BenchmarkBasicAuth_CacheEnabled(b *testing.B) { + b.ReportAllocs() + + ts := newTykTestServer() + defer ts.Close() + + session := testPrepareBasicAuth(false) + + validPassword := map[string]string{"Authorization": genAuthHeader("user", "password")} + + // Create base auth based key + ts.Run(b, test.TestCase{ + Method: "POST", + Path: "/tyk/keys/defaultuser", + Data: session, + AdminAuth: true, + Code: 200, + }) + + for i := 0; i < b.N; i++ { + ts.Run(b, []test.TestCase{ + {Method: "GET", Path: "/", Headers: validPassword, Code: 200}, + }...) + } +} + +func BenchmarkBasicAuth_CacheDisabled(b *testing.B) { + b.ReportAllocs() + + ts := newTykTestServer() + defer ts.Close() + + session := testPrepareBasicAuth(true) + + validPassword := map[string]string{"Authorization": genAuthHeader("user", "password")} + + // Create base auth based key + ts.Run(b, test.TestCase{ + Method: "POST", + Path: "/tyk/keys/defaultuser", + Data: session, + AdminAuth: true, + Code: 200, + }) + + for i := 0; i < b.N; i++ { + ts.Run(b, []test.TestCase{ + {Method: "GET", Path: "/", Headers: validPassword, Code: 200}, + }...) + } +} diff --git a/mw_hmac.go b/mw_hmac.go index 3dbe124ea4a..4d9c41fe715 100644 --- a/mw_hmac.go +++ b/mw_hmac.go @@ -73,7 +73,7 @@ func (hm *HMACMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Request, } // Get a session for the Key ID - secret, session, err := hm.getSecretAndSessionForKeyID(fieldValues.KeyID) + secret, session, err := hm.getSecretAndSessionForKeyID(r, fieldValues.KeyID) if err != nil { log.WithFields(logrus.Fields{ "prefix": "hmac", @@ -123,8 +123,7 @@ func (hm *HMACMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Request, // Set session state on context, we will need it later switch hm.Spec.BaseIdentityProvidedBy { case apidef.HMACKey, apidef.UnsetAuth: - ctxSetSession(r, &session) - ctxSetAuthToken(r, fieldValues.KeyID) + ctxSetSession(r, &session, fieldValues.KeyID, false) hm.setContextVars(r, fieldValues.KeyID) } @@ -222,8 +221,8 @@ type HMACFieldValues struct { Signature string } -func (hm *HMACMiddleware) getSecretAndSessionForKeyID(keyId string) (string, user.SessionState, error) { - session, keyExists := hm.CheckSessionAndIdentityForValidKey(keyId) +func (hm *HMACMiddleware) getSecretAndSessionForKeyID(r *http.Request, keyId string) (string, user.SessionState, error) { + session, keyExists := hm.CheckSessionAndIdentityForValidKey(keyId, r) if !keyExists { return "", session, errors.New("Key ID does not exist") } diff --git a/mw_js_plugin.go b/mw_js_plugin.go index c8570da6e48..422468f5fd0 100644 --- a/mw_js_plugin.go +++ b/mw_js_plugin.go @@ -11,6 +11,7 @@ import ( "net/url" "os" "path/filepath" + "reflect" "strings" "time" @@ -136,7 +137,6 @@ func (d *DynamicMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Reques specAsJson := specToJson(d.Spec) session := new(user.SessionState) - token := ctxGetAuthToken(r) // Encode the session object (if not a pre-process) if !d.Pre && d.UseSession { @@ -249,9 +249,12 @@ func (d *DynamicMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Reques r.URL.RawQuery = values.Encode() // Save the sesison data (if modified) - if !d.Pre && d.UseSession && len(newRequestData.SessionMeta) > 0 { - session.MetaData = mapStrsToIfaces(newRequestData.SessionMeta) - d.Spec.SessionManager.UpdateSession(token, session, session.Lifetime(d.Spec.SessionLifetime), false) + if !d.Pre && d.UseSession { + newMeta := mapStrsToIfaces(newRequestData.SessionMeta) + if !reflect.DeepEqual(session.MetaData, newMeta) { + session.MetaData = newMeta + ctxScheduleSessionUpdate(r) + } } log.WithFields(logrus.Fields{ @@ -277,8 +280,7 @@ func (d *DynamicMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Reques } if d.Auth { - ctxSetSession(r, &newRequestData.Session) - ctxSetAuthToken(r, newRequestData.AuthValue) + ctxSetSession(r, &newRequestData.Session, newRequestData.AuthValue, true) } return nil, http.StatusOK diff --git a/mw_jwt.go b/mw_jwt.go index 02b8318a208..6227313ade4 100644 --- a/mw_jwt.go +++ b/mw_jwt.go @@ -14,7 +14,6 @@ import ( cache "github.com/pmylund/go-cache" "github.com/TykTechnologies/tyk/apidef" - "github.com/TykTechnologies/tyk/storage" "github.com/TykTechnologies/tyk/user" ) @@ -22,6 +21,14 @@ type JWTMiddleware struct { BaseMiddleware } +const ( + KID = "kid" + SUB = "sub" + HMACSign = "hmac" + RSASign = "rsa" + ECDSASign = "ecdsa" +) + func (k *JWTMiddleware) Name() string { return "JWTMiddleware" } @@ -100,31 +107,27 @@ func (k *JWTMiddleware) getSecretFromURL(url, kid, keyType string) ([]byte, erro return nil, errors.New("No matching KID could be found") } -func (k *JWTMiddleware) getIdentityFomToken(token *jwt.Token) (string, bool) { - // Try using a kid or sub header - idFound := false - var tykId string - if token.Header["kid"] != nil { - tykId = token.Header["kid"].(string) - idFound = true - } - - if !idFound && token.Claims.(jwt.MapClaims)["sub"] != nil { - tykId = token.Claims.(jwt.MapClaims)["sub"].(string) - idFound = true +func (k *JWTMiddleware) getIdentityFromToken(token *jwt.Token) (string, error) { + // Check which claim is used for the id - kid or sub header + // If is not supposed to ignore KID - will use this as ID if not empty + if !k.Spec.APIDefinition.JWTSkipKid { + if tykId, idFound := token.Header[KID].(string); idFound { + log.Debug("Found: ", tykId) + return tykId, nil + } } - - log.Debug("Found: ", tykId) - return tykId, idFound + // In case KID was empty or was set to ignore KID ==> Will try to get the Id from JWTIdentityBaseField or fallback to 'sub' + tykId, err := k.getUserIdFromClaim(token.Claims.(jwt.MapClaims)) + return tykId, err } -func (k *JWTMiddleware) getSecret(token *jwt.Token) ([]byte, error) { +func (k *JWTMiddleware) getSecretToVerifySignature(r *http.Request, token *jwt.Token) ([]byte, error) { config := k.Spec.APIDefinition // Check for central JWT source if config.JWTSource != "" { // Is it a URL? if httpScheme.MatchString(config.JWTSource) { - secret, err := k.getSecretFromURL(config.JWTSource, token.Header["kid"].(string), k.Spec.JWTSigningMethod) + secret, err := k.getSecretFromURL(config.JWTSource, token.Header[KID].(string), k.Spec.JWTSigningMethod) if err != nil { return nil, err } @@ -140,7 +143,7 @@ func (k *JWTMiddleware) getSecret(token *jwt.Token) ([]byte, error) { // Is decoded url too? if httpScheme.MatchString(string(decodedCert)) { - secret, err := k.getSecretFromURL(string(decodedCert), token.Header["kid"].(string), k.Spec.JWTSigningMethod) + secret, err := k.getSecretFromURL(string(decodedCert), token.Header[KID].(string), k.Spec.JWTSigningMethod) if err != nil { return nil, err } @@ -148,19 +151,20 @@ func (k *JWTMiddleware) getSecret(token *jwt.Token) ([]byte, error) { return secret, nil } - return decodedCert, nil + return decodedCert, nil // Returns the decoded secret } - // Try using a kid or sub header - tykId, found := k.getIdentityFomToken(token) + // If we are here, there's no central JWT source - if !found { - return nil, errors.New("Key ID not found") + // Get the ID from the token (in KID header or configured claim or SUB claim) + tykId, err := k.getIdentityFromToken(token) + if err != nil { + return nil, err } // Couldn't base64 decode the kid, so lets try it raw log.Debug("Getting key: ", tykId) - session, rawKeyExists := k.CheckSessionAndIdentityForValidKey(tykId) + session, rawKeyExists := k.CheckSessionAndIdentityForValidKey(tykId, r) if !rawKeyExists { log.Info("Not found!") return nil, errors.New("token invalid, key not found") @@ -168,8 +172,8 @@ func (k *JWTMiddleware) getSecret(token *jwt.Token) ([]byte, error) { return []byte(session.JWTData.Secret), nil } -func (k *JWTMiddleware) getPolicyIDFromToken(token *jwt.Token) (string, bool) { - policyID, foundPolicy := token.Claims.(jwt.MapClaims)[k.Spec.JWTPolicyFieldName].(string) +func (k *JWTMiddleware) getPolicyIDFromToken(claims jwt.MapClaims) (string, bool) { + policyID, foundPolicy := claims[k.Spec.JWTPolicyFieldName].(string) if !foundPolicy { log.Error("Could not identify a policy to apply to this token from field!") return "", false @@ -178,18 +182,18 @@ func (k *JWTMiddleware) getPolicyIDFromToken(token *jwt.Token) (string, bool) { return policyID, true } -func (k *JWTMiddleware) getBasePolicyID(token *jwt.Token) (string, bool) { +func (k *JWTMiddleware) getBasePolicyID(r *http.Request, claims jwt.MapClaims) (string, bool) { if k.Spec.JWTPolicyFieldName != "" { - return k.getPolicyIDFromToken(token) + return k.getPolicyIDFromToken(claims) } else if k.Spec.JWTClientIDBaseField != "" { - clientID, clientIDFound := token.Claims.(jwt.MapClaims)[k.Spec.JWTClientIDBaseField].(string) + clientID, clientIDFound := claims[k.Spec.JWTClientIDBaseField].(string) if !clientIDFound { log.Error("Could not identify a policy to apply to this token from field!") return "", false } // Check for a regular token that matches this client ID - clientSession, exists := k.CheckSessionAndIdentityForValidKey(clientID) + clientSession, exists := k.CheckSessionAndIdentityForValidKey(clientID, r) if !exists { return "", false } @@ -206,37 +210,67 @@ func (k *JWTMiddleware) getBasePolicyID(token *jwt.Token) (string, bool) { return "", false } +func (k *JWTMiddleware) getUserIdFromClaim(claims jwt.MapClaims) (string, error) { + var userId string + var found = false + + if k.Spec.JWTIdentityBaseField != "" { + if userId, found = claims[k.Spec.JWTIdentityBaseField].(string); found { + if len(userId) > 0 { + log.WithField("userId", userId).Debug("Found User Id in Base Field") + return userId, nil + } + message := "found an empty user ID in predefined base field claim " + k.Spec.JWTIdentityBaseField + log.Error(message) + return "", errors.New(message) + } + + if !found { + log.WithField("Base Field", k.Spec.JWTIdentityBaseField).Warning("Base Field claim not found, trying to find user ID in 'sub' claim.") + } + } + + if userId, found = claims[SUB].(string); found { + if len(userId) > 0 { + log.WithField("userId", userId).Debug("Found User Id in 'sub' claim") + return userId, nil + } + message := "found an empty user ID in sub claim" + log.Error(message) + return "", errors.New(message) + } + + message := "no suitable claims for user ID were found" + log.Error(message) + return "", errors.New(message) +} + // processCentralisedJWT Will check a JWT token centrally against the secret stored in the API Definition. func (k *JWTMiddleware) processCentralisedJWT(r *http.Request, token *jwt.Token) (error, int) { log.Debug("JWT authority is centralised") - // Generate a virtual token - baseFieldData, baseFound := token.Claims.(jwt.MapClaims)[k.Spec.JWTIdentityBaseField].(string) - if !baseFound { - log.Warning("Base Field not found, using SUB") - var found bool - baseFieldData, found = token.Claims.(jwt.MapClaims)["sub"].(string) - if !found { - log.Error("ID Could not be generated. Failing Request.") - k.reportLoginFailure("[NOT FOUND]", r) - return errors.New("Key not authorized"), http.StatusForbidden - } + claims := token.Claims.(jwt.MapClaims) + baseFieldData, err := k.getUserIdFromClaim(claims) + if err != nil { + k.reportLoginFailure("[NOT FOUND]", r) + return err, http.StatusForbidden } - log.Debug("Base Field ID set to: ", baseFieldData) + + // Generate a virtual token data := []byte(baseFieldData) - tokenID := fmt.Sprintf("%x", md5.Sum(data)) - sessionID := k.Spec.OrgID + tokenID + keyID := fmt.Sprintf("%x", md5.Sum(data)) + sessionID := generateToken(k.Spec.OrgID, keyID) log.Debug("JWT Temporary session ID is: ", sessionID) - session, exists := k.CheckSessionAndIdentityForValidKey(sessionID) + session, exists := k.CheckSessionAndIdentityForValidKey(sessionID, r) if !exists { // Create it log.Debug("Key does not exist, creating") session = user.SessionState{} // We need a base policy as a template, either get it from the token itself OR a proxy client ID within Tyk - basePolicyID, foundPolicy := k.getBasePolicyID(token) + basePolicyID, foundPolicy := k.getBasePolicyID(r, claims) if !foundPolicy { k.reportLoginFailure(baseFieldData, r) return errors.New("Key not authorized: no matching policy found"), http.StatusForbidden @@ -256,19 +290,17 @@ func (k *JWTMiddleware) processCentralisedJWT(r *http.Request, token *jwt.Token) session.Alias = baseFieldData // Update the session in the session manager in case it gets called again - k.Spec.SessionManager.UpdateSession(sessionID, &session, session.Lifetime(k.Spec.SessionLifetime), false) log.Debug("Policy applied to key") switch k.Spec.BaseIdentityProvidedBy { case apidef.JWTClaim, apidef.UnsetAuth: - ctxSetSession(r, &session) - ctxSetAuthToken(r, sessionID) + ctxSetSession(r, &session, sessionID, true) } ctxSetJWTContextVars(k.Spec, r, token) return nil, http.StatusOK } else if k.Spec.JWTPolicyFieldName != "" { // extract policy ID from JWT token - policyID, foundPolicy := k.getPolicyIDFromToken(token) + policyID, foundPolicy := k.getPolicyIDFromToken(claims) if !foundPolicy { k.reportLoginFailure(baseFieldData, r) return errors.New("Key not authorized: no matching policy found"), http.StatusForbidden @@ -304,20 +336,14 @@ func (k *JWTMiddleware) processCentralisedJWT(r *http.Request, token *jwt.Token) return errors.New("Key not authorized: could not apply new policy"), http.StatusForbidden } - cacheKey := sessionID - if k.Spec.GlobalConfig.HashKeys { - cacheKey = storage.HashStr(sessionID) - } - // update session in cache - go SessionCache.Set(cacheKey, session, cache.DefaultExpiration) + go SessionCache.Set(session.KeyHash(), session, cache.DefaultExpiration) } } log.Debug("Key found") switch k.Spec.BaseIdentityProvidedBy { case apidef.JWTClaim, apidef.UnsetAuth: - ctxSetSession(r, &session) - ctxSetAuthToken(r, sessionID) + ctxSetSession(r, &session, sessionID, false) } ctxSetJWTContextVars(k.Spec, r, token) return nil, http.StatusOK @@ -332,23 +358,22 @@ func (k *JWTMiddleware) reportLoginFailure(tykId string, r *http.Request) { } func (k *JWTMiddleware) processOneToOneTokenMap(r *http.Request, token *jwt.Token) (error, int) { - tykId, found := k.getIdentityFomToken(token) - - if !found { + // Get the ID from the token + tykId, err := k.getIdentityFromToken(token) + if err != nil { k.reportLoginFailure(tykId, r) - return errors.New("Key id not found"), http.StatusNotFound + return err, http.StatusNotFound } log.Debug("Using raw key ID: ", tykId) - session, exists := k.CheckSessionAndIdentityForValidKey(tykId) + session, exists := k.CheckSessionAndIdentityForValidKey(tykId, r) if !exists { k.reportLoginFailure(tykId, r) return errors.New("Key not authorized"), http.StatusForbidden } log.Debug("Raw key ID found.") - ctxSetSession(r, &session) - ctxSetAuthToken(r, tykId) + ctxSetSession(r, &session, tykId, false) ctxSetJWTContextVars(k.Spec, r, token) return nil, http.StatusOK } @@ -396,35 +421,35 @@ func (k *JWTMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Request, _ token, err := parser.Parse(rawJWT, func(token *jwt.Token) (interface{}, error) { // Don't forget to validate the alg is what you expect: switch k.Spec.JWTSigningMethod { - case "hmac": + case HMACSign: if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { - return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"]) + return nil, fmt.Errorf("Unexpected signing method: %v and not HMAC signature", token.Header["alg"]) } - case "rsa": + case RSASign: if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok { - return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"]) + return nil, fmt.Errorf("Unexpected signing method: %v and not RSA signature", token.Header["alg"]) } - case "ecdsa": + case ECDSASign: if _, ok := token.Method.(*jwt.SigningMethodECDSA); !ok { - return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"]) + return nil, fmt.Errorf("Unexpected signing method: %v and not ECDSA signature", token.Header["alg"]) } default: - log.Warning("No signing method found in API Definition, defaulting to HMAC") + log.Warning("No signing method found in API Definition, defaulting to HMAC signature") if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"]) } } - val, err := k.getSecret(token) + val, err := k.getSecretToVerifySignature(r, token) if err != nil { log.Error("Couldn't get token: ", err) return nil, err } - if k.Spec.JWTSigningMethod == "rsa" { + if k.Spec.JWTSigningMethod == RSASign { asRSA, err := jwt.ParseRSAPublicKeyFromPEM(val) if err != nil { - log.Error("Failed to deccode JWT to RSA type") + log.WithError(err).Error("Failed to decode JWT to RSA type") return nil, err } return asRSA, nil @@ -434,7 +459,7 @@ func (k *JWTMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Request, _ }) if err == nil && token.Valid { - if jwtErr := k.validateJWTClaims(token.Claims.(jwt.MapClaims)); jwtErr != nil { + if jwtErr := k.timeValidateJWTClaims(token.Claims.(jwt.MapClaims)); jwtErr != nil { return errors.New("Key not authorized: " + jwtErr.Error()), http.StatusUnauthorized } @@ -450,18 +475,15 @@ func (k *JWTMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Request, _ } logEntry := getLogEntryForRequest(r, "", nil) logEntry.Info("Attempted JWT access with non-existent key.") - k.reportLoginFailure(tykId, r) - if err != nil { logEntry.Error("JWT validation error: ", err) return errors.New("Key not authorized:" + err.Error()), http.StatusForbidden - } else { - return errors.New("Key not authorized"), http.StatusForbidden } + return errors.New("Key not authorized"), http.StatusForbidden } -func (k *JWTMiddleware) validateJWTClaims(c jwt.MapClaims) *jwt.ValidationError { +func (k *JWTMiddleware) timeValidateJWTClaims(c jwt.MapClaims) *jwt.ValidationError { vErr := new(jwt.ValidationError) now := time.Now().Unix() diff --git a/mw_jwt_test.go b/mw_jwt_test.go index db43dd468de..41753c214e4 100644 --- a/mw_jwt_test.go +++ b/mw_jwt_test.go @@ -61,6 +61,18 @@ jQIDAQAB -----END PUBLIC KEY----- ` +const jwtRSAPubKeyinvalid = ` +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyqZ4rwKF8qCExS7kpY4c +nJa/37FMkJNkalZ3OuslLB0oRL8T4c94kdF4aeNzSFkSe2n99IBI6Ssl79vbfMZb ++t06L0Q94k+/P37x7+/RJZiff4y1VGjrnrnMI2iu9l4iBBRYzNmG6eblroEMMWlg +k5tysHgxB59CSNIcD9gqk1hx4n/FgOmvKsfQgWHNlPSDTRcWGWGhB2/XgNVYG2pO +lQxAPqLhBHeqGTXBbPfGF9cHzixpsPr6GtbzPwhsQ/8bPxoJ7hdfn+rzztks3d6+ +HWURcyNTLRe0mjXjjee9Z6+gZ+H+fS4pnP9tqT7IgU6ePUWTpjoiPtLexgsAa/ct +jQIDAQAB!!!! +-----END PUBLIC KEY----- +` + func createJWTSession() *user.SessionState { session := new(user.SessionState) session.Rate = 1000000.0 @@ -87,42 +99,77 @@ func createJWTSessionWithRSAWithPolicy(policyID string) *user.SessionState { return session } -// JWTSessionHMAC +type JwtCreator func() *user.SessionState + +func prepareGenericJWTSession(testName string, method string, claimName string, ApiSkipKid bool) (*APISpec, string) { + tokenKID := testKey(testName, "token") + + var jwtToken string + var sessionFunc JwtCreator + switch method { + default: + log.Warningf("Signing method '%s' is not recognised, defaulting to HMAC signature") + method = HMACSign + fallthrough + case HMACSign: + sessionFunc = createJWTSession + + jwtToken = createJWKTokenHMAC(func(t *jwt.Token) { + t.Claims.(jwt.MapClaims)["foo"] = "bar" + t.Claims.(jwt.MapClaims)["exp"] = time.Now().Add(time.Hour * 72).Unix() + + if claimName != KID { + t.Claims.(jwt.MapClaims)[claimName] = tokenKID + t.Header[KID] = "ignore-this-id" + } else { + t.Header[KID] = tokenKID + } + }) + case RSASign: + sessionFunc = createJWTSessionWithRSA + + jwtToken = createJWKToken(func(t *jwt.Token) { + t.Claims.(jwt.MapClaims)["foo"] = "bar" + t.Claims.(jwt.MapClaims)["exp"] = time.Now().Add(time.Hour * 72).Unix() + + if claimName != KID { + t.Claims.(jwt.MapClaims)[claimName] = tokenKID + t.Header[KID] = "ignore-this-id" + } else { + t.Header[KID] = tokenKID + } + }) + } -func prepareJWTSessionHMAC(tb testing.TB, isBench bool) string { spec := buildAndLoadAPI(func(spec *APISpec) { spec.UseKeylessAccess = false - spec.JWTSigningMethod = "hmac" + spec.JWTSigningMethod = method spec.EnableJWT = true spec.Proxy.ListenPath = "/" - })[0] + spec.JWTSkipKid = ApiSkipKid - session := createJWTSession() - tokenKID := testKey(tb, "token") - if isBench { - tokenKID += "-" + uuid.New() - } - spec.SessionManager.UpdateSession(tokenKID, session, 60, false) + if claimName != KID { + spec.JWTIdentityBaseField = claimName + } + })[0] + spec.SessionManager.UpdateSession(tokenKID, sessionFunc(), 60, false) - jwtToken := createJWKTokenHMAC(func(t *jwt.Token) { - t.Header["kid"] = tokenKID - t.Claims.(jwt.MapClaims)["foo"] = "bar" - t.Claims.(jwt.MapClaims)["exp"] = time.Now().Add(time.Hour * 72).Unix() - }) + return spec, jwtToken - return jwtToken } func TestJWTSessionHMAC(t *testing.T) { ts := newTykTestServer() defer ts.Close() - jwtToken := prepareJWTSessionHMAC(t, false) + //If we skip the check then the Id will be taken from SUB and the call will succeed + _, jwtToken := prepareGenericJWTSession(t.Name(), HMACSign, KID, false) + defer resetTestConfig() authHeaders := map[string]string{"authorization": jwtToken} t.Run("Request with valid JWT signed with HMAC", func(t *testing.T) { ts.Run(t, test.TestCase{ - Headers: authHeaders, Code: 200, + Headers: authHeaders, Code: http.StatusOK, }) }) } @@ -133,51 +180,97 @@ func BenchmarkJWTSessionHMAC(b *testing.B) { ts := newTykTestServer() defer ts.Close() - jwtToken := prepareJWTSessionHMAC(b, true) + //If we skip the check then the Id will be taken from SUB and the call will succeed + _, jwtToken := prepareGenericJWTSession(b.Name(), HMACSign, KID, false) + defer resetTestConfig() authHeaders := map[string]string{"authorization": jwtToken} for i := 0; i < b.N; i++ { ts.Run(b, test.TestCase{ - Headers: authHeaders, Code: 200, + Headers: authHeaders, Code: http.StatusOK, }) } } -// JWTSessionRSA -func prepareJWTSessionRSA(tb testing.TB, isBench bool) (*APISpec, string) { - spec := buildAndLoadAPI(func(spec *APISpec) { - spec.UseKeylessAccess = false - spec.JWTSigningMethod = "rsa" - spec.EnableJWT = true - spec.Proxy.ListenPath = "/" - })[0] +func TestJWTHMACIdInSubClaim(t *testing.T) { - session := createJWTSessionWithRSA() - tokenKID := testKey(tb, "token") - if isBench { - tokenKID += "-" + uuid.New() - } - spec.SessionManager.UpdateSession(tokenKID, session, 60, false) + ts := newTykTestServer() + defer ts.Close() - jwtToken := createJWKToken(func(t *jwt.Token) { - t.Header["kid"] = tokenKID - t.Claims.(jwt.MapClaims)["foo"] = "bar" - t.Claims.(jwt.MapClaims)["exp"] = time.Now().Add(time.Hour * 72).Unix() + //Same as above + _, jwtToken := prepareGenericJWTSession(t.Name(), HMACSign, SUB, true) + authHeaders := map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/HMAC/Id in SuB/Global-skip-kid/Api-skip-kid", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, Code: http.StatusOK, + }) }) - return spec, jwtToken + // For backward compatibility, if the new config are not set, and the id is in the 'sub' claim while the 'kid' claim + // in the header is not empty, then the jwt will return 403 - "Key not authorized:token invalid, key not found" + _, jwtToken = prepareGenericJWTSession(t.Name(), HMACSign, SUB, false) + authHeaders = map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/HMAC/Id in SuB/Global-dont-skip-kid/Api-dont-skip-kid", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, + Code: http.StatusForbidden, + BodyMatch: `Key not authorized:token invalid, key not found`, + }) + }) + + // Case where the gw always check the 'kid' claim first but if this JWTSkipCheckKidAsId is set on the api level, + // then it'll work + _, jwtToken = prepareGenericJWTSession(t.Name(), HMACSign, SUB, true) + defer resetTestConfig() + authHeaders = map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/HMAC/Id in SuB/Global-dont-skip-kid/Api-skip-kid", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, Code: http.StatusOK, + }) + }) } -func TestJWTSessionRSA(t *testing.T) { +func TestJWTRSAIdInSubClaim(t *testing.T) { ts := newTykTestServer() defer ts.Close() - _, jwtToken := prepareJWTSessionRSA(t, false) + _, jwtToken := prepareGenericJWTSession(t.Name(), RSASign, SUB, true) + authHeaders := map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/RSA/Id in SuB/Global-skip-kid/Api-skip-kid", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, Code: http.StatusOK, + }) + }) + + _, jwtToken = prepareGenericJWTSession(t.Name(), RSASign, SUB, false) + authHeaders = map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/RSA/Id in SuB/Global-dont-skip-kid/Api-dont-skip-kid", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, + Code: http.StatusForbidden, + BodyMatch: `Key not authorized:token invalid, key not found`, + }) + }) + + _, jwtToken = prepareGenericJWTSession(t.Name(), RSASign, SUB, true) + authHeaders = map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/RSA/Id in SuB/Global-dont-skip-kid/Api-skip-kid", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, Code: http.StatusOK, + }) + }) +} + +func TestJWTSessionRSA(t *testing.T) { + ts := newTykTestServer() + defer ts.Close() + //default values, keep backward compatibility + _, jwtToken := prepareGenericJWTSession(t.Name(), RSASign, KID, false) authHeaders := map[string]string{"authorization": jwtToken} t.Run("Request with valid JWT", func(t *testing.T) { ts.Run(t, test.TestCase{ - Headers: authHeaders, Code: 200, + Headers: authHeaders, Code: http.StatusOK, }) }) } @@ -188,12 +281,13 @@ func BenchmarkJWTSessionRSA(b *testing.B) { ts := newTykTestServer() defer ts.Close() - _, jwtToken := prepareJWTSessionRSA(b, true) + //default values, keep backward compatibility + _, jwtToken := prepareGenericJWTSession(b.Name(), RSASign, KID, false) authHeaders := map[string]string{"authorization": jwtToken} for i := 0; i < b.N; i++ { ts.Run(b, test.TestCase{ - Headers: authHeaders, Code: 200, + Headers: authHeaders, Code: http.StatusOK, }) } } @@ -202,7 +296,8 @@ func TestJWTSessionFailRSA_EmptyJWT(t *testing.T) { ts := newTykTestServer() defer ts.Close() - prepareJWTSessionRSA(t, false) + //default values, same as before (keeps backward compatibility) + prepareGenericJWTSession(t.Name(), RSASign, KID, false) authHeaders := map[string]string{"authorization": ""} t.Run("Request with empty authorization header", func(t *testing.T) { @@ -216,12 +311,13 @@ func TestJWTSessionFailRSA_NoAuthHeader(t *testing.T) { ts := newTykTestServer() defer ts.Close() - prepareJWTSessionRSA(t, false) + //default values, same as before (keeps backward compatibility) + prepareGenericJWTSession(t.Name(), RSASign, KID, false) authHeaders := map[string]string{} t.Run("Request without authorization header", func(t *testing.T) { ts.Run(t, test.TestCase{ - Headers: authHeaders, Code: 400, + Headers: authHeaders, Code: http.StatusBadRequest, BodyMatch: `Authorization field missing`, }) }) } @@ -230,12 +326,15 @@ func TestJWTSessionFailRSA_MalformedJWT(t *testing.T) { ts := newTykTestServer() defer ts.Close() - _, jwtToken := prepareJWTSessionRSA(t, false) + //default values, same as before (keeps backward compatibility) + _, jwtToken := prepareGenericJWTSession(t.Name(), RSASign, KID, false) authHeaders := map[string]string{"authorization": jwtToken + "ajhdkjhsdfkjashdkajshdkajhsdkajhsd"} t.Run("Request with malformed JWT", func(t *testing.T) { ts.Run(t, test.TestCase{ - Headers: authHeaders, Code: 403, + Headers: authHeaders, + Code: http.StatusForbidden, + BodyMatch: `Key not authorized:crypto/rsa: verification error`, }) }) } @@ -244,13 +343,16 @@ func TestJWTSessionFailRSA_MalformedJWT_NOTRACK(t *testing.T) { ts := newTykTestServer() defer ts.Close() - spec, jwtToken := prepareJWTSessionRSA(t, false) + //default values, same as before (keeps backward compatibility) + spec, jwtToken := prepareGenericJWTSession(t.Name(), RSASign, KID, false) spec.DoNotTrack = true - authHeaders := map[string]string{"authorization": jwtToken + "ajhdkjhsdfkjashdkajshdkajhsdkajhsd"} + t.Run("Request with malformed JWT no track", func(t *testing.T) { ts.Run(t, test.TestCase{ - Headers: authHeaders, Code: 403, + Headers: authHeaders, + Code: http.StatusForbidden, + BodyMatch: `Key not authorized:crypto/rsa: verification error`, }) }) } @@ -259,52 +361,30 @@ func TestJWTSessionFailRSA_WrongJWT(t *testing.T) { ts := newTykTestServer() defer ts.Close() - prepareJWTSessionRSA(t, false) - + //default values, same as before (keeps backward compatibility) + prepareGenericJWTSession(t.Name(), RSASign, KID, false) authHeaders := map[string]string{"authorization": "123"} + t.Run("Request with invalid JWT", func(t *testing.T) { ts.Run(t, test.TestCase{ - Headers: authHeaders, Code: 403, + Headers: authHeaders, + Code: http.StatusForbidden, + BodyMatch: `Key not authorized:token contains an invalid number of segments`, }) }) } -// TestJWTSessionRSABearer - -func prepareJWTSessionRSABearer(tb testing.TB, isBench bool) string { - spec := buildAndLoadAPI(func(spec *APISpec) { - spec.UseKeylessAccess = false - spec.JWTSigningMethod = "rsa" - spec.EnableJWT = true - spec.Proxy.ListenPath = "/" - })[0] - - session := createJWTSessionWithRSA() - tokenKID := testKey(tb, "token") - if isBench { - tokenKID += "-" + uuid.New() - } - spec.SessionManager.UpdateSession(tokenKID, session, 60, false) - - jwtToken := createJWKToken(func(t *jwt.Token) { - t.Header["kid"] = tokenKID - t.Claims.(jwt.MapClaims)["foo"] = "bar" - t.Claims.(jwt.MapClaims)["exp"] = time.Now().Add(time.Hour * 72).Unix() - }) - - return jwtToken -} - func TestJWTSessionRSABearer(t *testing.T) { ts := newTykTestServer() defer ts.Close() - jwtToken := prepareJWTSessionRSABearer(t, false) - + //default values, same as before (keeps backward compatibility) + _, jwtToken := prepareGenericJWTSession(t.Name(), RSASign, KID, false) authHeaders := map[string]string{"authorization": "Bearer " + jwtToken} + t.Run("Request with valid Bearer", func(t *testing.T) { ts.Run(t, test.TestCase{ - Headers: authHeaders, Code: 200, + Headers: authHeaders, Code: http.StatusOK, }) }) } @@ -315,13 +395,13 @@ func BenchmarkJWTSessionRSABearer(b *testing.B) { ts := newTykTestServer() defer ts.Close() - jwtToken := prepareJWTSessionRSABearer(b, true) - + //default values, same as before (keeps backward compatibility) + _, jwtToken := prepareGenericJWTSession(b.Name(), RSASign, KID, false) authHeaders := map[string]string{"authorization": "Bearer " + jwtToken} for i := 0; i < b.N; i++ { ts.Run(b, test.TestCase{ - Headers: authHeaders, Code: 200, + Headers: authHeaders, Code: http.StatusOK, }) } } @@ -330,12 +410,38 @@ func TestJWTSessionRSABearerInvalid(t *testing.T) { ts := newTykTestServer() defer ts.Close() - jwtToken := prepareJWTSessionRSABearer(t, false) + //default values, same as before (keeps backward compatibility) + _, jwtToken := prepareGenericJWTSession(t.Name(), RSASign, KID, false) + authHeaders := map[string]string{"authorization": "Bearer: " + jwtToken} // extra ":" makes the value invalid - authHeaders := map[string]string{"authorization": "Bearer: " + jwtToken} // extra ":" t.Run("Request with invalid Bearer", func(t *testing.T) { ts.Run(t, test.TestCase{ - Headers: authHeaders, Code: 403, + Headers: authHeaders, + Code: http.StatusForbidden, + BodyMatch: "Key not authorized:illegal base64 data at input byte 0", + }) + }) +} + +func TestJWTSessionRSABearerInvalidTwoBears(t *testing.T) { + ts := newTykTestServer() + defer ts.Close() + + //default values, same as before (keeps backward compatibility) + _, jwtToken := prepareGenericJWTSession(t.Name(), RSASign, KID, false) + authHeaders1 := map[string]string{"authorization": "Bearer bearer" + jwtToken} + + t.Run("Request with Bearer bearer", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders1, Code: http.StatusOK, //todo: fix code since it should be http.StatusForbidden + }) + }) + + authHeaders2 := map[string]string{"authorization": "bearer Bearer" + jwtToken} + + t.Run("Request with bearer Bearer", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders2, Code: http.StatusOK, //todo: fix code since it should be http.StatusForbidden }) }) } @@ -348,7 +454,7 @@ func prepareJWTSessionRSAWithRawSourceOnWithClientID(isBench bool) string { spec.OrgID = "default" spec.UseKeylessAccess = false spec.EnableJWT = true - spec.JWTSigningMethod = "rsa" + spec.JWTSigningMethod = RSASign spec.JWTSource = base64.StdEncoding.EncodeToString([]byte(jwtRSAPubKey)) spec.JWTIdentityBaseField = "user_id" spec.JWTClientIDBaseField = "azp" @@ -397,7 +503,7 @@ func TestJWTSessionRSAWithRawSourceOnWithClientID(t *testing.T) { t.Run("Initial request with no policy base field in JWT", func(t *testing.T) { ts.Run(t, test.TestCase{ - Headers: authHeaders, Code: 200, + Headers: authHeaders, Code: http.StatusOK, }) }) } @@ -413,7 +519,7 @@ func BenchmarkJWTSessionRSAWithRawSourceOnWithClientID(b *testing.B) { for i := 0; i < b.N; i++ { ts.Run(b, test.TestCase{ - Headers: authHeaders, Code: 200, + Headers: authHeaders, Code: http.StatusOK, }) } } @@ -424,7 +530,7 @@ func prepareJWTSessionRSAWithRawSource() string { buildAndLoadAPI(func(spec *APISpec) { spec.UseKeylessAccess = false spec.EnableJWT = true - spec.JWTSigningMethod = "rsa" + spec.JWTSigningMethod = RSASign spec.JWTSource = base64.StdEncoding.EncodeToString([]byte(jwtRSAPubKey)) spec.JWTIdentityBaseField = "user_id" spec.JWTPolicyFieldName = "policy_id" @@ -451,9 +557,9 @@ func TestJWTSessionRSAWithRawSource(t *testing.T) { jwtToken := prepareJWTSessionRSAWithRawSource() authHeaders := map[string]string{"authorization": jwtToken} - t.Run("Initial request with invalid policy", func(t *testing.T) { + t.Run("Initial request with valid policy", func(t *testing.T) { ts.Run(t, test.TestCase{ - Headers: authHeaders, Code: 200, + Headers: authHeaders, Code: http.StatusOK, }) }) } @@ -473,7 +579,7 @@ func BenchmarkJWTSessionRSAWithRawSource(b *testing.B) { b, test.TestCase{ Headers: authHeaders, - Code: 200, + Code: http.StatusOK, }, ) } @@ -486,7 +592,7 @@ func TestJWTSessionRSAWithRawSourceInvalidPolicyID(t *testing.T) { spec := buildAPI(func(spec *APISpec) { spec.UseKeylessAccess = false spec.EnableJWT = true - spec.JWTSigningMethod = "rsa" + spec.JWTSigningMethod = RSASign spec.JWTSource = base64.StdEncoding.EncodeToString([]byte(jwtRSAPubKey)) spec.JWTIdentityBaseField = "user_id" spec.JWTPolicyFieldName = "policy_id" @@ -508,7 +614,9 @@ func TestJWTSessionRSAWithRawSourceInvalidPolicyID(t *testing.T) { authHeaders := map[string]string{"authorization": jwtToken} t.Run("Initial request with invalid policy", func(t *testing.T) { ts.Run(t, test.TestCase{ - Headers: authHeaders, Code: 403, + Headers: authHeaders, + Code: http.StatusForbidden, + BodyMatch: "Key not authorized: no matching policy", }) }) } @@ -531,7 +639,7 @@ func TestJWTSessionExpiresAtValidationConfigs(t *testing.T) { spec := buildAPI(func(spec *APISpec) { spec.UseKeylessAccess = false spec.EnableJWT = true - spec.JWTSigningMethod = "rsa" + spec.JWTSigningMethod = RSASign spec.JWTSource = base64.StdEncoding.EncodeToString([]byte(jwtRSAPubKey)) spec.JWTIdentityBaseField = "user_id" spec.JWTPolicyFieldName = "policy_id" @@ -651,7 +759,8 @@ func TestJWTSessionIssueAtValidationConfigs(t *testing.T) { loadAPI(spec) ts.Run(t, test.TestCase{ - Headers: jwtAuthHeaderGen(+time.Second), Code: http.StatusOK, + Headers: jwtAuthHeaderGen(+time.Second), + Code: http.StatusOK, }) }) @@ -751,7 +860,7 @@ func TestJWTExistingSessionRSAWithRawSourceInvalidPolicyID(t *testing.T) { spec := buildAPI(func(spec *APISpec) { spec.UseKeylessAccess = false spec.EnableJWT = true - spec.JWTSigningMethod = "rsa" + spec.JWTSigningMethod = RSASign spec.JWTSource = base64.StdEncoding.EncodeToString([]byte(jwtRSAPubKey)) spec.JWTIdentityBaseField = "user_id" spec.JWTPolicyFieldName = "policy_id" @@ -789,7 +898,9 @@ func TestJWTExistingSessionRSAWithRawSourceInvalidPolicyID(t *testing.T) { authHeaders = map[string]string{"authorization": jwtTokenInvalidPolicy} t.Run("Request with invalid policy in JWT", func(t *testing.T) { ts.Run(t, test.TestCase{ - Headers: authHeaders, Code: http.StatusForbidden, + Headers: authHeaders, + Code: http.StatusForbidden, + BodyMatch: "Key not authorized: no matching policy", }) }) } @@ -801,7 +912,7 @@ func TestJWTExistingSessionRSAWithRawSourcePolicyIDChanged(t *testing.T) { spec := buildAPI(func(spec *APISpec) { spec.UseKeylessAccess = false spec.EnableJWT = true - spec.JWTSigningMethod = "rsa" + spec.JWTSigningMethod = RSASign spec.JWTSource = base64.StdEncoding.EncodeToString([]byte(jwtRSAPubKey)) spec.JWTIdentityBaseField = "user_id" spec.JWTPolicyFieldName = "policy_id" @@ -825,7 +936,7 @@ func TestJWTExistingSessionRSAWithRawSourcePolicyIDChanged(t *testing.T) { t.Claims.(jwt.MapClaims)["exp"] = time.Now().Add(time.Hour * 72).Unix() }) - sessionID := fmt.Sprintf("%x", md5.Sum([]byte("user"))) + sessionID := generateToken("", fmt.Sprintf("%x", md5.Sum([]byte("user")))) authHeaders := map[string]string{"authorization": jwtToken} t.Run("Initial request with 1st policy", func(t *testing.T) { @@ -878,7 +989,7 @@ func prepareJWTSessionRSAWithJWK() string { buildAndLoadAPI(func(spec *APISpec) { spec.UseKeylessAccess = false spec.EnableJWT = true - spec.JWTSigningMethod = "rsa" + spec.JWTSigningMethod = RSASign spec.JWTSource = testHttpJWK spec.JWTIdentityBaseField = "user_id" spec.JWTPolicyFieldName = "policy_id" @@ -937,7 +1048,7 @@ func prepareJWTSessionRSAWithEncodedJWK() (*APISpec, string) { spec := buildAPI(func(spec *APISpec) { spec.UseKeylessAccess = false spec.EnableJWT = true - spec.JWTSigningMethod = "rsa" + spec.JWTSigningMethod = RSASign spec.JWTIdentityBaseField = "user_id" spec.JWTPolicyFieldName = "policy_id" spec.Proxy.ListenPath = "/" @@ -1006,3 +1117,225 @@ func BenchmarkJWTSessionRSAWithEncodedJWK(b *testing.B) { ) } } + +func TestJWTHMACIdNewClaim(t *testing.T) { + ts := newTykTestServer() + defer ts.Close() + + //If we skip the check then the Id will be taken from SUB and the call will succeed + _, jwtToken := prepareGenericJWTSession(t.Name(), HMACSign, "user-id", true) + defer resetTestConfig() + authHeaders := map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/HMAC signature/id in user-id claim", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, Code: http.StatusOK, + }) + }) +} + +func TestJWTRSAIdInClaimsWithBaseField(t *testing.T) { + ts := newTykTestServer() + defer ts.Close() + + buildAndLoadAPI(func(spec *APISpec) { + spec.UseKeylessAccess = false + spec.EnableJWT = true + spec.JWTSigningMethod = RSASign + spec.JWTSource = base64.StdEncoding.EncodeToString([]byte(jwtRSAPubKey)) + spec.JWTIdentityBaseField = "user_id" + spec.JWTPolicyFieldName = "policy_id" + spec.Proxy.ListenPath = "/" + }) + + pID := createPolicy() + + //First test - user id in the configured base field 'user_id' + jwtToken := createJWKToken(func(t *jwt.Token) { + t.Header["kid"] = "12345" + t.Claims.(jwt.MapClaims)["foo"] = "bar" + t.Claims.(jwt.MapClaims)["user_id"] = "user123@test.com" + t.Claims.(jwt.MapClaims)["policy_id"] = pID + t.Claims.(jwt.MapClaims)["exp"] = time.Now().Add(time.Hour * 72).Unix() + }) + authHeaders := map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/RSA signature/user id in user_id claim", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, Code: http.StatusOK, + }) + }) + + //user-id claim configured but it's empty - returning an error + jwtToken = createJWKToken(func(t *jwt.Token) { + t.Header["kid"] = "12345" + t.Claims.(jwt.MapClaims)["foo"] = "bar" + t.Claims.(jwt.MapClaims)["user_id"] = "" + t.Claims.(jwt.MapClaims)["policy_id"] = pID + t.Claims.(jwt.MapClaims)["exp"] = time.Now().Add(time.Hour * 72).Unix() + }) + authHeaders = map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/RSA signature/empty user_id claim", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, + Code: http.StatusForbidden, + BodyMatch: "found an empty user ID in predefined base field claim user_id", + }) + }) + + //user-id claim configured but not found fallback to sub + jwtToken = createJWKToken(func(t *jwt.Token) { + t.Header["kid"] = "12345" + t.Claims.(jwt.MapClaims)["foo"] = "bar" + t.Claims.(jwt.MapClaims)["sub"] = "user123@test.com" + t.Claims.(jwt.MapClaims)["policy_id"] = pID + t.Claims.(jwt.MapClaims)["exp"] = time.Now().Add(time.Hour * 72).Unix() + }) + authHeaders = map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/RSA signature/user id in sub claim", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, Code: http.StatusOK, + }) + }) + + //user-id claim not found fallback to sub that is empty + jwtToken = createJWKToken(func(t *jwt.Token) { + t.Header["kid"] = "12345" + t.Claims.(jwt.MapClaims)["foo"] = "bar" + t.Claims.(jwt.MapClaims)["sub"] = "" + t.Claims.(jwt.MapClaims)["policy_id"] = pID + t.Claims.(jwt.MapClaims)["exp"] = time.Now().Add(time.Hour * 72).Unix() + }) + authHeaders = map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/RSA signature/empty sub claim", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, + Code: http.StatusForbidden, + BodyMatch: "found an empty user ID in sub claim", + }) + }) + + //user-id and sub claims not found + jwtToken = createJWKToken(func(t *jwt.Token) { + t.Header["kid"] = "12345" + t.Claims.(jwt.MapClaims)["foo"] = "bar" + t.Claims.(jwt.MapClaims)["policy_id"] = pID + t.Claims.(jwt.MapClaims)["exp"] = time.Now().Add(time.Hour * 72).Unix() + }) + authHeaders = map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/RSA signature/no base field or sub claims", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, + Code: http.StatusForbidden, + BodyMatch: "no suitable claims for user ID were found", + }) + }) +} + +func TestJWTRSAIdInClaimsWithoutBaseField(t *testing.T) { + ts := newTykTestServer() + defer ts.Close() + + buildAndLoadAPI(func(spec *APISpec) { + spec.UseKeylessAccess = false + spec.EnableJWT = true + spec.JWTSigningMethod = RSASign + spec.JWTSource = base64.StdEncoding.EncodeToString([]byte(jwtRSAPubKey)) + spec.JWTIdentityBaseField = "" + spec.JWTPolicyFieldName = "policy_id" + spec.Proxy.ListenPath = "/" + }) + + pID := createPolicy() + + jwtToken := createJWKToken(func(t *jwt.Token) { + t.Header["kid"] = "12345" + t.Claims.(jwt.MapClaims)["foo"] = "bar" + t.Claims.(jwt.MapClaims)["sub"] = "user123@test.com" //is ignored + t.Claims.(jwt.MapClaims)["policy_id"] = pID + t.Claims.(jwt.MapClaims)["exp"] = time.Now().Add(time.Hour * 72).Unix() + }) + authHeaders := map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/RSA signature/id found in default sub", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, Code: http.StatusOK, + }) + }) + + //Id is not found since there's no sub claim and user_id has't been set in the api def (spec.JWTIdentityBaseField) + jwtToken = createJWKToken(func(t *jwt.Token) { + t.Header["kid"] = "12345" + t.Claims.(jwt.MapClaims)["foo"] = "bar" + t.Claims.(jwt.MapClaims)["user_id"] = "user123@test.com" //is ignored + t.Claims.(jwt.MapClaims)["policy_id"] = pID + t.Claims.(jwt.MapClaims)["exp"] = time.Now().Add(time.Hour * 72).Unix() + }) + authHeaders = map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/RSA signature/no id claims", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, + Code: http.StatusForbidden, + BodyMatch: "no suitable claims for user ID were found", + }) + }) +} + +func TestJWTECDSASign(t *testing.T) { + ts := newTykTestServer() + defer ts.Close() + + //If we skip the check then the Id will be taken from SUB and the call will succeed + _, jwtToken := prepareGenericJWTSession(t.Name(), ECDSASign, KID, false) + defer resetTestConfig() + authHeaders := map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/ECDSA signature needs a test. currently defaults to HMAC", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, Code: http.StatusOK, + }) + }) +} + +func TestJWTUnknownSign(t *testing.T) { + ts := newTykTestServer() + defer ts.Close() + + //If we skip the check then the Id will be taken from SUB and the call will succeed + _, jwtToken := prepareGenericJWTSession(t.Name(), "bla", KID, false) + defer resetTestConfig() + authHeaders := map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/ECDSA signature needs a test. currently defaults to HMAC", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, Code: http.StatusOK, + }) + }) +} + +func TestJWTRSAInvalidPublickKey(t *testing.T) { + ts := newTykTestServer() + defer ts.Close() + + buildAndLoadAPI(func(spec *APISpec) { + spec.UseKeylessAccess = false + spec.EnableJWT = true + spec.JWTSigningMethod = RSASign + spec.JWTSource = base64.StdEncoding.EncodeToString([]byte(jwtRSAPubKeyinvalid)) + spec.JWTPolicyFieldName = "policy_id" + spec.Proxy.ListenPath = "/" + }) + + pID := createPolicy() + + jwtToken := createJWKToken(func(t *jwt.Token) { + t.Header["kid"] = "12345" + t.Claims.(jwt.MapClaims)["foo"] = "bar" + t.Claims.(jwt.MapClaims)["sub"] = "user123@test.com" //is ignored + t.Claims.(jwt.MapClaims)["policy_id"] = pID + t.Claims.(jwt.MapClaims)["exp"] = time.Now().Add(time.Hour * 72).Unix() + }) + authHeaders := map[string]string{"authorization": jwtToken} + t.Run("Request with valid JWT/RSA signature/invalid public key", func(t *testing.T) { + ts.Run(t, test.TestCase{ + Headers: authHeaders, + Code: http.StatusForbidden, + BodyMatch: "Key not authorized:Invalid Key: Key must be PEM encoded PKCS1 or PKCS8 private key", + }) + }) +} diff --git a/mw_oauth2_key_exists.go b/mw_oauth2_key_exists.go index 06b8df8aa40..6180179669f 100644 --- a/mw_oauth2_key_exists.go +++ b/mw_oauth2_key_exists.go @@ -42,7 +42,7 @@ func (k *Oauth2KeyExists) ProcessRequest(w http.ResponseWriter, r *http.Request, } accessToken := parts[1] - session, keyExists := k.CheckSessionAndIdentityForValidKey(accessToken) + session, keyExists := k.CheckSessionAndIdentityForValidKey(accessToken, r) if !keyExists { logEntry = getLogEntryForRequest(r, accessToken, nil) @@ -59,8 +59,7 @@ func (k *Oauth2KeyExists) ProcessRequest(w http.ResponseWriter, r *http.Request, // Set session state on context, we will need it later switch k.Spec.BaseIdentityProvidedBy { case apidef.OAuthKey, apidef.UnsetAuth: - ctxSetSession(r, &session) - ctxSetAuthToken(r, accessToken) + ctxSetSession(r, &session, accessToken, false) } // Request is valid, carry on diff --git a/mw_openid.go b/mw_openid.go index 91496e2a826..7eeafbe6e4c 100644 --- a/mw_openid.go +++ b/mw_openid.go @@ -160,17 +160,17 @@ func (k *OpenIDMW) ProcessRequest(w http.ResponseWriter, r *http.Request, _ inte } data := []byte(ouser.ID) - tokenID := fmt.Sprintf("%x", md5.Sum(data)) - sessionID := k.Spec.OrgID + tokenID + keyID := fmt.Sprintf("%x", md5.Sum(data)) + sessionID := generateToken(k.Spec.OrgID, keyID) if k.Spec.OpenIDOptions.SegregateByClient { // We are segregating by client, so use it as part of the internal token log.Debug("Client ID:", clientID) - sessionID = k.Spec.OrgID + fmt.Sprintf("%x", md5.Sum([]byte(clientID))) + tokenID + sessionID = generateToken(k.Spec.OrgID, fmt.Sprintf("%x", md5.Sum([]byte(clientID)))+keyID) } log.Debug("Generated Session ID: ", sessionID) - session, exists := k.CheckSessionAndIdentityForValidKey(sessionID) + session, exists := k.CheckSessionAndIdentityForValidKey(sessionID, r) if !exists { // Create it log.Debug("Key does not exist, creating") @@ -194,16 +194,13 @@ func (k *OpenIDMW) ProcessRequest(w http.ResponseWriter, r *http.Request, _ inte session.Alias = clientID + ":" + ouser.ID // Update the session in the session manager in case it gets called again - k.Spec.SessionManager.UpdateSession(sessionID, &session, session.Lifetime(k.Spec.SessionLifetime), false) log.Debug("Policy applied to key") - } // 4. Set session state on context, we will need it later switch k.Spec.BaseIdentityProvidedBy { case apidef.OIDCUser, apidef.UnsetAuth: - ctxSetSession(r, &session) - ctxSetAuthToken(r, sessionID) + ctxSetSession(r, &session, sessionID, true) } ctxSetJWTContextVars(k.Spec, r, token) diff --git a/mw_organisation_activity.go b/mw_organisation_activity.go index 4eb80fffd60..8b58323f5f6 100644 --- a/mw_organisation_activity.go +++ b/mw_organisation_activity.go @@ -38,7 +38,9 @@ func (k *OrganizationMonitor) EnabledForSpec() bool { func (k *OrganizationMonitor) ProcessRequest(w http.ResponseWriter, r *http.Request, conf interface{}) (error, int) { if k.Spec.GlobalConfig.ExperimentalProcessOrgOffThread { - return k.ProcessRequestOffThread(r) + // Make a copy of request before before sending to goroutine + r2 := r.WithContext(r.Context()) + return k.ProcessRequestOffThread(r2) } return k.ProcessRequestLive(r) } @@ -62,12 +64,13 @@ func (k *OrganizationMonitor) ProcessRequestLive(r *http.Request) (error, int) { // We found a session, apply the quota and rate limiter reason := k.sessionlimiter.ForwardMessage( + r, &session, k.Spec.OrgID, k.Spec.OrgSessionManager.Store(), session.Per > 0 && session.Rate > 0, true, - k.Spec.GlobalConfig, + &k.Spec.GlobalConfig, ) k.Spec.OrgSessionManager.UpdateSession(k.Spec.OrgID, &session, session.Lifetime(k.Spec.SessionLifetime), false) @@ -186,12 +189,13 @@ func (k *OrganizationMonitor) AllowAccessNext( // We found a session, apply the quota and rate limiter reason := k.sessionlimiter.ForwardMessage( + r, &session, k.Spec.OrgID, k.Spec.OrgSessionManager.Store(), session.Per > 0 && session.Rate > 0, true, - k.Spec.GlobalConfig, + &k.Spec.GlobalConfig, ) k.Spec.OrgSessionManager.UpdateSession(k.Spec.OrgID, &session, session.Lifetime(k.Spec.SessionLifetime), false) diff --git a/mw_rate_limiting.go b/mw_rate_limiting.go index f50bdf587b7..99e204e3ef2 100644 --- a/mw_rate_limiting.go +++ b/mw_rate_limiting.go @@ -66,21 +66,16 @@ func (k *RateLimitAndQuotaCheck) ProcessRequest(w http.ResponseWriter, r *http.R token := ctxGetAuthToken(r) storeRef := k.Spec.SessionManager.Store() - reason := sessionLimiter.ForwardMessage(session, + reason := sessionLimiter.ForwardMessage( + r, + session, token, storeRef, !k.Spec.DisableRateLimit, !k.Spec.DisableQuota, - k.Spec.GlobalConfig, + &k.Spec.GlobalConfig, ) - // If either are disabled, save the write roundtrip - if !k.Spec.DisableRateLimit || !k.Spec.DisableQuota { - // Ensure quota and rate data for this session are recorded - k.Spec.SessionManager.UpdateSession(token, session, session.Lifetime(k.Spec.SessionLifetime), false) - ctxSetSession(r, session) - } - switch reason { case sessionFailNone: case sessionFailRateLimit: diff --git a/mw_redis_cache.go b/mw_redis_cache.go index 2c34554704f..59459877b48 100644 --- a/mw_redis_cache.go +++ b/mw_redis_cache.go @@ -101,7 +101,7 @@ func (m *RedisCacheMiddleware) decodePayload(payload string) (string, string, er func (m *RedisCacheMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Request, _ interface{}) (error, int) { // Only allow idempotent (safe) methods - if r.Method != "GET" && r.Method != "HEAD" { + if r.Method != "GET" && r.Method != "HEAD" && r.Method != "OPTIONS" { return nil, http.StatusOK } diff --git a/mw_url_rewrite.go b/mw_url_rewrite.go index 114ada99b97..d9265d53825 100644 --- a/mw_url_rewrite.go +++ b/mw_url_rewrite.go @@ -239,6 +239,14 @@ func valToStr(v interface{}) string { } i++ } + case []interface{}: + tmpSlice := make([]string, 0, len(x)) + for _, val := range x { + if rec := valToStr(val); rec != "" { + tmpSlice = append(tmpSlice, url.QueryEscape(rec)) + } + } + s = strings.Join(tmpSlice, ",") default: log.Error("Context variable type is not supported: ", reflect.TypeOf(v)) } diff --git a/mw_url_rewrite_test.go b/mw_url_rewrite_test.go index 626ee8b6944..5c6a6166bfe 100644 --- a/mw_url_rewrite_test.go +++ b/mw_url_rewrite_test.go @@ -512,7 +512,7 @@ func TestRewriterTriggers(t *testing.T) { MetaData: map[string]interface{}{ "rewrite": "bar", }, - }) + }, "", false) got, err := urlRewrite(&testConf, tc.req) if err != nil { @@ -621,3 +621,20 @@ func TestInitTriggerRx(t *testing.T) { t.Errorf("Expected PayloadMatches initalized and matched, received no match") } } + +func TestValToStr(t *testing.T) { + + example := []interface{}{ + "abc", // string + int64(456), // int64 + 12.22, // float + "abc,def", // string url encode + } + + str := valToStr(example) + expected := "abc,456,12.22,abc%2Cdef" + + if str != expected { + t.Errorf("expected (%s) got (%s)", expected, str) + } +} diff --git a/mw_virtual_endpoint.go b/mw_virtual_endpoint.go index 63d1f0b7d98..888e5091046 100644 --- a/mw_virtual_endpoint.go +++ b/mw_virtual_endpoint.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "net/http" "os" + "reflect" "strconv" "time" @@ -145,7 +146,6 @@ func (d *VirtualEndpoint) ServeHTTPForCache(w http.ResponseWriter, r *http.Reque specAsJson := specToJson(d.Spec) session := new(user.SessionState) - token := ctxGetAuthToken(r) // Encode the session object (if not a pre-process) if vmeta.UseSession { @@ -214,9 +214,11 @@ func (d *VirtualEndpoint) ServeHTTPForCache(w http.ResponseWriter, r *http.Reque // Save the sesison data (if modified) if vmeta.UseSession { - session.MetaData = mapStrsToIfaces(newResponseData.SessionMeta) - d.Spec.SessionManager.UpdateSession(token, session, session.Lifetime(d.Spec.SessionLifetime), false) - ctxSetSession(r, session) + newMeta := mapStrsToIfaces(newResponseData.SessionMeta) + if !reflect.DeepEqual(session.MetaData, newMeta) { + session.MetaData = newMeta + ctxSetSession(r, session, "", true) + } } log.Debug("JSVM Virtual Endpoint execution took: (ns) ", time.Now().UnixNano()-t1) diff --git a/oauth_manager.go b/oauth_manager.go index 5972c0ee6ec..865ee2b6049 100644 --- a/oauth_manager.go +++ b/oauth_manager.go @@ -321,7 +321,7 @@ func (o *OAuthManager) HandleAccess(r *http.Request) *osin.Response { log.Debug("New token: ", new_token.(string)) log.Debug("Keys: ", session.OauthKeys) - keyName := o.API.OrgID + username + keyName := generateToken(o.API.OrgID, username) log.Debug("Updating user:", keyName) err := o.API.SessionManager.UpdateSession(keyName, session, session.Lifetime(o.API.SessionLifetime), false) diff --git a/rpc_storage_handler.go b/rpc_storage_handler.go index bb7c5bf0a50..acf73fb9ed9 100644 --- a/rpc_storage_handler.go +++ b/rpc_storage_handler.go @@ -8,6 +8,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/garyburd/redigo/redis" @@ -260,11 +261,15 @@ func (r *RPCStorageHandler) cleanKey(keyName string) string { } var rpcLoginMu sync.Mutex +var reLoginRunning uint32 func (r *RPCStorageHandler) ReAttemptLogin(err error) bool { - rpcLoginMu.Lock() - log.Warning("[RPC Store] Login failed, waiting 3s to re-attempt") + if atomic.LoadUint32(&reLoginRunning) == 1 { + return false + } + atomic.StoreUint32(&reLoginRunning, 1) + rpcLoginMu.Lock() if rpcLoadCount == 0 && !rpcEmergencyModeLoaded { log.Warning("[RPC Store] --> Detected cold start, attempting to load from cache") log.Warning("[RPC Store] ----> Found APIs... beginning emergency load") @@ -274,10 +279,15 @@ func (r *RPCStorageHandler) ReAttemptLogin(err error) bool { rpcLoginMu.Unlock() time.Sleep(time.Second * 3) + atomic.StoreUint32(&reLoginRunning, 0) + if strings.Contains(err.Error(), "Cannot obtain response during timeout") { r.ReConnect() return false } + + log.Warning("[RPC Store] Login failed, waiting 3s to re-attempt") + return r.Login() } @@ -887,6 +897,8 @@ func (r *RPCStorageHandler) CheckForReload(orgId string) { } else if !strings.Contains(err.Error(), "Cannot obtain response during") { log.Warning("[RPC STORE] RPC Reload Checker encountered unexpected error: ", err) } + + time.Sleep(1 * time.Second) } else if reload == true { // Do the reload! log.Warning("[RPC STORE] Received Reload instruction!") @@ -916,6 +928,10 @@ func (r *RPCStorageHandler) StartRPCKeepaliveWatcher() { for { if err := r.SetKey("0000", "0000", 10); err != nil { + log.WithError(err).WithFields(logrus.Fields{ + "prefix": "RPC Conn Mgr", + }).Info("Can't connect to RPC layer") + if r.IsAccessError(err) { if r.Login() { continue @@ -923,17 +939,10 @@ func (r *RPCStorageHandler) StartRPCKeepaliveWatcher() { } if strings.Contains(err.Error(), "Cannot obtain response during timeout") { - log.WithFields(logrus.Fields{ - "prefix": "RPC Conn Mgr", - }).Info("Can't connect to RPC layer") continue } } - log.WithFields(logrus.Fields{ - "prefix": "RPC Conn Mgr", - }).Info("RPC is alive") - time.Sleep(10 * time.Second) } } diff --git a/service_discovery.go b/service_discovery.go index 3a06b31dd0a..f51a1309c1f 100644 --- a/service_discovery.go +++ b/service_discovery.go @@ -6,7 +6,7 @@ import ( "strconv" "strings" - "github.com/jeffail/gabs" + "github.com/Jeffail/gabs" "github.com/TykTechnologies/tyk/apidef" ) diff --git a/session_manager.go b/session_manager.go index 12d69bddb27..e10d2eefc1c 100644 --- a/session_manager.go +++ b/session_manager.go @@ -1,6 +1,7 @@ package main import ( + "net/http" "time" "github.com/TykTechnologies/leakybucket" @@ -33,7 +34,7 @@ type SessionLimiter struct { bucketStore leakybucket.Storage } -func (l *SessionLimiter) doRollingWindowWrite(key, rateLimiterKey, rateLimiterSentinelKey string, currentSession *user.SessionState, store storage.Handler, globalConf config.Config) bool { +func (l *SessionLimiter) doRollingWindowWrite(key, rateLimiterKey, rateLimiterSentinelKey string, currentSession *user.SessionState, store storage.Handler, globalConf *config.Config) bool { log.Debug("[RATELIMIT] Inbound raw key is: ", key) log.Debug("[RATELIMIT] Rate limiter key is: ", rateLimiterKey) pipeline := globalConf.EnableNonTransactionalRateLimiter @@ -49,7 +50,6 @@ func (l *SessionLimiter) doRollingWindowWrite(key, rateLimiterKey, rateLimiterSe } //log.Info("break: ", (int(currentSession.Rate) - subtractor)) - if ratePerPeriodNow > int(currentSession.Rate)-subtractor { // Set a sentinel value with expire if globalConf.EnableSentinelRateLImiter { @@ -73,12 +73,12 @@ const ( // sessionFailReason if session limits have been exceeded. // Key values to manage rate are Rate and Per, e.g. Rate of 10 messages // Per 10 seconds -func (l *SessionLimiter) ForwardMessage(currentSession *user.SessionState, key string, store storage.Handler, enableRL, enableQ bool, globalConf config.Config) sessionFailReason { - rateLimiterKey := RateLimitKeyPrefix + storage.HashKey(key) - rateLimiterSentinelKey := RateLimitKeyPrefix + storage.HashKey(key) + ".BLOCKED" - +func (l *SessionLimiter) ForwardMessage(r *http.Request, currentSession *user.SessionState, key string, store storage.Handler, enableRL, enableQ bool, globalConf *config.Config) sessionFailReason { if enableRL { if globalConf.EnableSentinelRateLImiter { + rateLimiterKey := RateLimitKeyPrefix + currentSession.KeyHash() + rateLimiterSentinelKey := RateLimitKeyPrefix + currentSession.KeyHash() + ".BLOCKED" + go l.doRollingWindowWrite(key, rateLimiterKey, rateLimiterSentinelKey, currentSession, store, globalConf) // Check sentinel @@ -88,6 +88,9 @@ func (l *SessionLimiter) ForwardMessage(currentSession *user.SessionState, key s return sessionFailRateLimit } } else if globalConf.EnableRedisRollingLimiter { + rateLimiterKey := RateLimitKeyPrefix + currentSession.KeyHash() + rateLimiterSentinelKey := RateLimitKeyPrefix + currentSession.KeyHash() + ".BLOCKED" + if l.doRollingWindowWrite(key, rateLimiterKey, rateLimiterSentinelKey, currentSession, store, globalConf) { return sessionFailRateLimit } @@ -128,7 +131,7 @@ func (l *SessionLimiter) ForwardMessage(currentSession *user.SessionState, key s currentSession.Allowance-- } - if l.RedisQuotaExceeded(currentSession, key, store) { + if l.RedisQuotaExceeded(r, currentSession, key, store) { return sessionFailQuota } } @@ -137,7 +140,7 @@ func (l *SessionLimiter) ForwardMessage(currentSession *user.SessionState, key s } -func (l *SessionLimiter) RedisQuotaExceeded(currentSession *user.SessionState, key string, store storage.Handler) bool { +func (l *SessionLimiter) RedisQuotaExceeded(r *http.Request, currentSession *user.SessionState, key string, store storage.Handler) bool { // Are they unlimited? if currentSession.QuotaMax == -1 { // No quota set @@ -146,7 +149,7 @@ func (l *SessionLimiter) RedisQuotaExceeded(currentSession *user.SessionState, k // Create the key log.Debug("[QUOTA] Inbound raw key is: ", key) - rawKey := QuotaKeyPrefix + storage.HashKey(key) + rawKey := QuotaKeyPrefix + currentSession.KeyHash() log.Debug("[QUOTA] Quota limiter key is: ", rawKey) log.Debug("Renewing with TTL: ", currentSession.QuotaRenewalRate) // INCR the key (If it equals 1 - set EXPIRE) @@ -176,6 +179,7 @@ func (l *SessionLimiter) RedisQuotaExceeded(currentSession *user.SessionState, k if qInt == 1 { current := time.Now().Unix() currentSession.QuotaRenews = current + currentSession.QuotaRenewalRate + ctxScheduleSessionUpdate(r) } // If not, pass and set the values of the session to quotamax - counter @@ -186,5 +190,6 @@ func (l *SessionLimiter) RedisQuotaExceeded(currentSession *user.SessionState, k } else { currentSession.QuotaRemaining = remaining } + return false } diff --git a/storage/redis_cluster.go b/storage/redis_cluster.go index 4562432fea4..98db67413bc 100644 --- a/storage/redis_cluster.go +++ b/storage/redis_cluster.go @@ -545,7 +545,7 @@ func (r RedisCluster) GetAndDeleteSet(keyName string) []interface{} { return nil } - log.Debug("Analytics returned: ", redVal) + log.Debug("Analytics returned: ", len(redVal)) if len(redVal) == 0 { return nil } diff --git a/storage/storage.go b/storage/storage.go index e5c6ac566ce..f1c135dc096 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -1,11 +1,18 @@ package storage import ( + "crypto/sha256" + "encoding/base64" "encoding/hex" "errors" + "fmt" + "hash" + "strings" - "github.com/spaolacci/murmur3" + "github.com/buger/jsonparser" + "github.com/satori/go.uuid" + "github.com/TykTechnologies/murmur3" "github.com/TykTechnologies/tyk/config" logger "github.com/TykTechnologies/tyk/log" ) @@ -47,8 +54,85 @@ type Handler interface { RemoveSortedSetRange(string, string, string) error } +const defaultHashAlgorithm = "murmur64" + +// If hashing algorithm is empty, use legacy key generation +func GenerateToken(orgID, keyID, hashAlgorithm string) (string, error) { + if keyID == "" { + keyID = strings.Replace(uuid.NewV4().String(), "-", "", -1) + } + + if hashAlgorithm != "" { + _, err := hashFunction(hashAlgorithm) + if err != nil { + hashAlgorithm = defaultHashAlgorithm + } + + jsonToken := fmt.Sprintf(`{"org":"%s","id":"%s","h":"%s"}`, orgID, keyID, hashAlgorithm) + return base64.StdEncoding.EncodeToString([]byte(jsonToken)), err + } + + // Legacy keys + return orgID + keyID, nil +} + +// `{"` in base64 +const B64JSONPrefix = "ey" + +func TokenHashAlgo(token string) string { + // Legacy tokens not b64 and not JSON records + if strings.HasPrefix(token, B64JSONPrefix) { + if jsonToken, err := base64.StdEncoding.DecodeString(token); err == nil { + hashAlgo, _ := jsonparser.GetString(jsonToken, "h") + return hashAlgo + } + } + + return "" +} + +func TokenOrg(token string) string { + if strings.HasPrefix(token, B64JSONPrefix) { + if jsonToken, err := base64.StdEncoding.DecodeString(token); err == nil { + // Checking error in case if it is a legacy tooken which just by accided has the same b64JSON prefix + if org, err := jsonparser.GetString(jsonToken, "org"); err == nil { + return org + } + } + } + + // 24 is mongo bson id length + if len(token) > 24 { + return token[:24] + } + + return "" +} + +var ( + HashSha256 = "sha256" + HashMurmur32 = "murmur32" + HashMurmur64 = "murmur64" + HashMurmur128 = "murmur128" +) + +func hashFunction(algorithm string) (hash.Hash, error) { + switch algorithm { + case HashSha256: + return sha256.New(), nil + case HashMurmur64: + return murmur3.New64(), nil + case HashMurmur128: + return murmur3.New128(), nil + case "", HashMurmur32: + return murmur3.New32(), nil + default: + return murmur3.New32(), fmt.Errorf("Unknown key hash function: %s. Falling back to murmur32.", algorithm) + } +} + func HashStr(in string) string { - h := murmur3.New32() + h, _ := hashFunction(TokenHashAlgo(in)) h.Write([]byte(in)) return hex.EncodeToString(h.Sum(nil)) } diff --git a/templates/default_webhook.json b/templates/default_webhook.json index bdd6ffcf657..495bd63a886 100644 --- a/templates/default_webhook.json +++ b/templates/default_webhook.json @@ -6,7 +6,7 @@ "response_code": "{{.Meta.HostInfo.ResponseCode}}", "tcp_error": "{{.Meta.HostInfo.IsTCPError}}", "host": "{{.Meta.HostInfo.MetaData.host_name}}", - "api_id": "{{.Meta.HostInfo.MetaData.api_id}}", + "api_id": "{{.Meta.HostInfo.MetaData.api_id}}" } {{ else if eq .Type "HostUp"}} { @@ -16,7 +16,7 @@ "response_code": "{{.Meta.HostInfo.ResponseCode}}", "tcp_error": "{{.Meta.HostInfo.IsTCPError}}", "host": "{{.Meta.HostInfo.MetaData.host_name}}", - "api_id": "{{.Meta.HostInfo.MetaData.api_id}}", + "api_id": "{{.Meta.HostInfo.MetaData.api_id}}" } {{ else if eq .Type "TriggerExceeded"}} { diff --git a/tyk.conf.example b/tyk.conf.example index 1881da30fd3..537ffbdab1e 100644 --- a/tyk.conf.example +++ b/tyk.conf.example @@ -13,7 +13,8 @@ "username": "", "password": "", "database": 0, - "optimisation_max_idle": 500 + "optimisation_max_idle": 2000, + "optimisation_max_active": 4000 }, "enable_analytics": false, "analytics_config": { @@ -23,8 +24,10 @@ "optimisations_use_async_session_write": true, "allow_master_keys": false, "policies": { - "policy_source": "file", - }, + "policy_source": "file" + }, "hash_keys": true, - "suppress_redis_signal_reload": false + "suppress_redis_signal_reload": false, + "force_global_session_lifetime": false, + "max_idle_connections_per_host": 500 } diff --git a/user/session.go b/user/session.go index 6cc0d300808..bcd6d67eb4e 100644 --- a/user/session.go +++ b/user/session.go @@ -1,9 +1,6 @@ package user import ( - "github.com/spaolacci/murmur3" - "gopkg.in/vmihailenco/msgpack.v2" - "github.com/TykTechnologies/tyk/config" logger "github.com/TykTechnologies/tyk/log" ) @@ -72,27 +69,24 @@ type SessionState struct { IdExtractorDeadline int64 `json:"id_extractor_deadline" msg:"id_extractor_deadline"` SessionLifetime int64 `bson:"session_lifetime" json:"session_lifetime"` - firstSeenHash string + // Used to store token hash + keyHash string } -func (s *SessionState) SetFirstSeenHash() { - s.firstSeenHash = s.Hash() +func (s *SessionState) KeyHash() string { + if s.keyHash == "" { + panic("KeyHash cache not found. You should call `SetKeyHash` before.") + } + + return s.keyHash } -func (s *SessionState) Hash() string { - encoded, err := msgpack.Marshal(s) - if err != nil { - log.Error("Error encoding session data: ", err) - return "" - } - murmurHasher := murmur3.New32() - murmurHasher.Write(encoded) - murmurHasher.Sum32() - return string(murmurHasher.Sum(nil)[:]) +func (s *SessionState) SetKeyHash(hash string) { + s.keyHash = hash } -func (s *SessionState) HasChanged() bool { - return s.firstSeenHash == "" || s.firstSeenHash != s.Hash() +func (s *SessionState) KeyHashEmpty() bool { + return s.keyHash == "" } func (s *SessionState) Lifetime(fallback int64) int64 { diff --git a/user/session_test.go b/user/session_test.go deleted file mode 100644 index a4b77d44d0b..00000000000 --- a/user/session_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package user - -import ( - "testing" -) - -func BenchmarkHash(b *testing.B) { - s := SessionState{ - Allowance: 1000.0, - Rate: 1000.0, - Per: 1, - Expires: 1458669677, - QuotaRemaining: 1000, - QuotaRenewalRate: 3600, - } - - b.ReportAllocs() - for i := 0; i < b.N; i++ { - { - s.Hash() - } - } -} diff --git a/vendor/github.com/spaolacci/murmur3/LICENSE b/vendor/github.com/TykTechnologies/murmur3/LICENSE similarity index 100% rename from vendor/github.com/spaolacci/murmur3/LICENSE rename to vendor/github.com/TykTechnologies/murmur3/LICENSE diff --git a/vendor/github.com/spaolacci/murmur3/README.md b/vendor/github.com/TykTechnologies/murmur3/README.md similarity index 97% rename from vendor/github.com/spaolacci/murmur3/README.md rename to vendor/github.com/TykTechnologies/murmur3/README.md index 1edf62300d2..e463678a05e 100644 --- a/vendor/github.com/spaolacci/murmur3/README.md +++ b/vendor/github.com/TykTechnologies/murmur3/README.md @@ -1,6 +1,8 @@ murmur3 ======= +[![Build Status](https://travis-ci.org/spaolacci/murmur3.svg?branch=master)](https://travis-ci.org/spaolacci/murmur3) + Native Go implementation of Austin Appleby's third MurmurHash revision (aka MurmurHash3). diff --git a/vendor/github.com/spaolacci/murmur3/murmur.go b/vendor/github.com/TykTechnologies/murmur3/murmur.go similarity index 95% rename from vendor/github.com/spaolacci/murmur3/murmur.go rename to vendor/github.com/TykTechnologies/murmur3/murmur.go index f99557cc3ec..1252cf73a79 100644 --- a/vendor/github.com/spaolacci/murmur3/murmur.go +++ b/vendor/github.com/TykTechnologies/murmur3/murmur.go @@ -3,8 +3,6 @@ // license that can be found in the LICENSE file. /* -Native (and fast) implementation of Austin Appleby's MurmurHash3. - Package murmur3 implements Austin Appleby's non-cryptographic MurmurHash3. Reference implementation: @@ -26,6 +24,7 @@ type digest struct { clen int // Digested input cumulative length. tail []byte // 0 to Size()-1 bytes view of `buf'. buf [16]byte // Expected (but not required) to be Size() large. + seed uint32 // Seed for initializing the hash. bmixer } diff --git a/vendor/github.com/spaolacci/murmur3/murmur128.go b/vendor/github.com/TykTechnologies/murmur3/murmur128.go similarity index 80% rename from vendor/github.com/spaolacci/murmur3/murmur128.go rename to vendor/github.com/TykTechnologies/murmur3/murmur128.go index 16c34d6fbc8..a4b618b5f3d 100644 --- a/vendor/github.com/spaolacci/murmur3/murmur128.go +++ b/vendor/github.com/TykTechnologies/murmur3/murmur128.go @@ -18,6 +18,7 @@ var ( _ bmixer = new(digest128) ) +// Hash128 represents a 128-bit hasher // Hack: the standard api doesn't define any Hash128 interface. type Hash128 interface { hash.Hash @@ -31,8 +32,13 @@ type digest128 struct { h2 uint64 // Unfinalized running hash part 2. } -func New128() Hash128 { +// New128 returns a 128-bit hasher +func New128() Hash128 { return New128WithSeed(0) } + +// New128WithSeed returns a 128-bit hasher set with explicit seed value +func New128WithSeed(seed uint32) Hash128 { d := new(digest128) + d.seed = seed d.bmixer = d d.Reset() return d @@ -40,10 +46,10 @@ func New128() Hash128 { func (d *digest128) Size() int { return 16 } -func (d *digest128) reset() { d.h1, d.h2 = 0, 0 } +func (d *digest128) reset() { d.h1, d.h2 = uint64(d.seed), uint64(d.seed) } func (d *digest128) Sum(b []byte) []byte { - h1, h2 := d.h1, d.h2 + h1, h2 := d.Sum128() return append(b, byte(h1>>56), byte(h1>>48), byte(h1>>40), byte(h1>>32), byte(h1>>24), byte(h1>>16), byte(h1>>8), byte(h1), @@ -181,8 +187,16 @@ func rotl64(x uint64, r byte) uint64 { // hasher := New128() // hasher.Write(data) // return hasher.Sum128() -func Sum128(data []byte) (h1 uint64, h2 uint64) { - d := &digest128{h1: 0, h2: 0} +func Sum128(data []byte) (h1 uint64, h2 uint64) { return Sum128WithSeed(data, 0) } + +// Sum128WithSeed returns the MurmurHash3 sum of data. It is equivalent to the +// following sequence (without the extra burden and the extra allocation): +// hasher := New128WithSeed(seed) +// hasher.Write(data) +// return hasher.Sum128() +func Sum128WithSeed(data []byte, seed uint32) (h1 uint64, h2 uint64) { + d := &digest128{h1: uint64(seed), h2: uint64(seed)} + d.seed = seed d.tail = d.bmix(data) d.clen = len(data) return d.Sum128() diff --git a/vendor/github.com/spaolacci/murmur3/murmur32.go b/vendor/github.com/TykTechnologies/murmur3/murmur32.go similarity index 100% rename from vendor/github.com/spaolacci/murmur3/murmur32.go rename to vendor/github.com/TykTechnologies/murmur3/murmur32.go diff --git a/vendor/github.com/spaolacci/murmur3/murmur64.go b/vendor/github.com/TykTechnologies/murmur3/murmur64.go similarity index 53% rename from vendor/github.com/spaolacci/murmur3/murmur64.go rename to vendor/github.com/TykTechnologies/murmur3/murmur64.go index fdd4398e398..65a410ae0b9 100644 --- a/vendor/github.com/spaolacci/murmur3/murmur64.go +++ b/vendor/github.com/TykTechnologies/murmur3/murmur64.go @@ -14,13 +14,17 @@ var ( // digest64 is half a digest128. type digest64 digest128 -func New64() hash.Hash64 { - d := (*digest64)(New128().(*digest128)) +// New64 returns a 64-bit hasher +func New64() hash.Hash64 { return New64WithSeed(0) } + +// New64WithSeed returns a 64-bit hasher set with explicit seed value +func New64WithSeed(seed uint32) hash.Hash64 { + d := (*digest64)(New128WithSeed(seed).(*digest128)) return d } func (d *digest64) Sum(b []byte) []byte { - h1 := d.h1 + h1 := d.Sum64() return append(b, byte(h1>>56), byte(h1>>48), byte(h1>>40), byte(h1>>32), byte(h1>>24), byte(h1>>16), byte(h1>>8), byte(h1)) @@ -36,8 +40,16 @@ func (d *digest64) Sum64() uint64 { // hasher := New64() // hasher.Write(data) // return hasher.Sum64() -func Sum64(data []byte) uint64 { - d := &digest128{h1: 0, h2: 0} +func Sum64(data []byte) uint64 { return Sum64WithSeed(data, 0) } + +// Sum64WithSeed returns the MurmurHash3 sum of data. It is equivalent to the +// following sequence (without the extra burden and the extra allocation): +// hasher := New64WithSeed(seed) +// hasher.Write(data) +// return hasher.Sum64() +func Sum64WithSeed(data []byte, seed uint32) uint64 { + d := &digest128{h1: uint64(seed), h2: uint64(seed)} + d.seed = seed d.tail = d.bmix(data) d.clen = len(data) h1, _ := d.Sum128() diff --git a/vendor/github.com/buger/jsonparser/Dockerfile b/vendor/github.com/buger/jsonparser/Dockerfile new file mode 100644 index 00000000000..37fc9fd0b4d --- /dev/null +++ b/vendor/github.com/buger/jsonparser/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.6 + +RUN go get github.com/Jeffail/gabs +RUN go get github.com/bitly/go-simplejson +RUN go get github.com/pquerna/ffjson +RUN go get github.com/antonholmquist/jason +RUN go get github.com/mreiferson/go-ujson +RUN go get -tags=unsafe -u github.com/ugorji/go/codec +RUN go get github.com/mailru/easyjson + +WORKDIR /go/src/github.com/buger/jsonparser +ADD . /go/src/github.com/buger/jsonparser \ No newline at end of file diff --git a/vendor/github.com/buger/jsonparser/LICENSE b/vendor/github.com/buger/jsonparser/LICENSE new file mode 100644 index 00000000000..ac25aeb7da2 --- /dev/null +++ b/vendor/github.com/buger/jsonparser/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 Leonid Bugaev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/buger/jsonparser/Makefile b/vendor/github.com/buger/jsonparser/Makefile new file mode 100644 index 00000000000..e843368cf10 --- /dev/null +++ b/vendor/github.com/buger/jsonparser/Makefile @@ -0,0 +1,36 @@ +SOURCE = parser.go +CONTAINER = jsonparser +SOURCE_PATH = /go/src/github.com/buger/jsonparser +BENCHMARK = JsonParser +BENCHTIME = 5s +TEST = . +DRUN = docker run -v `pwd`:$(SOURCE_PATH) -i -t $(CONTAINER) + +build: + docker build -t $(CONTAINER) . + +race: + $(DRUN) --env GORACE="halt_on_error=1" go test ./. $(ARGS) -v -race -timeout 15s + +bench: + $(DRUN) go test $(LDFLAGS) -test.benchmem -bench $(BENCHMARK) ./benchmark/ $(ARGS) -benchtime $(BENCHTIME) -v + +bench_local: + $(DRUN) go test $(LDFLAGS) -test.benchmem -bench . $(ARGS) -benchtime $(BENCHTIME) -v + +profile: + $(DRUN) go test $(LDFLAGS) -test.benchmem -bench $(BENCHMARK) ./benchmark/ $(ARGS) -memprofile mem.mprof -v + $(DRUN) go test $(LDFLAGS) -test.benchmem -bench $(BENCHMARK) ./benchmark/ $(ARGS) -cpuprofile cpu.out -v + $(DRUN) go test $(LDFLAGS) -test.benchmem -bench $(BENCHMARK) ./benchmark/ $(ARGS) -c + +test: + $(DRUN) go test $(LDFLAGS) ./ -run $(TEST) -timeout 10s $(ARGS) -v + +fmt: + $(DRUN) go fmt ./... + +vet: + $(DRUN) go vet ./. + +bash: + $(DRUN) /bin/bash \ No newline at end of file diff --git a/vendor/github.com/buger/jsonparser/README.md b/vendor/github.com/buger/jsonparser/README.md new file mode 100644 index 00000000000..73264f254e5 --- /dev/null +++ b/vendor/github.com/buger/jsonparser/README.md @@ -0,0 +1,353 @@ +[![Go Report Card](https://goreportcard.com/badge/github.com/buger/jsonparser)](https://goreportcard.com/report/github.com/buger/jsonparser) ![License](https://img.shields.io/dub/l/vibe-d.svg) +# Alternative JSON parser for Go (so far fastest) + +It does not require you to know the structure of the payload (eg. create structs), and allows accessing fields by providing the path to them. It is up to **10 times faster** than standard `encoding/json` package (depending on payload size and usage), **allocates no memory**. See benchmarks below. + +## Rationale +Originally I made this for a project that relies on a lot of 3rd party APIs that can be unpredictable and complex. +I love simplicity and prefer to avoid external dependecies. `encoding/json` requires you to know exactly your data structures, or if you prefer to use `map[string]interface{}` instead, it will be very slow and hard to manage. +I investigated what's on the market and found that most libraries are just wrappers around `encoding/json`, there is few options with own parsers (`ffjson`, `easyjson`), but they still requires you to create data structures. + + +Goal of this project is to push JSON parser to the performance limits and not sucrifice with compliance and developer user experience. + +## Example +For the given JSON our goal is to extract the user's full name, number of github followers and avatar. + +```go +import "github.com/buger/jsonparser" + +... + +data := []byte(`{ + "person": { + "name": { + "first": "Leonid", + "last": "Bugaev", + "fullName": "Leonid Bugaev" + }, + "github": { + "handle": "buger", + "followers": 109 + }, + "avatars": [ + { "url": "https://avatars1.githubusercontent.com/u/14009?v=3&s=460", "type": "thumbnail" } + ] + }, + "company": { + "name": "Acme" + } +}`) + +// You can specify key path by providing arguments to Get function +jsonparser.Get(data, "person", "name", "fullName") + +// There is `GetInt` and `GetBoolean` helpers if you exactly know key data type +jsonparser.GetInt(data, "person", "github", "followers") + +// When you try to get object, it will return you []byte slice pointer to data containing it +// In `company` it will be `{"name": "Acme"}` +jsonparser.Get(data, "company") + +// If the key doesn't exist it will throw an error +var size int64 +if value, _, err := jsonparser.GetInt(data, "company", "size"); err == nil { + size = value +} + +// You can use `ArrayEach` helper to iterate items [item1, item2 .... itemN] +jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { + fmt.Println(jsonparser.Get(value, "url")) +}, "person", "avatars") + +// Or use can access fields by index! +jsonparser.GetInt("person", "avatars", "[0]", "url") + +// You can use `ObjectEach` helper to iterate objects { "key1":object1, "key2":object2, .... "keyN":objectN } +jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { + fmt.Printf("Key: '%s'\n Value: '%s'\n Type: %s\n", string(key), string(value), dataType) + return nil +}, "person", "name") + +// The most efficient way to extract multiple keys is `EachKey` + +paths := [][]string{ + []string{"person", "name", "fullName"}, + []string{"person", "avatars", "[0]", "url"}, + []string{"company", "url"}, +} +jsonparser.EachKey(data, func(idx int, value []byte, vt jsonparser.ValueType, err error){ + switch idx { + case 0: // []string{"person", "name", "fullName"} + ... + case 1: // []string{"person", "avatars", "[0]", "url"} + ... + case 2: // []string{"company", "url"}, + ... + } +}, paths...) + +// For more information see docs below +``` + +## Need to speedup your app? + +I'm available for consulting and can help you push your app performance to the limits. Ping me at: leonsbox@gmail.com. + +## Reference + +Library API is really simple. You just need the `Get` method to perform any operation. The rest is just helpers around it. + +You also can view API at [godoc.org](https://godoc.org/github.com/buger/jsonparser) + + +### **`Get`** +```go +func Get(data []byte, keys ...string) (value []byte, dataType jsonparser.ValueType, offset int, err error) +``` +Receives data structure, and key path to extract value from. + +Returns: +* `value` - Pointer to original data structure containing key value, or just empty slice if nothing found or error +* `dataType` - Can be: `NotExist`, `String`, `Number`, `Object`, `Array`, `Boolean` or `Null` +* `offset` - Offset from provided data structure where key value ends. Used mostly internally, for example for `ArrayEach` helper. +* `err` - If the key is not found or any other parsing issue, it should return error. If key not found it also sets `dataType` to `NotExist` + +Accepts multiple keys to specify path to JSON value (in case of quering nested structures). +If no keys are provided it will try to extract the closest JSON value (simple ones or object/array), useful for reading streams or arrays, see `ArrayEach` implementation. + +Note that keys can be an array indexes: `jsonparser.GetInt("person", "avatars", "[0]", "url")`, pretty cool, yeah? + +### **`GetString`** +```go +func GetString(data []byte, keys ...string) (val string, err error) +``` +Returns strings properly handing escaped and unicode characters. Note that this will cause additional memory allocations. + +### **`GetUnsafeString`** +If you need string in your app, and ready to sacrifice with support of escaped symbols in favor of speed. It returns string mapped to existing byte slice memory, without any allocations: +```go +s, _, := jsonparser.GetUnsafeString(data, "person", "name", "title") +switch s { + case 'CEO': + ... + case 'Engineer' + ... + ... +} +``` +Note that `unsafe` here means that your string will exist until GC will free underlying byte slice, for most of cases it means that you can use this string only in current context, and should not pass it anywhere externally: through channels or any other way. + + +### **`GetBoolean`**, **`GetInt`** and **`GetFloat`** +```go +func GetBoolean(data []byte, keys ...string) (val bool, err error) + +func GetFloat(data []byte, keys ...string) (val float64, err error) + +func GetInt(data []byte, keys ...string) (val float64, err error) +``` +If you know the key type, you can use the helpers above. +If key data type do not match, it will return error. + +### **`ArrayEach`** +```go +func ArrayEach(data []byte, cb func(value []byte, dataType jsonparser.ValueType, offset int, err error), keys ...string) +``` +Needed for iterating arrays, accepts a callback function with the same return arguments as `Get`. + +### **`ObjectEach`** +```go +func ObjectEach(data []byte, callback func(key []byte, value []byte, dataType ValueType, offset int) error, keys ...string) (err error) +``` +Needed for iterating object, accepts a callback function. Example: +```go +var handler func([]byte, []byte, jsonparser.ValueType, int) error +handler = func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { + //do stuff here +} +jsonparser.ObjectEach(myJson, handler) +``` + + +### **`EachKey`** +```go +func EachKey(data []byte, cb func(idx int, value []byte, dataType jsonparser.ValueType, err error), paths ...[]string) +``` +When you need to read multiple keys, and you do not afraid of low-level API `EachKey` is your friend. It read payload only single time, and calls callback function once path is found. For example when you call multiple times `Get`, it has to process payload multiple times, each time you call it. Depending on payload `EachKey` can be multiple times faster than `Get`. Path can use nested keys as well! + +```go +paths := [][]string{ + []string{"uuid"}, + []string{"tz"}, + []string{"ua"}, + []string{"st"}, +} +var data SmallPayload + +jsonparser.EachKey(smallFixture, func(idx int, value []byte, vt jsonparser.ValueType, err error){ + switch idx { + case 0: + data.Uuid, _ = value + case 1: + v, _ := jsonparser.ParseInt(value) + data.Tz = int(v) + case 2: + data.Ua, _ = value + case 3: + v, _ := jsonparser.ParseInt(value) + data.St = int(v) + } +}, paths...) +``` + +### **`Set`** +```go +func Set(data []byte, setValue []byte, keys ...string) (value []byte, err error) +``` +Receives existing data structure, key path to set, and value to set at that key. *This functionality is experimental.* + +Returns: +* `value` - Pointer to original data structure with updated or added key value. +* `err` - If any parsing issue, it should return error. + +Accepts multiple keys to specify path to JSON value (in case of updating or creating nested structures). + +Note that keys can be an array indexes: `jsonparser.Set(data, []byte("http://github.com"), "person", "avatars", "[0]", "url")` + + +## What makes it so fast? +* It does not rely on `encoding/json`, `reflection` or `interface{}`, the only real package dependency is `bytes`. +* Operates with JSON payload on byte level, providing you pointers to the original data structure: no memory allocation. +* No automatic type conversions, by default everything is a []byte, but it provides you value type, so you can convert by yourself (there is few helpers included). +* Does not parse full record, only keys you specified + + +## Benchmarks + +There are 3 benchmark types, trying to simulate real-life usage for small, medium and large JSON payloads. +For each metric, the lower value is better. Time/op is in nanoseconds. Values better than standard encoding/json marked as bold text. +Benchmarks run on standard Linode 1024 box. + +Compared libraries: +* https://golang.org/pkg/encoding/json +* https://github.com/Jeffail/gabs +* https://github.com/a8m/djson +* https://github.com/bitly/go-simplejson +* https://github.com/antonholmquist/jason +* https://github.com/mreiferson/go-ujson +* https://github.com/ugorji/go/codec +* https://github.com/pquerna/ffjson +* https://github.com/mailru/easyjson +* https://github.com/buger/jsonparser + +#### TLDR +If you want to skip next sections we have 2 winner: `jsonparser` and `easyjson`. +`jsonparser` is up to 10 times faster than standard `encoding/json` package (depending on payload size and usage), and almost infinitely (literally) better in memory consumption because it operates with data on byte level, and provide direct slice pointers. +`easyjson` wins in CPU in medium tests and frankly i'm impressed with this package: it is remarkable results considering that it is almost drop-in replacement for `encoding/json` (require some code generation). + +It's hard to fully compare `jsonparser` and `easyjson` (or `ffson`), they a true parsers and fully process record, unlike `jsonparser` which parse only keys you specified. + +If you searching for replacement of `encoding/json` while keeping structs, `easyjson` is an amazing choise. If you want to process dynamic JSON, have memory constrains, or more control over your data you should try `jsonparser`. + +`jsonparser` performance heavily depends on usage, and it works best when you do not need to process full record, only some keys. The more calls you need to make, the slower it will be, in contrast `easyjson` (or `ffjson`, `encoding/json`) parser record only 1 time, and then you can make as many calls as you want. + +With great power comes great responsibility! :) + + +#### Small payload + +Each test processes 190 bytes of http log as a JSON record. +It should read multiple fields. +https://github.com/buger/jsonparser/blob/master/benchmark/benchmark_small_payload_test.go + +Library | time/op | bytes/op | allocs/op + ------ | ------- | -------- | ------- +encoding/json struct | 7879 | 880 | 18 +encoding/json interface{} | 8946 | 1521 | 38 +Jeffail/gabs | 10053 | 1649 | 46 +bitly/go-simplejson | 10128 | 2241 | 36 +antonholmquist/jason | 27152 | 7237 | 101 +github.com/ugorji/go/codec | 8806 | 2176 | 31 +mreiferson/go-ujson | **7008** | **1409** | 37 +a8m/djson | 3862 | 1249 | 30 +pquerna/ffjson | **3769** | **624** | **15** +mailru/easyjson | **2002** | **192** | **9** +buger/jsonparser | **1367** | **0** | **0** +buger/jsonparser (EachKey API) | **809** | **0** | **0** + +Winners are ffjson, easyjson and jsonparser, where jsonparser is up to 9.8x faster than encoding/json and 4.6x faster than ffjson, and slightly faster than easyjson. +If you look at memory allocation, jsonparser has no rivals, as it makes no data copy and operates with raw []byte structures and pointers to it. + +#### Medium payload + +Each test processes a 2.4kb JSON record (based on Clearbit API). +It should read multiple nested fields and 1 array. + +https://github.com/buger/jsonparser/blob/master/benchmark/benchmark_medium_payload_test.go + +| Library | time/op | bytes/op | allocs/op | +| ------- | ------- | -------- | --------- | +| encoding/json struct | 57749 | 1336 | 29 | +| encoding/json interface{} | 79297 | 10627 | 215 | +| Jeffail/gabs | 83807 | 11202 | 235 | +| bitly/go-simplejson | 88187 | 17187 | 220 | +| antonholmquist/jason | 94099 | 19013 | 247 | +| github.com/ugorji/go/codec | 114719 | 6712 | 152 | +| mreiferson/go-ujson | **56972** | 11547 | 270 | +| a8m/djson | 28525 | 10196 | 198 | +| pquerna/ffjson | **20298** | **856** | **20** | +| mailru/easyjson | **10512** | **336** | **12** | +| buger/jsonparser | **15955** | **0** | **0** | +| buger/jsonparser (EachKey API) | **8916** | **0** | **0** | + +The difference between ffjson and jsonparser in CPU usage is smaller, while the memory consumption difference is growing. On the other hand `easyjson` shows remarkable performance for medium payload. + +`gabs`, `go-simplejson` and `jason` are based on encoding/json and map[string]interface{} and actually only helpers for unstructured JSON, their performance correlate with `encoding/json interface{}`, and they will skip next round. +`go-ujson` while have its own parser, shows same performance as `encoding/json`, also skips next round. Same situation with `ugorji/go/codec`, but it showed unexpectedly bad performance for complex payloads. + + +#### Large payload + +Each test processes a 24kb JSON record (based on Discourse API) +It should read 2 arrays, and for each item in array get a few fields. +Basically it means processing a full JSON file. + +https://github.com/buger/jsonparser/blob/master/benchmark/benchmark_large_payload_test.go + +| Library | time/op | bytes/op | allocs/op | +| --- | --- | --- | --- | +| encoding/json struct | 748336 | 8272 | 307 | +| encoding/json interface{} | 1224271 | 215425 | 3395 | +| a8m/djson | 510082 | 213682 | 2845 | +| pquerna/ffjson | **312271** | **7792** | **298** | +| mailru/easyjson | **154186** | **6992** | **288** | +| buger/jsonparser | **85308** | **0** | **0** | + +`jsonparser` now is a winner, but do not forget that it is way more lighweight parser than `ffson` or `easyjson`, and they have to parser all the data, while `jsonparser` parse only what you need. All `ffjson`, `easysjon` and `jsonparser` have their own parsing code, and does not depend on `encoding/json` or `interface{}`, thats one of the reasons why they are so fast. `easyjson` also use a bit of `unsafe` package to reduce memory consuption (in theory it can lead to some unexpected GC issue, but i did not tested enough) + +Also last benchmark did not included `EachKey` test, because in this particular case we need to read lot of Array values, and using `ArrayEach` is more efficient. + +## Questions and support + +All bug-reports and suggestions should go though Github Issues. +If you have some private questions you can send them directly to me: leonsbox@gmail.com + +## Contributing + +1. Fork it +2. Create your feature branch (git checkout -b my-new-feature) +3. Commit your changes (git commit -am 'Added some feature') +4. Push to the branch (git push origin my-new-feature) +5. Create new Pull Request + +## Development + +All my development happens using Docker, and repo include some Make tasks to simplify development. + +* `make build` - builds docker image, usually can be called only once +* `make test` - run tests +* `make fmt` - run go fmt +* `make bench` - run benchmarks (if you need to run only single benchmark modify `BENCHMARK` variable in make file) +* `make profile` - runs benchmark and generate 3 files- `cpu.out`, `mem.mprof` and `benchmark.test` binary, which can be used for `go tool pprof` +* `make bash` - enter container (i use it for running `go tool pprof` above) diff --git a/vendor/github.com/buger/jsonparser/bytes.go b/vendor/github.com/buger/jsonparser/bytes.go new file mode 100644 index 00000000000..d8e76492994 --- /dev/null +++ b/vendor/github.com/buger/jsonparser/bytes.go @@ -0,0 +1,28 @@ +package jsonparser + +// About 3x faster then strconv.ParseInt because does not check for range error and support only base 10, which is enough for JSON +func parseInt(bytes []byte) (v int64, ok bool) { + if len(bytes) == 0 { + return 0, false + } + + var neg bool = false + if bytes[0] == '-' { + neg = true + bytes = bytes[1:] + } + + for _, c := range bytes { + if c >= '0' && c <= '9' { + v = (10 * v) + int64(c-'0') + } else { + return 0, false + } + } + + if neg { + return -v, true + } else { + return v, true + } +} diff --git a/vendor/github.com/buger/jsonparser/bytes_safe.go b/vendor/github.com/buger/jsonparser/bytes_safe.go new file mode 100644 index 00000000000..f9b95973ef5 --- /dev/null +++ b/vendor/github.com/buger/jsonparser/bytes_safe.go @@ -0,0 +1,21 @@ +// +build appengine appenginevm + +package jsonparser + +import ( + "strconv" +) + +// See fastbytes_unsafe.go for explanation on why *[]byte is used (signatures must be consistent with those in that file) + +func equalStr(b *[]byte, s string) bool { + return string(*b) == s +} + +func parseFloat(b *[]byte) (float64, error) { + return strconv.ParseFloat(string(*b), 64) +} + +func bytesToString(b *[]byte) string { + return string(*b) +} diff --git a/vendor/github.com/buger/jsonparser/bytes_unsafe.go b/vendor/github.com/buger/jsonparser/bytes_unsafe.go new file mode 100644 index 00000000000..847e96ad239 --- /dev/null +++ b/vendor/github.com/buger/jsonparser/bytes_unsafe.go @@ -0,0 +1,31 @@ +// +build !appengine,!appenginevm + +package jsonparser + +import ( + "strconv" + "unsafe" +) + +// +// The reason for using *[]byte rather than []byte in parameters is an optimization. As of Go 1.6, +// the compiler cannot perfectly inline the function when using a non-pointer slice. That is, +// the non-pointer []byte parameter version is slower than if its function body is manually +// inlined, whereas the pointer []byte version is equally fast to the manually inlined +// version. Instruction count in assembly taken from "go tool compile" confirms this difference. +// +// TODO: Remove hack after Go 1.7 release +// +func equalStr(b *[]byte, s string) bool { + return *(*string)(unsafe.Pointer(b)) == s +} + +func parseFloat(b *[]byte) (float64, error) { + return strconv.ParseFloat(*(*string)(unsafe.Pointer(b)), 64) +} + +// A hack until issue golang/go#2632 is fixed. +// See: https://github.com/golang/go/issues/2632 +func bytesToString(b *[]byte) string { + return *(*string)(unsafe.Pointer(b)) +} diff --git a/vendor/github.com/buger/jsonparser/escape.go b/vendor/github.com/buger/jsonparser/escape.go new file mode 100644 index 00000000000..49669b94207 --- /dev/null +++ b/vendor/github.com/buger/jsonparser/escape.go @@ -0,0 +1,173 @@ +package jsonparser + +import ( + "bytes" + "unicode/utf8" +) + +// JSON Unicode stuff: see https://tools.ietf.org/html/rfc7159#section-7 + +const supplementalPlanesOffset = 0x10000 +const highSurrogateOffset = 0xD800 +const lowSurrogateOffset = 0xDC00 + +const basicMultilingualPlaneReservedOffset = 0xDFFF +const basicMultilingualPlaneOffset = 0xFFFF + +func combineUTF16Surrogates(high, low rune) rune { + return supplementalPlanesOffset + (high-highSurrogateOffset)<<10 + (low - lowSurrogateOffset) +} + +const badHex = -1 + +func h2I(c byte) int { + switch { + case c >= '0' && c <= '9': + return int(c - '0') + case c >= 'A' && c <= 'F': + return int(c - 'A' + 10) + case c >= 'a' && c <= 'f': + return int(c - 'a' + 10) + } + return badHex +} + +// decodeSingleUnicodeEscape decodes a single \uXXXX escape sequence. The prefix \u is assumed to be present and +// is not checked. +// In JSON, these escapes can either come alone or as part of "UTF16 surrogate pairs" that must be handled together. +// This function only handles one; decodeUnicodeEscape handles this more complex case. +func decodeSingleUnicodeEscape(in []byte) (rune, bool) { + // We need at least 6 characters total + if len(in) < 6 { + return utf8.RuneError, false + } + + // Convert hex to decimal + h1, h2, h3, h4 := h2I(in[2]), h2I(in[3]), h2I(in[4]), h2I(in[5]) + if h1 == badHex || h2 == badHex || h3 == badHex || h4 == badHex { + return utf8.RuneError, false + } + + // Compose the hex digits + return rune(h1<<12 + h2<<8 + h3<<4 + h4), true +} + +// isUTF16EncodedRune checks if a rune is in the range for non-BMP characters, +// which is used to describe UTF16 chars. +// Source: https://en.wikipedia.org/wiki/Plane_(Unicode)#Basic_Multilingual_Plane +func isUTF16EncodedRune(r rune) bool { + return highSurrogateOffset <= r && r <= basicMultilingualPlaneReservedOffset +} + +func decodeUnicodeEscape(in []byte) (rune, int) { + if r, ok := decodeSingleUnicodeEscape(in); !ok { + // Invalid Unicode escape + return utf8.RuneError, -1 + } else if r <= basicMultilingualPlaneOffset && !isUTF16EncodedRune(r) { + // Valid Unicode escape in Basic Multilingual Plane + return r, 6 + } else if r2, ok := decodeSingleUnicodeEscape(in[6:]); !ok { // Note: previous decodeSingleUnicodeEscape success guarantees at least 6 bytes remain + // UTF16 "high surrogate" without manditory valid following Unicode escape for the "low surrogate" + return utf8.RuneError, -1 + } else if r2 < lowSurrogateOffset { + // Invalid UTF16 "low surrogate" + return utf8.RuneError, -1 + } else { + // Valid UTF16 surrogate pair + return combineUTF16Surrogates(r, r2), 12 + } +} + +// backslashCharEscapeTable: when '\X' is found for some byte X, it is to be replaced with backslashCharEscapeTable[X] +var backslashCharEscapeTable = [...]byte{ + '"': '"', + '\\': '\\', + '/': '/', + 'b': '\b', + 'f': '\f', + 'n': '\n', + 'r': '\r', + 't': '\t', +} + +// unescapeToUTF8 unescapes the single escape sequence starting at 'in' into 'out' and returns +// how many characters were consumed from 'in' and emitted into 'out'. +// If a valid escape sequence does not appear as a prefix of 'in', (-1, -1) to signal the error. +func unescapeToUTF8(in, out []byte) (inLen int, outLen int) { + if len(in) < 2 || in[0] != '\\' { + // Invalid escape due to insufficient characters for any escape or no initial backslash + return -1, -1 + } + + // https://tools.ietf.org/html/rfc7159#section-7 + switch e := in[1]; e { + case '"', '\\', '/', 'b', 'f', 'n', 'r', 't': + // Valid basic 2-character escapes (use lookup table) + out[0] = backslashCharEscapeTable[e] + return 2, 1 + case 'u': + // Unicode escape + if r, inLen := decodeUnicodeEscape(in); inLen == -1 { + // Invalid Unicode escape + return -1, -1 + } else { + // Valid Unicode escape; re-encode as UTF8 + outLen := utf8.EncodeRune(out, r) + return inLen, outLen + } + } + + return -1, -1 +} + +// unescape unescapes the string contained in 'in' and returns it as a slice. +// If 'in' contains no escaped characters: +// Returns 'in'. +// Else, if 'out' is of sufficient capacity (guaranteed if cap(out) >= len(in)): +// 'out' is used to build the unescaped string and is returned with no extra allocation +// Else: +// A new slice is allocated and returned. +func Unescape(in, out []byte) ([]byte, error) { + firstBackslash := bytes.IndexByte(in, '\\') + if firstBackslash == -1 { + return in, nil + } + + // Get a buffer of sufficient size (allocate if needed) + if cap(out) < len(in) { + out = make([]byte, len(in)) + } else { + out = out[0:len(in)] + } + + // Copy the first sequence of unescaped bytes to the output and obtain a buffer pointer (subslice) + copy(out, in[:firstBackslash]) + in = in[firstBackslash:] + buf := out[firstBackslash:] + + for len(in) > 0 { + // Unescape the next escaped character + inLen, bufLen := unescapeToUTF8(in, buf) + if inLen == -1 { + return nil, MalformedStringEscapeError + } + + in = in[inLen:] + buf = buf[bufLen:] + + // Copy everything up until the next backslash + nextBackslash := bytes.IndexByte(in, '\\') + if nextBackslash == -1 { + copy(buf, in) + buf = buf[len(in):] + break + } else { + copy(buf, in[:nextBackslash]) + buf = buf[nextBackslash:] + in = in[nextBackslash:] + } + } + + // Trim the out buffer to the amount that was actually emitted + return out[:len(out)-len(buf)], nil +} diff --git a/vendor/github.com/buger/jsonparser/parser.go b/vendor/github.com/buger/jsonparser/parser.go new file mode 100644 index 00000000000..b85f0cd2966 --- /dev/null +++ b/vendor/github.com/buger/jsonparser/parser.go @@ -0,0 +1,999 @@ +package jsonparser + +import ( + "bytes" + "errors" + "fmt" + "math" + "strconv" + "strings" +) + +// Errors +var ( + KeyPathNotFoundError = errors.New("Key path not found") + UnknownValueTypeError = errors.New("Unknown value type") + MalformedJsonError = errors.New("Malformed JSON error") + MalformedStringError = errors.New("Value is string, but can't find closing '\"' symbol") + MalformedArrayError = errors.New("Value is array, but can't find closing ']' symbol") + MalformedObjectError = errors.New("Value looks like object, but can't find closing '}' symbol") + MalformedValueError = errors.New("Value looks like Number/Boolean/None, but can't find its end: ',' or '}' symbol") + MalformedStringEscapeError = errors.New("Encountered an invalid escape sequence in a string") +) + +// How much stack space to allocate for unescaping JSON strings; if a string longer +// than this needs to be escaped, it will result in a heap allocation +const unescapeStackBufSize = 64 + +func tokenEnd(data []byte) int { + for i, c := range data { + switch c { + case ' ', '\n', '\r', '\t', ',', '}', ']': + return i + } + } + + return len(data) +} + +// Find position of next character which is not whitespace +func nextToken(data []byte) int { + for i, c := range data { + switch c { + case ' ', '\n', '\r', '\t': + continue + default: + return i + } + } + + return -1 +} + +// Find position of last character which is not whitespace +func lastToken(data []byte) int { + for i := len(data) - 1; i >= 0; i-- { + switch data[i] { + case ' ', '\n', '\r', '\t': + continue + default: + return i + } + } + + return -1 +} + +// Tries to find the end of string +// Support if string contains escaped quote symbols. +func stringEnd(data []byte) (int, bool) { + escaped := false + for i, c := range data { + if c == '"' { + if !escaped { + return i + 1, false + } else { + j := i - 1 + for { + if j < 0 || data[j] != '\\' { + return i + 1, true // even number of backslashes + } + j-- + if j < 0 || data[j] != '\\' { + break // odd number of backslashes + } + j-- + + } + } + } else if c == '\\' { + escaped = true + } + } + + return -1, escaped +} + +// Find end of the data structure, array or object. +// For array openSym and closeSym will be '[' and ']', for object '{' and '}' +func blockEnd(data []byte, openSym byte, closeSym byte) int { + level := 0 + i := 0 + ln := len(data) + + for i < ln { + switch data[i] { + case '"': // If inside string, skip it + se, _ := stringEnd(data[i+1:]) + if se == -1 { + return -1 + } + i += se + case openSym: // If open symbol, increase level + level++ + case closeSym: // If close symbol, increase level + level-- + + // If we have returned to the original level, we're done + if level == 0 { + return i + 1 + } + } + i++ + } + + return -1 +} + +func searchKeys(data []byte, keys ...string) int { + keyLevel := 0 + level := 0 + i := 0 + ln := len(data) + lk := len(keys) + + if lk == 0 { + return 0 + } + + var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings + + for i < ln { + switch data[i] { + case '"': + i++ + keyBegin := i + + strEnd, keyEscaped := stringEnd(data[i:]) + if strEnd == -1 { + return -1 + } + i += strEnd + keyEnd := i - 1 + + valueOffset := nextToken(data[i:]) + if valueOffset == -1 { + return -1 + } + + i += valueOffset + + // if string is a key, and key level match + if data[i] == ':' && keyLevel == level-1 { + key := data[keyBegin:keyEnd] + + // for unescape: if there are no escape sequences, this is cheap; if there are, it is a + // bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize + var keyUnesc []byte + if !keyEscaped { + keyUnesc = key + } else if ku, err := Unescape(key, stackbuf[:]); err != nil { + return -1 + } else { + keyUnesc = ku + } + + if equalStr(&keyUnesc, keys[level-1]) { + keyLevel++ + // If we found all keys in path + if keyLevel == lk { + return i + 1 + } + } + } else { + i-- + } + case '{': + level++ + case '}': + level-- + if level == keyLevel { + keyLevel-- + } + case '[': + // If we want to get array element by index + if keyLevel == level && keys[level][0] == '[' { + aIdx, err := strconv.Atoi(keys[level][1 : len(keys[level])-1]) + if err != nil { + return -1 + } + var curIdx int + var valueFound []byte + var valueOffset int + ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { + if curIdx == aIdx { + valueFound = value + valueOffset = offset + if dataType == String { + valueOffset = valueOffset - 2 + valueFound = data[i + valueOffset:i + valueOffset + len(value) + 2] + } + } + curIdx += 1 + }) + + if valueFound == nil { + return -1 + } else { + subIndex := searchKeys(valueFound, keys[level+1:]...) + if subIndex < 0 { + return -1 + } + return i + valueOffset + subIndex + } + } else { + // Do not search for keys inside arrays + if arraySkip := blockEnd(data[i:], '[', ']'); arraySkip == -1 { + return -1 + } else { + i += arraySkip - 1 + } + } + } + + i++ + } + + return -1 +} + +var bitwiseFlags []int64 + +func init() { + for i := 0; i < 63; i++ { + bitwiseFlags = append(bitwiseFlags, int64(math.Pow(2, float64(i)))) + } +} + +func sameTree(p1, p2 []string) bool { + minLen := len(p1) + if len(p2) < minLen { + minLen = len(p2) + } + + for pi_1, p_1 := range p1[:minLen] { + if p2[pi_1] != p_1 { + return false + } + } + + return true +} + +func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]string) int { + var pathFlags int64 + var level, pathsMatched, i int + ln := len(data) + + var maxPath int + for _, p := range paths { + if len(p) > maxPath { + maxPath = len(p) + } + } + + var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings + pathsBuf := make([]string, maxPath) + + for i < ln { + switch data[i] { + case '"': + i++ + keyBegin := i + + strEnd, keyEscaped := stringEnd(data[i:]) + if strEnd == -1 { + return -1 + } + i += strEnd + + keyEnd := i - 1 + + valueOffset := nextToken(data[i:]) + if valueOffset == -1 { + return -1 + } + + i += valueOffset + + // if string is a key, and key level match + if data[i] == ':' { + match := -1 + key := data[keyBegin:keyEnd] + + // for unescape: if there are no escape sequences, this is cheap; if there are, it is a + // bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize + var keyUnesc []byte + if !keyEscaped { + keyUnesc = key + } else if ku, err := Unescape(key, stackbuf[:]); err != nil { + return -1 + } else { + keyUnesc = ku + } + + if maxPath >= level { + if level < 1 { + cb(-1, []byte{}, Unknown, MalformedJsonError) + return -1 + } + pathsBuf[level-1] = bytesToString(&keyUnesc) + + for pi, p := range paths { + if len(p) != level || pathFlags&bitwiseFlags[pi+1] != 0 || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) { + continue + } + + match = pi + + i++ + pathsMatched++ + pathFlags |= bitwiseFlags[pi+1] + + v, dt, of, e := Get(data[i:]) + cb(pi, v, dt, e) + + if of != -1 { + i += of + } + + if pathsMatched == len(paths) { + return i + } + } + } + + if match == -1 { + tokenOffset := nextToken(data[i+1:]) + i += tokenOffset + + if data[i] == '{' { + blockSkip := blockEnd(data[i:], '{', '}') + i += blockSkip + 1 + } + } + + switch data[i] { + case '{', '}', '[', '"': + i-- + } + } else { + i-- + } + case '{': + level++ + case '}': + level-- + case '[': + var arrIdxFlags int64 + var pIdxFlags int64 + + if level < 0 { + cb(-1, []byte{}, Unknown, MalformedJsonError) + return -1 + } + + for pi, p := range paths { + if len(p) < level+1 || pathFlags&bitwiseFlags[pi+1] != 0 || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) { + continue + } + + aIdx, _ := strconv.Atoi(p[level][1 : len(p[level])-1]) + arrIdxFlags |= bitwiseFlags[aIdx+1] + pIdxFlags |= bitwiseFlags[pi+1] + } + + if arrIdxFlags > 0 { + level++ + + var curIdx int + arrOff, _ := ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { + if arrIdxFlags&bitwiseFlags[curIdx+1] != 0 { + for pi, p := range paths { + if pIdxFlags&bitwiseFlags[pi+1] != 0 { + aIdx, _ := strconv.Atoi(p[level-1][1 : len(p[level-1])-1]) + + if curIdx == aIdx { + of := searchKeys(value, p[level:]...) + + pathsMatched++ + pathFlags |= bitwiseFlags[pi+1] + + if of != -1 { + v, dt, _, e := Get(value[of:]) + cb(pi, v, dt, e) + } + } + } + } + } + + curIdx += 1 + }) + + if pathsMatched == len(paths) { + return i + } + + i += arrOff - 1 + } else { + // Do not search for keys inside arrays + if arraySkip := blockEnd(data[i:], '[', ']'); arraySkip == -1 { + return -1 + } else { + i += arraySkip - 1 + } + } + case ']': + level-- + } + + i++ + } + + return -1 +} + +// Data types available in valid JSON data. +type ValueType int + +const ( + NotExist = ValueType(iota) + String + Number + Object + Array + Boolean + Null + Unknown +) + +func (vt ValueType) String() string { + switch vt { + case NotExist: + return "non-existent" + case String: + return "string" + case Number: + return "number" + case Object: + return "object" + case Array: + return "array" + case Boolean: + return "boolean" + case Null: + return "null" + default: + return "unknown" + } +} + +var ( + trueLiteral = []byte("true") + falseLiteral = []byte("false") + nullLiteral = []byte("null") +) + +func createInsertComponent(keys []string, setValue []byte, comma, object bool) []byte { + var buffer bytes.Buffer + if comma { + buffer.WriteString(",") + } + if object { + buffer.WriteString("{") + } + buffer.WriteString("\"") + buffer.WriteString(keys[0]) + buffer.WriteString("\":") + for i := 1; i < len(keys); i++ { + buffer.WriteString("{\"") + buffer.WriteString(keys[i]) + buffer.WriteString("\":") + } + buffer.Write(setValue) + buffer.WriteString(strings.Repeat("}", len(keys)-1)) + if object { + buffer.WriteString("}") + } + return buffer.Bytes() +} + +/* + +Set - Receives existing data structure, path to set, and data to set at that key. + +Returns: +`value` - modified byte array +`err` - On any parsing error + +*/ +func Set(data []byte, setValue []byte, keys ...string) (value []byte, err error) { + // ensure keys are set + if len(keys) == 0 { + return nil, KeyPathNotFoundError + } + + _, _, startOffset, endOffset, err := internalGet(data, keys...) + if err != nil { + if err != KeyPathNotFoundError { + // problem parsing the data + return []byte{}, err + } + // full path doesnt exist + // does any subpath exist? + var depth int + for i := range keys { + _, _, start, end, sErr := internalGet(data, keys[:i+1]...) + if sErr != nil { + break + } else { + endOffset = end + startOffset = start + depth++ + } + } + comma := true + object := false + if endOffset == -1 { + firstToken := nextToken(data) + // We can't set a top-level key if data isn't an object + if len(data) == 0 || data[firstToken] != '{' { + return nil, KeyPathNotFoundError + } + // Don't need a comma if the input is an empty object + secondToken := firstToken + 1 + nextToken(data[firstToken+1:]) + if data[secondToken] == '}' { + comma = false + } + // Set the top level key at the end (accounting for any trailing whitespace) + // This assumes last token is valid like '}', could check and return error + endOffset = lastToken(data) + } + depthOffset := endOffset + if depth != 0 { + // if subpath is a non-empty object, add to it + if data[startOffset] == '{' && data[startOffset+1+nextToken(data[startOffset+1:])]!='}' { + depthOffset-- + startOffset = depthOffset + // otherwise, over-write it with a new object + } else { + comma = false + object = true + } + } else { + startOffset = depthOffset + } + value = append(data[:startOffset], append(createInsertComponent(keys[depth:], setValue, comma, object), data[depthOffset:]...)...) + } else { + // path currently exists + startComponent := data[:startOffset] + endComponent := data[endOffset:] + + value = make([]byte, len(startComponent)+len(endComponent)+len(setValue)) + newEndOffset := startOffset + len(setValue) + copy(value[0:startOffset], startComponent) + copy(value[startOffset:newEndOffset], setValue) + copy(value[newEndOffset:], endComponent) + } + return value, nil +} + +func getType(data []byte, offset int) ([]byte, ValueType, int, error) { + var dataType ValueType + endOffset := offset + + // if string value + if data[offset] == '"' { + dataType = String + if idx, _ := stringEnd(data[offset+1:]); idx != -1 { + endOffset += idx + 1 + } else { + return []byte{}, dataType, offset, MalformedStringError + } + } else if data[offset] == '[' { // if array value + dataType = Array + // break label, for stopping nested loops + endOffset = blockEnd(data[offset:], '[', ']') + + if endOffset == -1 { + return []byte{}, dataType, offset, MalformedArrayError + } + + endOffset += offset + } else if data[offset] == '{' { // if object value + dataType = Object + // break label, for stopping nested loops + endOffset = blockEnd(data[offset:], '{', '}') + + if endOffset == -1 { + return []byte{}, dataType, offset, MalformedObjectError + } + + endOffset += offset + } else { + // Number, Boolean or None + end := tokenEnd(data[endOffset:]) + + if end == -1 { + return nil, dataType, offset, MalformedValueError + } + + value := data[offset : endOffset+end] + + switch data[offset] { + case 't', 'f': // true or false + if bytes.Equal(value, trueLiteral) || bytes.Equal(value, falseLiteral) { + dataType = Boolean + } else { + return nil, Unknown, offset, UnknownValueTypeError + } + case 'u', 'n': // undefined or null + if bytes.Equal(value, nullLiteral) { + dataType = Null + } else { + return nil, Unknown, offset, UnknownValueTypeError + } + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-': + dataType = Number + default: + return nil, Unknown, offset, UnknownValueTypeError + } + + endOffset += end + } + return data[offset:endOffset], dataType, endOffset, nil +} + +/* +Get - Receives data structure, and key path to extract value from. + +Returns: +`value` - Pointer to original data structure containing key value, or just empty slice if nothing found or error +`dataType` - Can be: `NotExist`, `String`, `Number`, `Object`, `Array`, `Boolean` or `Null` +`offset` - Offset from provided data structure where key value ends. Used mostly internally, for example for `ArrayEach` helper. +`err` - If key not found or any other parsing issue it should return error. If key not found it also sets `dataType` to `NotExist` + +Accept multiple keys to specify path to JSON value (in case of quering nested structures). +If no keys provided it will try to extract closest JSON value (simple ones or object/array), useful for reading streams or arrays, see `ArrayEach` implementation. +*/ +func Get(data []byte, keys ...string) (value []byte, dataType ValueType, offset int, err error) { + a, b, _, d, e := internalGet(data, keys...) + return a, b, d, e +} + +func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, offset, endOffset int, err error) { + if len(keys) > 0 { + if offset = searchKeys(data, keys...); offset == -1 { + return []byte{}, NotExist, -1, -1, KeyPathNotFoundError + } + } + + // Go to closest value + nO := nextToken(data[offset:]) + if nO == -1 { + return []byte{}, NotExist, offset, -1, MalformedJsonError + } + + offset += nO + value, dataType, endOffset, err = getType(data, offset) + if err != nil { + return value, dataType, offset, endOffset, err + } + + // Strip quotes from string values + if dataType == String { + value = value[1 : len(value)-1] + } + + return value, dataType, offset, endOffset, nil +} + +// ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`. +func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int, err error), keys ...string) (offset int, err error) { + if len(data) == 0 { + return -1, MalformedObjectError + } + + offset = 1 + + if len(keys) > 0 { + if offset = searchKeys(data, keys...); offset == -1 { + return offset, KeyPathNotFoundError + } + + // Go to closest value + nO := nextToken(data[offset:]) + if nO == -1 { + return offset, MalformedJsonError + } + + offset += nO + + if data[offset] != '[' { + return offset, MalformedArrayError + } + + offset++ + } + + nO := nextToken(data[offset:]) + if nO == -1 { + return offset, MalformedJsonError + } + + offset += nO + + if data[offset] == ']' { + return offset, nil + } + + for true { + v, t, o, e := Get(data[offset:]) + + if e != nil { + return offset, e + } + + if o == 0 { + break + } + + if t != NotExist { + cb(v, t, offset+o-len(v), e) + } + + if e != nil { + break + } + + offset += o + + skipToToken := nextToken(data[offset:]) + if skipToToken == -1 { + return offset, MalformedArrayError + } + offset += skipToToken + + if data[offset] == ']' { + break + } + + if data[offset] != ',' { + return offset, MalformedArrayError + } + + offset++ + } + + return offset, nil +} + +// ObjectEach iterates over the key-value pairs of a JSON object, invoking a given callback for each such entry +func ObjectEach(data []byte, callback func(key []byte, value []byte, dataType ValueType, offset int) error, keys ...string) (err error) { + var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings + offset := 0 + + // Descend to the desired key, if requested + if len(keys) > 0 { + if off := searchKeys(data, keys...); off == -1 { + return KeyPathNotFoundError + } else { + offset = off + } + } + + // Validate and skip past opening brace + if off := nextToken(data[offset:]); off == -1 { + return MalformedObjectError + } else if offset += off; data[offset] != '{' { + return MalformedObjectError + } else { + offset++ + } + + // Skip to the first token inside the object, or stop if we find the ending brace + if off := nextToken(data[offset:]); off == -1 { + return MalformedJsonError + } else if offset += off; data[offset] == '}' { + return nil + } + + // Loop pre-condition: data[offset] points to what should be either the next entry's key, or the closing brace (if it's anything else, the JSON is malformed) + for offset < len(data) { + // Step 1: find the next key + var key []byte + + // Check what the the next token is: start of string, end of object, or something else (error) + switch data[offset] { + case '"': + offset++ // accept as string and skip opening quote + case '}': + return nil // we found the end of the object; stop and return success + default: + return MalformedObjectError + } + + // Find the end of the key string + var keyEscaped bool + if off, esc := stringEnd(data[offset:]); off == -1 { + return MalformedJsonError + } else { + key, keyEscaped = data[offset:offset+off-1], esc + offset += off + } + + // Unescape the string if needed + if keyEscaped { + if keyUnescaped, err := Unescape(key, stackbuf[:]); err != nil { + return MalformedStringEscapeError + } else { + key = keyUnescaped + } + } + + // Step 2: skip the colon + if off := nextToken(data[offset:]); off == -1 { + return MalformedJsonError + } else if offset += off; data[offset] != ':' { + return MalformedJsonError + } else { + offset++ + } + + // Step 3: find the associated value, then invoke the callback + if value, valueType, off, err := Get(data[offset:]); err != nil { + return err + } else if err := callback(key, value, valueType, offset+off); err != nil { // Invoke the callback here! + return err + } else { + offset += off + } + + // Step 4: skip over the next comma to the following token, or stop if we hit the ending brace + if off := nextToken(data[offset:]); off == -1 { + return MalformedArrayError + } else { + offset += off + switch data[offset] { + case '}': + return nil // Stop if we hit the close brace + case ',': + offset++ // Ignore the comma + default: + return MalformedObjectError + } + } + + // Skip to the next token after the comma + if off := nextToken(data[offset:]); off == -1 { + return MalformedArrayError + } else { + offset += off + } + } + + return MalformedObjectError // we shouldn't get here; it's expected that we will return via finding the ending brace +} + +// GetUnsafeString returns the value retrieved by `Get`, use creates string without memory allocation by mapping string to slice memory. It does not handle escape symbols. +func GetUnsafeString(data []byte, keys ...string) (val string, err error) { + v, _, _, e := Get(data, keys...) + + if e != nil { + return "", e + } + + return bytesToString(&v), nil +} + +// GetString returns the value retrieved by `Get`, cast to a string if possible, trying to properly handle escape and utf8 symbols +// If key data type do not match, it will return an error. +func GetString(data []byte, keys ...string) (val string, err error) { + v, t, _, e := Get(data, keys...) + + if e != nil { + return "", e + } + + if t != String { + return "", fmt.Errorf("Value is not a string: %s", string(v)) + } + + // If no escapes return raw conten + if bytes.IndexByte(v, '\\') == -1 { + return string(v), nil + } + + return ParseString(v) +} + +// GetFloat returns the value retrieved by `Get`, cast to a float64 if possible. +// The offset is the same as in `Get`. +// If key data type do not match, it will return an error. +func GetFloat(data []byte, keys ...string) (val float64, err error) { + v, t, _, e := Get(data, keys...) + + if e != nil { + return 0, e + } + + if t != Number { + return 0, fmt.Errorf("Value is not a number: %s", string(v)) + } + + return ParseFloat(v) +} + +// GetInt returns the value retrieved by `Get`, cast to a int64 if possible. +// If key data type do not match, it will return an error. +func GetInt(data []byte, keys ...string) (val int64, err error) { + v, t, _, e := Get(data, keys...) + + if e != nil { + return 0, e + } + + if t != Number { + return 0, fmt.Errorf("Value is not a number: %s", string(v)) + } + + return ParseInt(v) +} + +// GetBoolean returns the value retrieved by `Get`, cast to a bool if possible. +// The offset is the same as in `Get`. +// If key data type do not match, it will return error. +func GetBoolean(data []byte, keys ...string) (val bool, err error) { + v, t, _, e := Get(data, keys...) + + if e != nil { + return false, e + } + + if t != Boolean { + return false, fmt.Errorf("Value is not a boolean: %s", string(v)) + } + + return ParseBoolean(v) +} + +// ParseBoolean parses a Boolean ValueType into a Go bool (not particularly useful, but here for completeness) +func ParseBoolean(b []byte) (bool, error) { + switch { + case bytes.Equal(b, trueLiteral): + return true, nil + case bytes.Equal(b, falseLiteral): + return false, nil + default: + return false, MalformedValueError + } +} + +// ParseString parses a String ValueType into a Go string (the main parsing work is unescaping the JSON string) +func ParseString(b []byte) (string, error) { + var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings + if bU, err := Unescape(b, stackbuf[:]); err != nil { + return "", MalformedValueError + } else { + return string(bU), nil + } +} + +// ParseNumber parses a Number ValueType into a Go float64 +func ParseFloat(b []byte) (float64, error) { + if v, err := parseFloat(&b); err != nil { + return 0, MalformedValueError + } else { + return v, nil + } +} + +// ParseInt parses a Number ValueType into a Go int64 +func ParseInt(b []byte) (int64, error) { + if v, ok := parseInt(b); !ok { + return 0, MalformedValueError + } else { + return v, nil + } +} diff --git a/vendor/github.com/spaolacci/murmur3/.gitignore b/vendor/github.com/spaolacci/murmur3/.gitignore deleted file mode 100644 index 00268614f04..00000000000 --- a/vendor/github.com/spaolacci/murmur3/.gitignore +++ /dev/null @@ -1,22 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe diff --git a/vendor/github.com/spaolacci/murmur3/murmur_test.go b/vendor/github.com/spaolacci/murmur3/murmur_test.go deleted file mode 100644 index 32789bdb4fa..00000000000 --- a/vendor/github.com/spaolacci/murmur3/murmur_test.go +++ /dev/null @@ -1,229 +0,0 @@ -package murmur3 - -import ( - "hash" - "testing" -) - -var data = []struct { - h32 uint32 - h64_1 uint64 - h64_2 uint64 - s string -}{ - {0x00000000, 0x0000000000000000, 0x0000000000000000, ""}, - {0x248bfa47, 0xcbd8a7b341bd9b02, 0x5b1e906a48ae1d19, "hello"}, - {0x149bbb7f, 0x342fac623a5ebc8e, 0x4cdcbc079642414d, "hello, world"}, - {0xe31e8a70, 0xb89e5988b737affc, 0x664fc2950231b2cb, "19 Jan 2038 at 3:14:07 AM"}, - {0xd5c48bfc, 0xcd99481f9ee902c9, 0x695da1a38987b6e7, "The quick brown fox jumps over the lazy dog."}, -} - -func TestRef(t *testing.T) { - for _, elem := range data { - - var h32 hash.Hash32 = New32() - h32.Write([]byte(elem.s)) - if v := h32.Sum32(); v != elem.h32 { - t.Errorf("'%s': 0x%x (want 0x%x)", elem.s, v, elem.h32) - } - - if v := Sum32([]byte(elem.s)); v != elem.h32 { - t.Errorf("'%s': 0x%x (want 0x%x)", elem.s, v, elem.h32) - } - - var h64 hash.Hash64 = New64() - h64.Write([]byte(elem.s)) - if v := h64.Sum64(); v != elem.h64_1 { - t.Errorf("'%s': 0x%x (want 0x%x)", elem.s, v, elem.h64_1) - } - - if v := Sum64([]byte(elem.s)); v != elem.h64_1 { - t.Errorf("'%s': 0x%x (want 0x%x)", elem.s, v, elem.h64_1) - } - - var h128 Hash128 = New128() - h128.Write([]byte(elem.s)) - if v1, v2 := h128.Sum128(); v1 != elem.h64_1 || v2 != elem.h64_2 { - t.Errorf("'%s': 0x%x-0x%x (want 0x%x-0x%x)", elem.s, v1, v2, elem.h64_1, elem.h64_2) - } - - if v1, v2 := Sum128([]byte(elem.s)); v1 != elem.h64_1 || v2 != elem.h64_2 { - t.Errorf("'%s': 0x%x-0x%x (want 0x%x-0x%x)", elem.s, v1, v2, elem.h64_1, elem.h64_2) - } - } -} - -func TestIncremental(t *testing.T) { - for _, elem := range data { - h32 := New32() - h128 := New128() - for i, j, k := 0, 0, len(elem.s); i < k; i = j { - j = 2*i + 3 - if j > k { - j = k - } - s := elem.s[i:j] - print(s + "|") - h32.Write([]byte(s)) - h128.Write([]byte(s)) - } - println() - if v := h32.Sum32(); v != elem.h32 { - t.Errorf("'%s': 0x%x (want 0x%x)", elem.s, v, elem.h32) - } - if v1, v2 := h128.Sum128(); v1 != elem.h64_1 || v2 != elem.h64_2 { - t.Errorf("'%s': 0x%x-0x%x (want 0x%x-0x%x)", elem.s, v1, v2, elem.h64_1, elem.h64_2) - } - } -} - -//--- - -func bench32(b *testing.B, length int) { - buf := make([]byte, length) - b.SetBytes(int64(length)) - b.ResetTimer() - for i := 0; i < b.N; i++ { - Sum32(buf) - } -} - -func Benchmark32_1(b *testing.B) { - bench32(b, 1) -} -func Benchmark32_2(b *testing.B) { - bench32(b, 2) -} -func Benchmark32_4(b *testing.B) { - bench32(b, 4) -} -func Benchmark32_8(b *testing.B) { - bench32(b, 8) -} -func Benchmark32_16(b *testing.B) { - bench32(b, 16) -} -func Benchmark32_32(b *testing.B) { - bench32(b, 32) -} -func Benchmark32_64(b *testing.B) { - bench32(b, 64) -} -func Benchmark32_128(b *testing.B) { - bench32(b, 128) -} -func Benchmark32_256(b *testing.B) { - bench32(b, 256) -} -func Benchmark32_512(b *testing.B) { - bench32(b, 512) -} -func Benchmark32_1024(b *testing.B) { - bench32(b, 1024) -} -func Benchmark32_2048(b *testing.B) { - bench32(b, 2048) -} -func Benchmark32_4096(b *testing.B) { - bench32(b, 4096) -} -func Benchmark32_8192(b *testing.B) { - bench32(b, 8192) -} - -//--- - -func benchPartial32(b *testing.B, length int) { - buf := make([]byte, length) - b.SetBytes(int64(length)) - - start := (32 / 8) / 2 - chunks := 7 - k := length / chunks - tail := (length - start) % k - - b.ResetTimer() - for i := 0; i < b.N; i++ { - hasher := New32() - hasher.Write(buf[0:start]) - - for j := start; j+k <= length; j += k { - hasher.Write(buf[j : j+k]) - } - - hasher.Write(buf[length-tail:]) - hasher.Sum32() - } -} - -func BenchmarkPartial32_8(b *testing.B) { - benchPartial32(b, 8) -} -func BenchmarkPartial32_16(b *testing.B) { - benchPartial32(b, 16) -} -func BenchmarkPartial32_32(b *testing.B) { - benchPartial32(b, 32) -} -func BenchmarkPartial32_64(b *testing.B) { - benchPartial32(b, 64) -} -func BenchmarkPartial32_128(b *testing.B) { - benchPartial32(b, 128) -} - -//--- - -func bench128(b *testing.B, length int) { - buf := make([]byte, length) - b.SetBytes(int64(length)) - b.ResetTimer() - for i := 0; i < b.N; i++ { - Sum128(buf) - } -} - -func Benchmark128_1(b *testing.B) { - bench128(b, 1) -} -func Benchmark128_2(b *testing.B) { - bench128(b, 2) -} -func Benchmark128_4(b *testing.B) { - bench128(b, 4) -} -func Benchmark128_8(b *testing.B) { - bench128(b, 8) -} -func Benchmark128_16(b *testing.B) { - bench128(b, 16) -} -func Benchmark128_32(b *testing.B) { - bench128(b, 32) -} -func Benchmark128_64(b *testing.B) { - bench128(b, 64) -} -func Benchmark128_128(b *testing.B) { - bench128(b, 128) -} -func Benchmark128_256(b *testing.B) { - bench128(b, 256) -} -func Benchmark128_512(b *testing.B) { - bench128(b, 512) -} -func Benchmark128_1024(b *testing.B) { - bench128(b, 1024) -} -func Benchmark128_2048(b *testing.B) { - bench128(b, 2048) -} -func Benchmark128_4096(b *testing.B) { - bench128(b, 4096) -} -func Benchmark128_8192(b *testing.B) { - bench128(b, 8192) -} - -//--- diff --git a/vendor/vendor.json b/vendor/vendor.json index 6cca4a0746a..292fe50d9ee 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -78,6 +78,12 @@ "revision": "55ff0f4b9b3de929d99bad783723811ff7add9b6", "revisionTime": "2016-12-01T17:12:39Z" }, + { + "checksumSHA1": "FfOYh5n7gVJDqDcBbxySdCzzs1Y=", + "path": "github.com/TykTechnologies/murmur3", + "revision": "1915e687e465f5085f46a3a1339fb2d004e29dfb", + "revisionTime": "2018-06-02T12:20:59Z" + }, { "checksumSHA1": "v64AG0uGiKryWV/kqq5360zw1ts=", "path": "github.com/TykTechnologies/openid2go/openid", @@ -114,6 +120,12 @@ "revision": "5f729f2fb50a301153cae84ff5c58981d51c095a", "revisionTime": "2017-01-02T09:48:09Z" }, + { + "checksumSHA1": "yqETaGs43hbxuaBGUP7YT/tiRDg=", + "path": "github.com/buger/jsonparser", + "revision": "bb14bb6c38f6cf1706ef55278891d184b6a51b0e", + "revisionTime": "2017-06-03T06:26:59Z" + }, { "checksumSHA1": "m6QMy+u/IX6knzP28DqMn4sdLSw=", "path": "github.com/cenk/backoff", diff --git a/version.go b/version.go index ccdff9ba6d1..4e79fb1dbb8 100644 --- a/version.go +++ b/version.go @@ -1,3 +1,3 @@ package main -const VERSION = "v2.6.0" +const VERSION = "v2.7.0"