diff --git a/analytics.go b/analytics.go index 93fe092216e..623cd97cc3b 100644 --- a/analytics.go +++ b/analytics.go @@ -108,26 +108,18 @@ type NormaliseURLPatterns struct { Custom []*regexp.Regexp } -func initNormalisationPatterns() NormaliseURLPatterns { - thesePatterns := NormaliseURLPatterns{} +func initNormalisationPatterns() (pats NormaliseURLPatterns) { + pats.UUIDs = regexp.MustCompile(`[0-9a-fA-F]{8}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{12}`) + pats.IDs = regexp.MustCompile(`\/(\d+)`) - uuidPat := regexp.MustCompile(`[0-9a-fA-F]{8}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{12}`) - numPat := regexp.MustCompile(`\/(\d+)`) - - custPats := []*regexp.Regexp{} for _, pattern := range config.AnalyticsConfig.NormaliseUrls.Custom { if patRe, err := regexp.Compile(pattern); err != nil { log.Error("failed to compile custom pattern: ", err) } else { - custPats = append(custPats, patRe) + pats.Custom = append(pats.Custom, patRe) } } - - thesePatterns.UUIDs = uuidPat - thesePatterns.IDs = numPat - thesePatterns.Custom = custPats - - return thesePatterns + return } func (a *AnalyticsRecord) NormalisePath() { diff --git a/api.go b/api.go index 924f67bb6f2..dc89f0df42f 100644 --- a/api.go +++ b/api.go @@ -635,16 +635,13 @@ func handleAddOrUpdateApi(apiID string, r *http.Request) ([]byte, int) { return createError("Request malformed"), 400 } - if apiID != "" { - if newDef.APIID != apiID { - log.Error("PUT operation on different APIIDs") - return createError("Request APIID does not match that in Definition! For Updtae operations these must match."), 400 - } + if apiID != "" && newDef.APIID != apiID { + log.Error("PUT operation on different APIIDs") + return createError("Request APIID does not match that in Definition! For Updtae operations these must match."), 400 } // Create a filename - defFilename := newDef.APIID + ".json" - defFilePath := filepath.Join(config.AppPath, defFilename) + defFilePath := filepath.Join(config.AppPath, newDef.APIID+".json") // If it exists, delete it if _, err := os.Stat(defFilePath); err == nil { @@ -686,8 +683,7 @@ func handleAddOrUpdateApi(apiID string, r *http.Request) ([]byte, int) { func handleDeleteAPI(apiID string) ([]byte, int) { // Generate a filename - defFilename := apiID + ".json" - defFilePath := filepath.Join(config.AppPath, defFilename) + defFilePath := filepath.Join(config.AppPath, apiID+".json") // If it exists, delete it if _, err := os.Stat(defFilePath); err != nil { @@ -1172,7 +1168,6 @@ func createKeyHandler(w http.ResponseWriter, r *http.Request) { doJSONWrite(w, 405, createError("Method not supported")) return } - responseObj := APIModifyKeySuccess{} var newSession SessionState if err := json.NewDecoder(r.Body).Decode(&newSession); err != nil { @@ -1270,10 +1265,11 @@ func createKeyHandler(w http.ResponseWriter, r *http.Request) { } - responseObj.Action = "create" - responseObj.Key = newKey - responseObj.Status = "ok" - + responseObj := APIModifyKeySuccess{ + Action: "create", + Key: newKey, + Status: "ok", + } responseMessage, err := json.Marshal(&responseObj) if err != nil { log.WithFields(logrus.Fields{ diff --git a/api_definition.go b/api_definition.go index b43ce33a538..b0989429ee3 100644 --- a/api_definition.go +++ b/api_definition.go @@ -257,7 +257,6 @@ func (a *APIDefinitionLoader) LoadDefinitionsFromDashboardService(endpoint, secr } list := NodeResponseOK{} - if err := json.Unmarshal(retBody, &list); err != nil { log.Error("Failed to decode body: ", err, "Response was: ", string(retBody)) log.Info("--> Retrying in 5s") @@ -448,7 +447,6 @@ func (a *APIDefinitionLoader) compilePathSpec(paths []string, specType URLStatus } func (a *APIDefinitionLoader) compileExtendedPathSpec(paths []apidef.EndPointMeta, specType URLStatus) []URLSpec { - // transform an extended configuration URL into an array of URLSpecs // This way we can iterate the whole array once, on match we break with status urlSpec := []URLSpec{} @@ -466,7 +464,6 @@ func (a *APIDefinitionLoader) compileExtendedPathSpec(paths []apidef.EndPointMet } func (a *APIDefinitionLoader) compileCachedPathSpec(paths []string) []URLSpec { - // transform an extended configuration URL into an array of URLSpecs // This way we can iterate the whole array once, on match we break with status urlSpec := []URLSpec{} @@ -496,7 +493,6 @@ func (a *APIDefinitionLoader) loadBlobTemplate(blob string) (*textTemplate.Templ } func (a *APIDefinitionLoader) compileTransformPathSpec(paths []apidef.TemplateMeta, stat URLStatus) []URLSpec { - // transform an extended configuration URL into an array of URLSpecs // This way we can iterate the whole array once, on match we break with status urlSpec := []URLSpec{} @@ -544,7 +540,6 @@ func (a *APIDefinitionLoader) compileTransformPathSpec(paths []apidef.TemplateMe } func (a *APIDefinitionLoader) compileInjectedHeaderSpec(paths []apidef.HeaderInjectionMeta, stat URLStatus) []URLSpec { - // transform an extended configuration URL into an array of URLSpecs // This way we can iterate the whole array once, on match we break with status urlSpec := []URLSpec{} @@ -566,7 +561,6 @@ func (a *APIDefinitionLoader) compileInjectedHeaderSpec(paths []apidef.HeaderInj } func (a *APIDefinitionLoader) compileMethodTransformSpec(paths []apidef.MethodTransformMeta, stat URLStatus) []URLSpec { - // transform an extended configuration URL into an array of URLSpecs // This way we can iterate the whole array once, on match we break with status urlSpec := []URLSpec{} @@ -583,7 +577,6 @@ func (a *APIDefinitionLoader) compileMethodTransformSpec(paths []apidef.MethodTr } func (a *APIDefinitionLoader) compileTimeoutPathSpec(paths []apidef.HardTimeoutMeta, stat URLStatus) []URLSpec { - // transform an extended configuration URL into an array of URLSpecs // This way we can iterate the whole array once, on match we break with status urlSpec := []URLSpec{} @@ -601,7 +594,6 @@ func (a *APIDefinitionLoader) compileTimeoutPathSpec(paths []apidef.HardTimeoutM } func (a *APIDefinitionLoader) compileRequestSizePathSpec(paths []apidef.RequestSizeMeta, stat URLStatus) []URLSpec { - // transform an extended configuration URL into an array of URLSpecs // This way we can iterate the whole array once, on match we break with status urlSpec := []URLSpec{} @@ -619,7 +611,6 @@ func (a *APIDefinitionLoader) compileRequestSizePathSpec(paths []apidef.RequestS } func (a *APIDefinitionLoader) compileCircuitBreakerPathSpec(paths []apidef.CircuitBreakerMeta, stat URLStatus, apiSpec *APISpec) []URLSpec { - // transform an extended configuration URL into an array of URLSpecs // This way we can iterate the whole array once, on match we break with status urlSpec := []URLSpec{} @@ -686,7 +677,6 @@ func (a *APIDefinitionLoader) compileCircuitBreakerPathSpec(paths []apidef.Circu } func (a *APIDefinitionLoader) compileURLRewritesPathSpec(paths []apidef.URLRewriteMeta, stat URLStatus) []URLSpec { - // transform an extended configuration URL into an array of URLSpecs // This way we can iterate the whole array once, on match we break with status urlSpec := []URLSpec{} @@ -704,7 +694,6 @@ func (a *APIDefinitionLoader) compileURLRewritesPathSpec(paths []apidef.URLRewri } func (a *APIDefinitionLoader) compileVirtualPathspathSpec(paths []apidef.VirtualMeta, stat URLStatus, apiSpec *APISpec) []URLSpec { - if !config.EnableJSVM { return nil } @@ -1101,8 +1090,7 @@ func (a *APISpec) GetVersionData(r *http.Request) (*apidef.VersionInfo, []URLSpe whiteListStatus, wlOk := a.WhiteListEnabled[versionKey] if !rxOk { - log.Error("no RX Paths found for version") - log.Error(versionKey) + log.Error("no RX Paths found for version ", versionKey) return &version, versionRxPaths, versionWLStatus, VersionDoesNotExist } diff --git a/config.go b/config.go index 1ec390409ff..8ea87ee3f10 100644 --- a/config.go +++ b/config.go @@ -8,7 +8,7 @@ import ( "time" "github.com/kelseyhightower/envconfig" - uuid "github.com/satori/go.uuid" + "github.com/satori/go.uuid" "github.com/TykTechnologies/tyk/apidef" ) diff --git a/coprocess.go b/coprocess.go index e43ec70ba68..086b231447f 100644 --- a/coprocess.go +++ b/coprocess.go @@ -190,7 +190,7 @@ func (m *CoProcessMiddleware) IsEnabledForSpec() bool { supportedDrivers := []apidef.MiddlewareDriver{apidef.PythonDriver, apidef.LuaDriver, apidef.GrpcDriver} for _, driver := range supportedDrivers { - if m.TykMiddleware.Spec.CustomMiddleware.Driver == driver && CoProcessName == string(driver) { + if m.TykMiddleware.Spec.CustomMiddleware.Driver == driver && CoProcessName == driver { usesCoProcessMiddleware = true break } diff --git a/coprocess_grpc.go b/coprocess_grpc.go index 1ea0bafb58e..ee648c610a8 100644 --- a/coprocess_grpc.go +++ b/coprocess_grpc.go @@ -18,7 +18,7 @@ import ( ) // CoProcessName specifies the driver name. -const CoProcessName = "grpc" +const CoProcessName = apidef.GrpcDriver // MessageType sets the default message type. var MessageType = coprocess.ProtobufMessage diff --git a/coprocess_lua.go b/coprocess_lua.go index 9e2134ffae7..1752bb19c38 100644 --- a/coprocess_lua.go +++ b/coprocess_lua.go @@ -83,7 +83,7 @@ import ( ) // CoProcessName specifies the driver name. -const CoProcessName = "lua" +const CoProcessName = apidef.LuaDriver const ( // ModuleBasePath points to the Tyk modules path. diff --git a/coprocess_python.go b/coprocess_python.go index 66a2a16df92..a70551b8b72 100644 --- a/coprocess_python.go +++ b/coprocess_python.go @@ -182,7 +182,7 @@ import ( ) // CoProcessName declares the driver name. -const CoProcessName = "python" +const CoProcessName = apidef.PythonDriver // MessageType sets the default message type. var MessageType = coprocess.ProtobufMessage diff --git a/coprocess_test.go b/coprocess_test.go index f51cfc37b89..2c17615524a 100644 --- a/coprocess_test.go +++ b/coprocess_test.go @@ -24,7 +24,7 @@ import ( const baseMiddlewarePath = "middleware/python" var ( - CoProcessName = "test" + CoProcessName = apidef.MiddlewareDriver("test") MessageType = coprocess.ProtobufMessage testDispatcher, _ = NewCoProcessDispatcher() ) diff --git a/dashboard_register.go b/dashboard_register.go index 4a985232e85..cd194197dd1 100644 --- a/dashboard_register.go +++ b/dashboard_register.go @@ -94,8 +94,7 @@ func (h *HTTPDashboardHandler) Register() error { retBody, err := ioutil.ReadAll(response.Body) if response.StatusCode != 200 { - log.Error("Failed to register node, retrying in 5s") - log.Error(" --> Response was: ", string(retBody)) + log.Error("Failed to register node, retrying in 5s; Response was: ", string(retBody)) time.Sleep(time.Second * 5) return h.Register() } diff --git a/dq.go b/dq.go index bd9c134607b..06e7d0bd5c1 100644 --- a/dq.go +++ b/dq.go @@ -27,7 +27,8 @@ func getDQTopic() string { if config.DBAppConfOptions.NodeIsSegmented { if len(config.DBAppConfOptions.Tags) > 0 { tags := strings.Join(config.DBAppConfOptions.Tags, ".") - topic = topic + "." + tags + topic += "." + tags + } } @@ -94,13 +95,7 @@ func dqFlusher(d map[string]*dq.Quota) error { continue } - if time.Now().After(expT) { - QuotaHandler.TagDelete(k) - continue - } - - if s.IsExpired() { - // Remove expired data too + if !f || time.Now().After(expT) || s.IsExpired() { QuotaHandler.TagDelete(k) continue } diff --git a/gateway_test.go b/gateway_test.go index 733373c982c..6ae0654c73f 100644 --- a/gateway_test.go +++ b/gateway_test.go @@ -758,8 +758,8 @@ func TestDistributedQuotaSingleNode(t *testing.T) { defer spec.SessionManager.ResetQuota(keyId, session) recorder := httptest.NewRecorder() - param := make(url.Values) - req, err := http.NewRequest("GET", param.Encode(), nil) + + req, err := http.NewRequest("GET", "", nil) if err != nil { t.Fatal(err) } @@ -768,6 +768,11 @@ func TestDistributedQuotaSingleNode(t *testing.T) { config.DQSetMaster = true config.UseDistributedQuotaCounter = true config.DistributedQuotaFlushIntervalInMS = 100 + defer func() { + config.DQSetMaster = false + config.UseDistributedQuotaCounter = false + config.DistributedQuotaFlushIntervalInMS = 0 + }() startDQ(decideLeaderMechanism()) diff --git a/host_checker.go b/host_checker.go index 310badd5df9..7dcc4a2719b 100644 --- a/host_checker.go +++ b/host_checker.go @@ -20,7 +20,6 @@ var HostCheckerClient = &http.Client{Timeout: 500 * time.Millisecond} type HostData struct { CheckURL string - ID string Method string Headers map[string]string Body string @@ -95,26 +94,26 @@ func (h *HostUptimeChecker) HostReporter() { select { case okHost := <-h.okChan: // Clear host from unhealthylist if it exists - if h.unHealthyList[okHost.ID] { + if h.unHealthyList[okHost.CheckURL] { h.upCallback(okHost) - delete(h.unHealthyList, okHost.ID) + delete(h.unHealthyList, okHost.CheckURL) } go h.pingCallback(okHost) case failedHost := <-h.errorChan: newVal := 1 - if count, found := h.sampleCache.Get(failedHost.ID); found { + if count, found := h.sampleCache.Get(failedHost.CheckURL); found { newVal = count.(int) + 1 } - h.sampleCache.Set(failedHost.ID, newVal, cache.DefaultExpiration) + h.sampleCache.Set(failedHost.CheckURL, newVal, cache.DefaultExpiration) if newVal >= h.sampleTriggerLimit { log.Debug("[HOST CHECKER] [HOST WARNING]: ", failedHost.CheckURL) // Reset the count - h.sampleCache.Set(failedHost.ID, 1, cache.DefaultExpiration) + h.sampleCache.Set(failedHost.CheckURL, 1, cache.DefaultExpiration) // track it - h.unHealthyList[failedHost.ID] = true + h.unHealthyList[failedHost.CheckURL] = true // Call the custom callback hook go h.failureCallback(failedHost) } @@ -128,7 +127,7 @@ func (h *HostUptimeChecker) HostReporter() { } func (h *HostUptimeChecker) CheckHost(toCheck HostData) { - log.Debug("[HOST CHECKER] Checking: ", toCheck.CheckURL, toCheck.ID) + log.Debug("[HOST CHECKER] Checking: ", toCheck.CheckURL) t1 := time.Now() diff --git a/host_checker_manager.go b/host_checker_manager.go index 55afb7ce0b2..86b5d2999e3 100644 --- a/host_checker_manager.go +++ b/host_checker_manager.go @@ -311,7 +311,6 @@ func (hc *HostCheckerManager) PrepareTrackingHost(checkObject apidef.HostCheckOb hostData = HostData{ CheckURL: checkObject.CheckURL, - ID: checkObject.CheckURL, MetaData: make(map[string]string), Method: checkObject.Method, Headers: checkObject.Headers, diff --git a/main.go b/main.go index 3ef8a31b26e..a6d05396557 100644 --- a/main.go +++ b/main.go @@ -29,7 +29,7 @@ import ( "github.com/gorilla/mux" "github.com/justinas/alice" "github.com/lonelycode/gorpc" - osin "github.com/lonelycode/osin" + "github.com/lonelycode/osin" "github.com/rs/cors" "rsc.io/letsencrypt" diff --git a/middleware_virtual_endpoint.go b/middleware_virtual_endpoint.go index 8e27eda08bc..61e66ce197a 100644 --- a/middleware_virtual_endpoint.go +++ b/middleware_virtual_endpoint.go @@ -192,8 +192,8 @@ func (d *VirtualEndpoint) ServeHTTPForCache(w http.ResponseWriter, r *http.Reque // Decode the return object newResponseData := VMResponseObject{} if err := json.Unmarshal([]byte(returnDataStr), &newResponseData); err != nil { - log.Error("Failed to decode virtual endpoint response data on return from VM: ", err) - log.Error("--> Returned: ", returnDataStr) + log.Error("Failed to decode virtual endpoint response data on return from VM: ", err, + "; Returned: ", returnDataStr) return nil } diff --git a/oauth_manager.go b/oauth_manager.go index 172b1a39a94..99f9699409b 100644 --- a/oauth_manager.go +++ b/oauth_manager.go @@ -157,8 +157,7 @@ func (o *OAuthHandlers) HandleAuthorizePassthrough(w http.ResponseWriter, r *htt // Extract client data and check resp := o.Manager.HandleAuthorisation(r, false, "") if resp.IsError { - log.Error("There was an error with the request") - log.Error(resp) + log.Error("There was an error with the request: ", resp) // Something went wrong, write out the error details and kill the response w.WriteHeader(resp.ErrorStatusCode) responseMessage = createError(resp.StatusText) @@ -552,7 +551,6 @@ func (r *RedisOsinStorageInterface) GetClients(filter string, ignorePrefix bool) } theseClients := []osin.Client{} - for _, clientJSON := range clientJSON { client := new(OAuthClient) if err := json.Unmarshal([]byte(clientJSON), &client); err != nil { @@ -770,10 +768,9 @@ func (r *RedisOsinStorageInterface) LoadRefresh(token string) (*osin.AccessData, // new interface means having to make this nested... ick. accessData := osin.AccessData{} accessData.Client = new(OAuthClient) - if err := json.Unmarshal([]byte(accessJSON), &accessData); err != nil { - log.Error("Couldn't unmarshal OAuth auth data object (LoadRefresh): ", err) - log.Error("Decoding:", accessJSON) + log.Error("Couldn't unmarshal OAuth auth data object (LoadRefresh): ", err, + "; Decoding: ", accessJSON) return nil, err } @@ -837,10 +834,9 @@ func (r *RedisOsinStorageInterface) GetUser(username string) (*SessionState, err // new interface means having to make this nested... ick. session := SessionState{} - if err := json.Unmarshal([]byte(accessJSON), &session); err != nil { - log.Error("Couldn't unmarshal OAuth auth data object (LoadRefresh): ", err) - log.Error("Decoding:", accessJSON) + log.Error("Couldn't unmarshal OAuth auth data object (LoadRefresh): ", err, + "; Decoding: ", accessJSON) return nil, err } diff --git a/plugins.go b/plugins.go index 5d9f2779724..5fdb1466762 100644 --- a/plugins.go +++ b/plugins.go @@ -149,7 +149,6 @@ func (d *DynamicMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Reques // Decode the return object newRequestData := VMReturnObject{} - if err := json.Unmarshal([]byte(returnDataStr), &newRequestData); err != nil { log.WithFields(logrus.Fields{ "prefix": "jsvm", @@ -290,11 +289,11 @@ func (j *JSVM) LoadTykJSApi() { j.VM.Set("TykMakeHttpRequest", func(call otto.FunctionCall) otto.Value { jsonHRO := call.Argument(0).String() - hro := TykJSHttpRequest{} if jsonHRO == "undefined" { // Nope, return nothing return otto.Value{} } + hro := TykJSHttpRequest{} if err := json.Unmarshal([]byte(jsonHRO), &hro); err != nil { log.WithFields(logrus.Fields{ "prefix": "jsvm", diff --git a/res_handler_header_injector.go b/res_handler_header_injector.go index 1d88d3a75a5..0245245407c 100644 --- a/res_handler_header_injector.go +++ b/res_handler_header_injector.go @@ -19,15 +19,11 @@ type HeaderInjector struct { } func (h HeaderInjector) New(c interface{}, spec *APISpec) (TykResponseHandler, error) { - handler := HeaderInjector{} - moduleConfig := HeaderInjectorOptions{} - - if err := mapstructure.Decode(c, &moduleConfig); err != nil { + handler := HeaderInjector{Spec: spec} + if err := mapstructure.Decode(c, &handler.config); err != nil { log.Error(err) return nil, err } - handler.config = moduleConfig - handler.Spec = spec return handler, nil } @@ -36,7 +32,6 @@ func (h HeaderInjector) HandleResponse(rw http.ResponseWriter, res *http.Respons _, versionPaths, _, _ := h.Spec.GetVersionData(req) found, meta := h.Spec.CheckSpecMatchesStatus(req.URL.Path, req.Method, versionPaths, HeaderInjectedResponse) - if found { hmeta := meta.(*apidef.HeaderInjectionMeta) for _, dKey := range hmeta.DeleteHeaders { @@ -51,7 +46,6 @@ func (h HeaderInjector) HandleResponse(rw http.ResponseWriter, res *http.Respons for _, n := range h.config.RemoveHeaders { res.Header.Del(n) } - for h, v := range h.config.AddHeaders { res.Header.Add(h, v) } diff --git a/tyk_reverse_proxy_clone.go b/tyk_reverse_proxy_clone.go index a4c9fe76b89..53437c187f5 100644 --- a/tyk_reverse_proxy_clone.go +++ b/tyk_reverse_proxy_clone.go @@ -417,10 +417,10 @@ func GetTransport(timeOut int, rw http.ResponseWriter, req *http.Request, p *Rev // Use the default unless we've modified the timout if timeOut > 0 { log.Debug("Setting timeout for outbound request to: ", timeOut) - transport.Dial = (&net.Dialer{ + transport.DialContext = (&net.Dialer{ Timeout: time.Duration(timeOut) * time.Second, KeepAlive: 30 * time.Second, - }).Dial + }).DialContext transport.SetTimeout(timeOut) }