Merge branch 'dq-squash' into tcf-pubsub

analytics.go

@@ -108,26 +108,18 @@ type NormaliseURLPatterns struct {
 	Custom []*regexp.Regexp
 }
 
-func initNormalisationPatterns() NormaliseURLPatterns {
-	thesePatterns := NormaliseURLPatterns{}
+func initNormalisationPatterns() (pats NormaliseURLPatterns) {
+	pats.UUIDs = regexp.MustCompile(`[0-9a-fA-F]{8}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{12}`)
+	pats.IDs = regexp.MustCompile(`\/(\d+)`)
 
-	uuidPat := regexp.MustCompile(`[0-9a-fA-F]{8}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{4}(-)?[0-9a-fA-F]{12}`)
-	numPat := regexp.MustCompile(`\/(\d+)`)
-
-	custPats := []*regexp.Regexp{}
 	for _, pattern := range config.AnalyticsConfig.NormaliseUrls.Custom {
 		if patRe, err := regexp.Compile(pattern); err != nil {
 			log.Error("failed to compile custom pattern: ", err)
 		} else {
-			custPats = append(custPats, patRe)
+			pats.Custom = append(pats.Custom, patRe)
 		}
 	}
-
-	thesePatterns.UUIDs = uuidPat
-	thesePatterns.IDs = numPat
-	thesePatterns.Custom = custPats
-
-	return thesePatterns
+	return
 }
 
 func (a *AnalyticsRecord) NormalisePath() {

api.go

@@ -635,16 +635,13 @@ func handleAddOrUpdateApi(apiID string, r *http.Request) ([]byte, int) {
 		return createError("Request malformed"), 400
 	}
 
-	if apiID != "" {
-		if newDef.APIID != apiID {
-			log.Error("PUT operation on different APIIDs")
-			return createError("Request APIID does not match that in Definition! For Updtae operations these must match."), 400
-		}
+	if apiID != "" && newDef.APIID != apiID {
+		log.Error("PUT operation on different APIIDs")
+		return createError("Request APIID does not match that in Definition! For Updtae operations these must match."), 400
 	}
 
 	// Create a filename
-	defFilename := newDef.APIID + ".json"
-	defFilePath := filepath.Join(config.AppPath, defFilename)
+	defFilePath := filepath.Join(config.AppPath, newDef.APIID+".json")
 
 	// If it exists, delete it
 	if _, err := os.Stat(defFilePath); err == nil {
@@ -686,8 +683,7 @@ func handleAddOrUpdateApi(apiID string, r *http.Request) ([]byte, int) {
 
 func handleDeleteAPI(apiID string) ([]byte, int) {
 	// Generate a filename
-	defFilename := apiID + ".json"
-	defFilePath := filepath.Join(config.AppPath, defFilename)
+	defFilePath := filepath.Join(config.AppPath, apiID+".json")
 
 	// If it exists, delete it
 	if _, err := os.Stat(defFilePath); err != nil {
@@ -1172,7 +1168,6 @@ func createKeyHandler(w http.ResponseWriter, r *http.Request) {
 		doJSONWrite(w, 405, createError("Method not supported"))
 		return
 	}
-	responseObj := APIModifyKeySuccess{}
 
 	var newSession SessionState
 	if err := json.NewDecoder(r.Body).Decode(&newSession); err != nil {
@@ -1270,10 +1265,11 @@ func createKeyHandler(w http.ResponseWriter, r *http.Request) {
 
 	}
 
-	responseObj.Action = "create"
-	responseObj.Key = newKey
-	responseObj.Status = "ok"
-
+	responseObj := APIModifyKeySuccess{
+		Action: "create",
+		Key:    newKey,
+		Status: "ok",
+	}
 	responseMessage, err := json.Marshal(&responseObj)
 	if err != nil {
 		log.WithFields(logrus.Fields{

api_definition.go

@@ -257,7 +257,6 @@ func (a *APIDefinitionLoader) LoadDefinitionsFromDashboardService(endpoint, secr
 	}
 
 	list := NodeResponseOK{}
-
 	if err := json.Unmarshal(retBody, &list); err != nil {
 		log.Error("Failed to decode body: ", err, "Response was: ", string(retBody))
 		log.Info("--> Retrying in 5s")
@@ -448,7 +447,6 @@ func (a *APIDefinitionLoader) compilePathSpec(paths []string, specType URLStatus
 }
 
 func (a *APIDefinitionLoader) compileExtendedPathSpec(paths []apidef.EndPointMeta, specType URLStatus) []URLSpec {
-
 	// transform an extended configuration URL into an array of URLSpecs
 	// This way we can iterate the whole array once, on match we break with status
 	urlSpec := []URLSpec{}
@@ -466,7 +464,6 @@ func (a *APIDefinitionLoader) compileExtendedPathSpec(paths []apidef.EndPointMet
 }
 
 func (a *APIDefinitionLoader) compileCachedPathSpec(paths []string) []URLSpec {
-
 	// transform an extended configuration URL into an array of URLSpecs
 	// This way we can iterate the whole array once, on match we break with status
 	urlSpec := []URLSpec{}
@@ -496,7 +493,6 @@ func (a *APIDefinitionLoader) loadBlobTemplate(blob string) (*textTemplate.Templ
 }
 
 func (a *APIDefinitionLoader) compileTransformPathSpec(paths []apidef.TemplateMeta, stat URLStatus) []URLSpec {
-
 	// transform an extended configuration URL into an array of URLSpecs
 	// This way we can iterate the whole array once, on match we break with status
 	urlSpec := []URLSpec{}
@@ -544,7 +540,6 @@ func (a *APIDefinitionLoader) compileTransformPathSpec(paths []apidef.TemplateMe
 }
 
 func (a *APIDefinitionLoader) compileInjectedHeaderSpec(paths []apidef.HeaderInjectionMeta, stat URLStatus) []URLSpec {
-
 	// transform an extended configuration URL into an array of URLSpecs
 	// This way we can iterate the whole array once, on match we break with status
 	urlSpec := []URLSpec{}
@@ -566,7 +561,6 @@ func (a *APIDefinitionLoader) compileInjectedHeaderSpec(paths []apidef.HeaderInj
 }
 
 func (a *APIDefinitionLoader) compileMethodTransformSpec(paths []apidef.MethodTransformMeta, stat URLStatus) []URLSpec {
-
 	// transform an extended configuration URL into an array of URLSpecs
 	// This way we can iterate the whole array once, on match we break with status
 	urlSpec := []URLSpec{}
@@ -583,7 +577,6 @@ func (a *APIDefinitionLoader) compileMethodTransformSpec(paths []apidef.MethodTr
 }
 
 func (a *APIDefinitionLoader) compileTimeoutPathSpec(paths []apidef.HardTimeoutMeta, stat URLStatus) []URLSpec {
-
 	// transform an extended configuration URL into an array of URLSpecs
 	// This way we can iterate the whole array once, on match we break with status
 	urlSpec := []URLSpec{}
@@ -601,7 +594,6 @@ func (a *APIDefinitionLoader) compileTimeoutPathSpec(paths []apidef.HardTimeoutM
 }
 
 func (a *APIDefinitionLoader) compileRequestSizePathSpec(paths []apidef.RequestSizeMeta, stat URLStatus) []URLSpec {
-
 	// transform an extended configuration URL into an array of URLSpecs
 	// This way we can iterate the whole array once, on match we break with status
 	urlSpec := []URLSpec{}
@@ -619,7 +611,6 @@ func (a *APIDefinitionLoader) compileRequestSizePathSpec(paths []apidef.RequestS
 }
 
 func (a *APIDefinitionLoader) compileCircuitBreakerPathSpec(paths []apidef.CircuitBreakerMeta, stat URLStatus, apiSpec *APISpec) []URLSpec {
-
 	// transform an extended configuration URL into an array of URLSpecs
 	// This way we can iterate the whole array once, on match we break with status
 	urlSpec := []URLSpec{}
@@ -686,7 +677,6 @@ func (a *APIDefinitionLoader) compileCircuitBreakerPathSpec(paths []apidef.Circu
 }
 
 func (a *APIDefinitionLoader) compileURLRewritesPathSpec(paths []apidef.URLRewriteMeta, stat URLStatus) []URLSpec {
-
 	// transform an extended configuration URL into an array of URLSpecs
 	// This way we can iterate the whole array once, on match we break with status
 	urlSpec := []URLSpec{}
@@ -704,7 +694,6 @@ func (a *APIDefinitionLoader) compileURLRewritesPathSpec(paths []apidef.URLRewri
 }
 
 func (a *APIDefinitionLoader) compileVirtualPathspathSpec(paths []apidef.VirtualMeta, stat URLStatus, apiSpec *APISpec) []URLSpec {
-
 	if !config.EnableJSVM {
 		return nil
 	}
@@ -1101,8 +1090,7 @@ func (a *APISpec) GetVersionData(r *http.Request) (*apidef.VersionInfo, []URLSpe
 	whiteListStatus, wlOk := a.WhiteListEnabled[versionKey]
 
 	if !rxOk {
-		log.Error("no RX Paths found for version")
-		log.Error(versionKey)
+		log.Error("no RX Paths found for version ", versionKey)
 		return &version, versionRxPaths, versionWLStatus, VersionDoesNotExist
 	}
 

config.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/kelseyhightower/envconfig"
-	uuid "github.com/satori/go.uuid"
+	"github.com/satori/go.uuid"
 
 	"github.com/TykTechnologies/tyk/apidef"
 )

coprocess.go

@@ -190,7 +190,7 @@ func (m *CoProcessMiddleware) IsEnabledForSpec() bool {
 	supportedDrivers := []apidef.MiddlewareDriver{apidef.PythonDriver, apidef.LuaDriver, apidef.GrpcDriver}
 
 	for _, driver := range supportedDrivers {
-		if m.TykMiddleware.Spec.CustomMiddleware.Driver == driver && CoProcessName == string(driver) {
+		if m.TykMiddleware.Spec.CustomMiddleware.Driver == driver && CoProcessName == driver {
 			usesCoProcessMiddleware = true
 			break
 		}

coprocess_grpc.go

@@ -18,7 +18,7 @@ import (
 )
 
 // CoProcessName specifies the driver name.
-const CoProcessName = "grpc"
+const CoProcessName = apidef.GrpcDriver
 
 // MessageType sets the default message type.
 var MessageType = coprocess.ProtobufMessage

coprocess_lua.go

@@ -83,7 +83,7 @@ import (
 )
 
 // CoProcessName specifies the driver name.
-const CoProcessName = "lua"
+const CoProcessName = apidef.LuaDriver
 
 const (
 	// ModuleBasePath points to the Tyk modules path.

coprocess_python.go

@@ -182,7 +182,7 @@ import (
 )
 
 // CoProcessName declares the driver name.
-const CoProcessName = "python"
+const CoProcessName = apidef.PythonDriver
 
 // MessageType sets the default message type.
 var MessageType = coprocess.ProtobufMessage

coprocess_test.go

@@ -24,7 +24,7 @@ import (
 const baseMiddlewarePath = "middleware/python"
 
 var (
-	CoProcessName     = "test"
+	CoProcessName     = apidef.MiddlewareDriver("test")
 	MessageType       = coprocess.ProtobufMessage
 	testDispatcher, _ = NewCoProcessDispatcher()
 )

dashboard_register.go

@@ -94,8 +94,7 @@ func (h *HTTPDashboardHandler) Register() error {
 	retBody, err := ioutil.ReadAll(response.Body)
 
 	if response.StatusCode != 200 {
-		log.Error("Failed to register node, retrying in 5s")
-		log.Error(" --> Response was: ", string(retBody))
+		log.Error("Failed to register node, retrying in 5s; Response was: ", string(retBody))
 		time.Sleep(time.Second * 5)
 		return h.Register()
 	}

dq.go

@@ -27,7 +27,8 @@ func getDQTopic() string {
 	if config.DBAppConfOptions.NodeIsSegmented {
 		if len(config.DBAppConfOptions.Tags) > 0 {
 			tags := strings.Join(config.DBAppConfOptions.Tags, ".")
-			topic = topic + "." + tags
+			topic += "." + tags
+
 		}
 	}
 
@@ -94,13 +95,7 @@ func dqFlusher(d map[string]*dq.Quota) error {
 					continue
 				}
 
-				if time.Now().After(expT) {
-					QuotaHandler.TagDelete(k)
-					continue
-				}
-
-				if s.IsExpired() {
-					// Remove expired data too
+				if !f || time.Now().After(expT) || s.IsExpired() {
 					QuotaHandler.TagDelete(k)
 					continue
 				}

gateway_test.go

@@ -758,8 +758,8 @@ func TestDistributedQuotaSingleNode(t *testing.T) {
 	defer spec.SessionManager.ResetQuota(keyId, session)
 
 	recorder := httptest.NewRecorder()
-	param := make(url.Values)
-	req, err := http.NewRequest("GET", param.Encode(), nil)
+
+	req, err := http.NewRequest("GET", "", nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -768,6 +768,11 @@ func TestDistributedQuotaSingleNode(t *testing.T) {
 	config.DQSetMaster = true
 	config.UseDistributedQuotaCounter = true
 	config.DistributedQuotaFlushIntervalInMS = 100
+	defer func() {
+		config.DQSetMaster = false
+		config.UseDistributedQuotaCounter = false
+		config.DistributedQuotaFlushIntervalInMS = 0
+	}()
 
 	startDQ(decideLeaderMechanism())
 

host_checker.go

@@ -20,7 +20,6 @@ var HostCheckerClient = &http.Client{Timeout: 500 * time.Millisecond}
 
 type HostData struct {
 	CheckURL string
-	ID       string
 	Method   string
 	Headers  map[string]string
 	Body     string
@@ -95,26 +94,26 @@ func (h *HostUptimeChecker) HostReporter() {
 		select {
 		case okHost := <-h.okChan:
 			// Clear host from unhealthylist if it exists
-			if h.unHealthyList[okHost.ID] {
+			if h.unHealthyList[okHost.CheckURL] {
 				h.upCallback(okHost)
-				delete(h.unHealthyList, okHost.ID)
+				delete(h.unHealthyList, okHost.CheckURL)
 			}
 			go h.pingCallback(okHost)
 
 		case failedHost := <-h.errorChan:
 			newVal := 1
-			if count, found := h.sampleCache.Get(failedHost.ID); found {
+			if count, found := h.sampleCache.Get(failedHost.CheckURL); found {
 				newVal = count.(int) + 1
 			}
 
-			h.sampleCache.Set(failedHost.ID, newVal, cache.DefaultExpiration)
+			h.sampleCache.Set(failedHost.CheckURL, newVal, cache.DefaultExpiration)
 
 			if newVal >= h.sampleTriggerLimit {
 				log.Debug("[HOST CHECKER] [HOST WARNING]: ", failedHost.CheckURL)
 				// Reset the count
-				h.sampleCache.Set(failedHost.ID, 1, cache.DefaultExpiration)
+				h.sampleCache.Set(failedHost.CheckURL, 1, cache.DefaultExpiration)
 				// track it
-				h.unHealthyList[failedHost.ID] = true
+				h.unHealthyList[failedHost.CheckURL] = true
 				// Call the custom callback hook
 				go h.failureCallback(failedHost)
 			}
@@ -128,7 +127,7 @@ func (h *HostUptimeChecker) HostReporter() {
 }
 
 func (h *HostUptimeChecker) CheckHost(toCheck HostData) {
-	log.Debug("[HOST CHECKER] Checking: ", toCheck.CheckURL, toCheck.ID)
+	log.Debug("[HOST CHECKER] Checking: ", toCheck.CheckURL)
 
 	t1 := time.Now()
 

host_checker_manager.go

@@ -311,7 +311,6 @@ func (hc *HostCheckerManager) PrepareTrackingHost(checkObject apidef.HostCheckOb
 
 	hostData = HostData{
 		CheckURL: checkObject.CheckURL,
-		ID:       checkObject.CheckURL,
 		MetaData: make(map[string]string),
 		Method:   checkObject.Method,
 		Headers:  checkObject.Headers,

main.go

@@ -29,7 +29,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/justinas/alice"
 	"github.com/lonelycode/gorpc"
-	osin "github.com/lonelycode/osin"
+	"github.com/lonelycode/osin"
 	"github.com/rs/cors"
 	"rsc.io/letsencrypt"
 

middleware_virtual_endpoint.go

@@ -192,8 +192,8 @@ func (d *VirtualEndpoint) ServeHTTPForCache(w http.ResponseWriter, r *http.Reque
 	// Decode the return object
 	newResponseData := VMResponseObject{}
 	if err := json.Unmarshal([]byte(returnDataStr), &newResponseData); err != nil {
-		log.Error("Failed to decode virtual endpoint response data on return from VM: ", err)
-		log.Error("--> Returned: ", returnDataStr)
+		log.Error("Failed to decode virtual endpoint response data on return from VM: ", err,
+			"; Returned: ", returnDataStr)
 		return nil
 	}