Fix 404 during reloads

Issue was happening on high load, because mainRouter object was
initialized with fresh router, before API routes was loaded into it.

Additionally added more mutexes.

api_definition.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"text/template"
 	"time"
@@ -116,6 +117,7 @@ type ExtendedCircuitBreakerMeta struct {
 // flattened URL list is checked for matching paths and then it's status evaluated if found.
 type APISpec struct {
 	*apidef.APIDefinition
+	sync.Mutex
 
 	RxPaths                  map[string][]URLSpec
 	WhiteListEnabled         map[string]bool

api_loader.go

@@ -518,20 +518,20 @@ func (d *DummyProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	d.SH.ServeHTTP(w, r)
 }
 
-func loadGlobalApps() {
+func loadGlobalApps(router *mux.Router) {
 	// we need to make a full copy of the slice, as loadApps will
 	// use in-place to sort the apis.
 	apisMu.RLock()
 	specs := make([]*APISpec, len(apiSpecs))
 	copy(specs, apiSpecs)
 	apisMu.RUnlock()
-	loadApps(specs, mainRouter)
+	loadApps(specs, router)
 
 	if config.Global.NewRelic.AppName != "" {
 		log.WithFields(logrus.Fields{
 			"prefix": "main",
 		}).Info("Adding NewRelic instrumentation")
-		AddNewRelicInstrumentation(NewRelicApplication, mainRouter)
+		AddNewRelicInstrumentation(NewRelicApplication, router)
 	}
 }
 

gateway_test.go

@@ -706,6 +706,29 @@ func TestWebsocketsUpstreamUpgradeRequest(t *testing.T) {
 	})
 }
 
+func TestConcurrencyReloads(t *testing.T) {
+	var wg sync.WaitGroup
+
+	ts := newTykTestServer()
+	defer ts.Close()
+
+	buildAndLoadAPI()
+
+	for i := 0; i < 10; i++ {
+		wg.Add(1)
+		go func() {
+			ts.Run(t, test.TestCase{Path: "/sample", Code: 200})
+			wg.Done()
+		}()
+	}
+
+	for j := 0; j < 5; j++ {
+		buildAndLoadAPI()
+	}
+
+	wg.Wait()
+}
+
 func TestWebsocketsSeveralOpenClose(t *testing.T) {
 	config.Global.HttpServerOptions.EnableWebSockets = true
 	defer resetTestConfig()

helpers_test.go

@@ -253,7 +253,6 @@ type tykTestServerConfig struct {
 	delay              time.Duration
 	hotReload          bool
 	overrideDefaults   bool
-	lastResponse       *http.Response
 }
 
 type tykTestServer struct {
@@ -488,10 +487,11 @@ func buildAPI(apiGens ...func(spec *APISpec)) (specs []*APISpec) {
 func loadAPI(specs ...*APISpec) (out []*APISpec) {
 	oldPath := config.Global.AppPath
 	config.Global.AppPath, _ = ioutil.TempDir("", "apps")
+
 	defer func() {
+		os.RemoveAll(config.Global.AppPath)
 		config.Global.AppPath = oldPath
 	}()
-	defer os.RemoveAll(config.Global.AppPath)
 
 	for i, spec := range specs {
 		specBytes, _ := json.Marshal(spec)

main.go

@@ -126,6 +126,9 @@ func apisByIDLen() int {
 
 // Create all globals and init connection handlers
 func setupGlobals() {
+	reloadMu.Lock()
+	defer reloadMu.Unlock()
+
 	mainRouter = mux.NewRouter()
 	controlRouter = mux.NewRouter()
 
@@ -656,21 +659,23 @@ func doReload() {
 	log.WithFields(logrus.Fields{
 		"prefix": "main",
 	}).Info("Preparing new router")
-	mainRouter = mux.NewRouter()
+	newRouter := mux.NewRouter()
 	if config.Global.HttpServerOptions.OverrideDefaults {
-		mainRouter.SkipClean(config.Global.HttpServerOptions.SkipURLCleaning)
+		newRouter.SkipClean(config.Global.HttpServerOptions.SkipURLCleaning)
 	}
 
 	if config.Global.ControlAPIPort == 0 {
-		loadAPIEndpoints(mainRouter)
+		loadAPIEndpoints(newRouter)
 	}
 
-	loadGlobalApps()
+	loadGlobalApps(newRouter)
 
 	log.WithFields(logrus.Fields{
 		"prefix": "main",
 	}).Info("API reload complete")
 
+	mainRouter = newRouter
+
 	// Unset these
 	rpcEmergencyModeLoaded = false
 	rpcEmergencyMode = false
@@ -1321,7 +1326,7 @@ func listen(l, controlListener net.Listener, err error) {
 		if !rpcEmergencyMode {
 			count := syncAPISpecs()
 			if count > 0 {
-				loadGlobalApps()
+				loadGlobalApps(mainRouter)
 				syncPolicies()
 			}
 
@@ -1397,7 +1402,7 @@ func listen(l, controlListener net.Listener, err error) {
 		if !rpcEmergencyMode {
 			count := syncAPISpecs()
 			if count > 0 {
-				loadGlobalApps()
+				loadGlobalApps(mainRouter)
 				syncPolicies()
 			}
 

reverse_proxy.go

@@ -435,6 +435,7 @@ func httpTransport(timeOut int, rw http.ResponseWriter, req *http.Request, p *Re
 
 func (p *ReverseProxy) WrappedServeHTTP(rw http.ResponseWriter, req *http.Request, withCache bool) *http.Response {
 	// 1. Check if timeouts are set for this endpoint
+	p.TykAPISpec.Lock()
 	if p.TykAPISpec.HTTPTransport == nil {
 		_, timeout := p.CheckHardTimeoutEnforced(p.TykAPISpec, req)
 		p.TykAPISpec.HTTPTransport = httpTransport(timeout, rw, req, p)
@@ -443,6 +444,7 @@ func (p *ReverseProxy) WrappedServeHTTP(rw http.ResponseWriter, req *http.Reques
 		// as it was already hijacked and now is being used for other connection
 		p.TykAPISpec.HTTPTransport.(*WSDialer).RW = rw
 	}
+	p.TykAPISpec.Unlock()
 
 	ctx := req.Context()
 	if cn, ok := rw.(http.CloseNotifier); ok {
@@ -527,11 +529,14 @@ func (p *ReverseProxy) WrappedServeHTTP(rw http.ResponseWriter, req *http.Reques
 	if cert := getUpstreamCertificate(outreq.Host, p.TykAPISpec); cert != nil {
 		tlsCertificates = []tls.Certificate{*cert}
 	}
+
+	p.TykAPISpec.Lock()
 	if outReqIsWebsocket {
 		p.TykAPISpec.HTTPTransport.(*WSDialer).TLSClientConfig.Certificates = tlsCertificates
 	} else {
 		p.TykAPISpec.HTTPTransport.(*http.Transport).TLSClientConfig.Certificates = tlsCertificates
 	}
+	p.TykAPISpec.Unlock()
 
 	// do request round trip
 	var res *http.Response

rpc_backup_handlers.go

@@ -95,7 +95,6 @@ func doLoadWithBackup(specs []*APISpec) {
 	log.Warning("[RPC Backup] --> Initialised JSVM")
 
 	newRouter := mux.NewRouter()
-	mainRouter = newRouter
 
 	log.Warning("[RPC Backup] --> Set up routers")
 	log.Warning("[RPC Backup] --> Loading endpoints")
@@ -108,12 +107,14 @@ func doLoadWithBackup(specs []*APISpec) {
 
 	if config.Global.NewRelic.AppName != "" {
 		log.Warning("[RPC Backup] --> Adding NewRelic instrumentation")
-		AddNewRelicInstrumentation(NewRelicApplication, mainRouter)
+		AddNewRelicInstrumentation(NewRelicApplication, newRouter)
 		log.Warning("[RPC Backup] --> NewRelic instrumentation added")
 	}
 
 	newServeMux := http.NewServeMux()
-	newServeMux.Handle("/", mainRouter)
+	newServeMux.Handle("/", newRouter)
+
+	mainRouter = newRouter
 
 	http.DefaultServeMux = newServeMux
 	log.Warning("[RPC Backup] --> Replaced muxer")