Fix 404 during reloads

Issue was happening on high load, because mainRouter object was
initialized with fresh router, before API routes was loaded into it.

Additionally added more mutexes.

api_loader.go

@@ -518,20 +518,20 @@ func (d *DummyProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	d.SH.ServeHTTP(w, r)
 }
 
-func loadGlobalApps() {
+func loadGlobalApps(router *mux.Router) {
 	// we need to make a full copy of the slice, as loadApps will
 	// use in-place to sort the apis.
 	apisMu.RLock()
 	specs := make([]*APISpec, len(apiSpecs))
 	copy(specs, apiSpecs)
 	apisMu.RUnlock()
-	loadApps(specs, mainRouter)
+	loadApps(specs, router)
 
 	if config.Global.NewRelic.AppName != "" {
 		log.WithFields(logrus.Fields{
 			"prefix": "main",
 		}).Info("Adding NewRelic instrumentation")
-		AddNewRelicInstrumentation(NewRelicApplication, mainRouter)
+		AddNewRelicInstrumentation(NewRelicApplication, router)
 	}
 }
 

gateway_test.go

@@ -706,6 +706,27 @@ func TestWebsocketsUpstreamUpgradeRequest(t *testing.T) {
 	})
 }
 
+func TestConcurrencyReloads(t *testing.T) {
+	var wg sync.WaitGroup
+
+	ts := newTykTestServer()
+	defer ts.Close()
+
+	buildAndLoadAPI()
+
+	for i := 0; i < 10; i++ {
+		wg.Add(1)
+		go func() {
+			ts.Run(t, test.TestCase{Path: "/sample", Code: 200})
+			wg.Done()
+		}()
+
+		go buildAndLoadAPI()
+	}
+
+	wg.Wait()
+}
+
 func TestWebsocketsSeveralOpenClose(t *testing.T) {
 	config.Global.HttpServerOptions.EnableWebSockets = true
 	defer resetTestConfig()

helpers_test.go

@@ -253,7 +253,6 @@ type tykTestServerConfig struct {
 	delay              time.Duration
 	hotReload          bool
 	overrideDefaults   bool
-	lastResponse       *http.Response
 }
 
 type tykTestServer struct {

main.go

@@ -126,6 +126,9 @@ func apisByIDLen() int {
 
 // Create all globals and init connection handlers
 func setupGlobals() {
+	reloadMu.Lock()
+	defer reloadMu.Unlock()
+
 	mainRouter = mux.NewRouter()
 	controlRouter = mux.NewRouter()
 
@@ -656,21 +659,23 @@ func doReload() {
 	log.WithFields(logrus.Fields{
 		"prefix": "main",
 	}).Info("Preparing new router")
-	mainRouter = mux.NewRouter()
+	newRouter := mux.NewRouter()
 	if config.Global.HttpServerOptions.OverrideDefaults {
-		mainRouter.SkipClean(config.Global.HttpServerOptions.SkipURLCleaning)
+		newRouter.SkipClean(config.Global.HttpServerOptions.SkipURLCleaning)
 	}
 
 	if config.Global.ControlAPIPort == 0 {
-		loadAPIEndpoints(mainRouter)
+		loadAPIEndpoints(newRouter)
 	}
 
-	loadGlobalApps()
+	loadGlobalApps(newRouter)
 
 	log.WithFields(logrus.Fields{
 		"prefix": "main",
 	}).Info("API reload complete")
 
+	mainRouter = newRouter
+
 	// Unset these
 	rpcEmergencyModeLoaded = false
 	rpcEmergencyMode = false
@@ -1321,7 +1326,7 @@ func listen(l, controlListener net.Listener, err error) {
 		if !rpcEmergencyMode {
 			count := syncAPISpecs()
 			if count > 0 {
-				loadGlobalApps()
+				loadGlobalApps(mainRouter)
 				syncPolicies()
 			}
 
@@ -1397,7 +1402,7 @@ func listen(l, controlListener net.Listener, err error) {
 		if !rpcEmergencyMode {
 			count := syncAPISpecs()
 			if count > 0 {
-				loadGlobalApps()
+				loadGlobalApps(mainRouter)
 				syncPolicies()
 			}
 

rpc_backup_handlers.go

@@ -95,7 +95,6 @@ func doLoadWithBackup(specs []*APISpec) {
 	log.Warning("[RPC Backup] --> Initialised JSVM")
 
 	newRouter := mux.NewRouter()
-	mainRouter = newRouter
 
 	log.Warning("[RPC Backup] --> Set up routers")
 	log.Warning("[RPC Backup] --> Loading endpoints")
@@ -108,12 +107,14 @@ func doLoadWithBackup(specs []*APISpec) {
 
 	if config.Global.NewRelic.AppName != "" {
 		log.Warning("[RPC Backup] --> Adding NewRelic instrumentation")
-		AddNewRelicInstrumentation(NewRelicApplication, mainRouter)
+		AddNewRelicInstrumentation(NewRelicApplication, newRouter)
 		log.Warning("[RPC Backup] --> NewRelic instrumentation added")
 	}
 
 	newServeMux := http.NewServeMux()
-	newServeMux.Handle("/", mainRouter)
+	newServeMux.Handle("/", newRouter)
+
+	mainRouter = newRouter
 
 	http.DefaultServeMux = newServeMux
 	log.Warning("[RPC Backup] --> Replaced muxer")