Merge branch 'master' into http2

config/config.go

@@ -126,8 +126,10 @@ type HttpServerOptionsConfig struct {
 	MinVersion            uint16     `json:"min_version"`
 	FlushInterval         int        `json:"flush_interval"`
 	SkipURLCleaning       bool       `json:"skip_url_cleaning"`
+  SkipTargetPathEscaping bool       `json:"skip_target_path_escaping"`
 	Ciphers               []string   `json:"ssl_ciphers"`
 	UseHttp2			  bool		 `json:"use_http2"`
+
 }
 
 type AuthOverrideConf struct {

gateway_test.go

@@ -47,6 +47,7 @@ var (
 const defaultListenPort = 8080
 
 var defaultTestConfig config.Config
+var testServerRouter *mux.Router
 
 func resetTestConfig() {
 	configMu.Lock()
@@ -75,9 +76,10 @@ func reloadSimulation() {
 }
 
 func TestMain(m *testing.M) {
+	testServerRouter = testHttpHandler()
 	testServer := &http.Server{
 		Addr:           testHttpListen,
-		Handler:        testHttpHandler(),
+		Handler:        testServerRouter,
 		ReadTimeout:    1 * time.Second,
 		WriteTimeout:   1 * time.Second,
 		MaxHeaderBytes: 1 << 20,
@@ -266,6 +268,203 @@ func TestParambasedAuth(t *testing.T) {
 	})
 }
 
+func TestSkipTargetPassEscapingOff(t *testing.T) {
+	ts := newTykTestServer()
+	defer ts.Close()
+	defer resetTestConfig()
+
+	t.Run("With escaping, default", func(t *testing.T) {
+		config.Global.HttpServerOptions.SkipTargetPathEscaping = false
+
+		buildAndLoadAPI(func(spec *APISpec) {
+			spec.Proxy.ListenPath = "/"
+		})
+
+		ts.Run(t, []test.TestCase{
+			{Path: "/(abc,xyz)?arg=val", BodyMatch: `"Url":"/%28abc,xyz%29?arg=val`},
+			{Path: "/%28abc,xyz%29?arg=val", BodyMatch: `"Url":"/%28abc,xyz%29?arg=val`},
+		}...)
+	})
+
+	t.Run("Without escaping", func(t *testing.T) {
+		config.Global.HttpServerOptions.SkipTargetPathEscaping = true
+
+		buildAndLoadAPI(func(spec *APISpec) {
+			spec.Proxy.ListenPath = "/"
+		})
+
+		ts.Run(t, []test.TestCase{
+			{Path: "/(abc,xyz)?arg=val", BodyMatch: `"Url":"/(abc,xyz)?arg=val"`},
+			{Path: "/%28abc,xyz%29?arg=val", BodyMatch: `"Url":"/%28abc,xyz%29?arg=val"`},
+		}...)
+	})
+
+	t.Run("With escaping, listen path and target URL are set, StripListenPath is OFF", func(t *testing.T) {
+		config.Global.HttpServerOptions.SkipTargetPathEscaping = false
+
+		buildAndLoadAPI(func(spec *APISpec) {
+			spec.Proxy.StripListenPath = false
+			spec.Proxy.ListenPath = "/listen_me"
+			spec.Proxy.TargetURL = testHttpAny + "/sent_to_me"
+		})
+
+		ts.Run(t, []test.TestCase{
+			{Path: "/listen_me/(abc,xyz)?arg=val", BodyMatch: `"Url":"/sent_to_me/listen_me/%28abc,xyz%29?arg=val"`},
+			{Path: "/listen_me/%28abc,xyz%29?arg=val", BodyMatch: `"Url":"/sent_to_me/listen_me/%28abc,xyz%29?arg=val"`},
+		}...)
+	})
+
+	t.Run("Without escaping, listen path and target URL are set, StripListenPath is OFF", func(t *testing.T) {
+		config.Global.HttpServerOptions.SkipTargetPathEscaping = true
+
+		buildAndLoadAPI(func(spec *APISpec) {
+			spec.Proxy.StripListenPath = false
+			spec.Proxy.ListenPath = "/listen_me"
+			spec.Proxy.TargetURL = testHttpAny + "/sent_to_me"
+		})
+
+		ts.Run(t, []test.TestCase{
+			{Path: "/listen_me/(abc,xyz)?arg=val", BodyMatch: `"Url":"/sent_to_me/listen_me/(abc,xyz)?arg=val"`},
+			{Path: "/listen_me/%28abc,xyz%29?arg=val", BodyMatch: `"Url":"/sent_to_me/listen_me/%28abc,xyz%29?arg=val"`},
+		}...)
+	})
+
+	t.Run("With escaping, listen path and target URL are set, StripListenPath is ON", func(t *testing.T) {
+		config.Global.HttpServerOptions.SkipTargetPathEscaping = false
+
+		buildAndLoadAPI(func(spec *APISpec) {
+			spec.Proxy.StripListenPath = true
+			spec.Proxy.ListenPath = "/listen_me"
+			spec.Proxy.TargetURL = testHttpAny + "/sent_to_me"
+		})
+
+		ts.Run(t, []test.TestCase{
+			{Path: "/listen_me/(abc,xyz)?arg=val", BodyMatch: `"Url":"/sent_to_me/%28abc,xyz%29?arg=val"`},
+			{Path: "/listen_me/%28abc,xyz%29?arg=val", BodyMatch: `"Url":"/sent_to_me/%28abc,xyz%29?arg=val"`},
+		}...)
+	})
+
+	t.Run("Without escaping, listen path and target URL are set, StripListenPath is ON", func(t *testing.T) {
+		config.Global.HttpServerOptions.SkipTargetPathEscaping = true
+
+		buildAndLoadAPI(func(spec *APISpec) {
+			spec.Proxy.StripListenPath = true
+			spec.Proxy.ListenPath = "/listen_me"
+			spec.Proxy.TargetURL = testHttpAny + "/sent_to_me"
+		})
+
+		ts.Run(t, []test.TestCase{
+			{Path: "/listen_me/(abc,xyz)?arg=val", BodyMatch: `"Url":"/sent_to_me/(abc,xyz)?arg=val"`},
+			{Path: "/listen_me/%28abc,xyz%29?arg=val", BodyMatch: `"Url":"/sent_to_me/%28abc,xyz%29?arg=val"`},
+		}...)
+	})
+}
+
+func TestSkipTargetPassEscapingOffWithSkipURLCleaningTrue(t *testing.T) {
+	config.Global.HttpServerOptions.OverrideDefaults = true
+	config.Global.HttpServerOptions.SkipURLCleaning = true
+	defer resetTestConfig()
+
+	// here we expect that test gateway will be sending to test upstream requests with not cleaned URI
+	// so test upstream shouldn't reply with 301 and process them as well
+	prevSkipClean := defaultTestConfig.HttpServerOptions.OverrideDefaults &&
+		defaultTestConfig.HttpServerOptions.SkipURLCleaning
+	testServerRouter.SkipClean(true)
+	defer testServerRouter.SkipClean(prevSkipClean)
+
+	ts := newTykTestServer()
+	defer ts.Close()
+
+	t.Run("With escaping, default", func(t *testing.T) {
+		config.Global.HttpServerOptions.SkipTargetPathEscaping = false
+
+		buildAndLoadAPI(func(spec *APISpec) {
+			spec.Proxy.ListenPath = "/"
+		})
+
+		ts.Run(t, []test.TestCase{
+			{Path: "/abc/xyz/http%3A%2F%2Ftest.com?arg=val", BodyMatch: `"Url":"/abc/xyz/http%3A%2F%2Ftest.com?arg=val`},
+		}...)
+	})
+
+	t.Run("Without escaping, default", func(t *testing.T) {
+		config.Global.HttpServerOptions.SkipTargetPathEscaping = true
+
+		buildAndLoadAPI(func(spec *APISpec) {
+			spec.Proxy.ListenPath = "/"
+		})
+
+		ts.Run(t, []test.TestCase{
+			{Path: "/abc/xyz/http%3A%2F%2Ftest.com?arg=val", BodyMatch: `"Url":"/abc/xyz/http%3A%2F%2Ftest.com?arg=val`},
+		}...)
+	})
+
+	t.Run("With escaping, listen path and target URL are set, StripListenPath is OFF", func(t *testing.T) {
+		config.Global.HttpServerOptions.SkipTargetPathEscaping = false
+
+		buildAndLoadAPI(func(spec *APISpec) {
+			spec.Proxy.StripListenPath = false
+			spec.Proxy.ListenPath = "/listen_me"
+			spec.Proxy.TargetURL = testHttpAny + "/sent_to_me"
+		})
+
+		ts.Run(t, []test.TestCase{
+			{Path: "/listen_me/(abc,xyz)?arg=val", BodyMatch: `"Url":"/sent_to_me/listen_me/%28abc,xyz%29?arg=val"`},
+			{Path: "/listen_me/%28abc,xyz%29?arg=val", BodyMatch: `"Url":"/sent_to_me/listen_me/%28abc,xyz%29?arg=val"`},
+			{Path: "/listen_me/http%3A%2F%2Ftest.com?arg=val", BodyMatch: `"Url":"/sent_to_me/listen_me/http%3A%2F%2Ftest.com?arg=val`},
+		}...)
+	})
+
+	t.Run("Without escaping, listen path and target URL are set, StripListenPath is OFF", func(t *testing.T) {
+		config.Global.HttpServerOptions.SkipTargetPathEscaping = true
+
+		buildAndLoadAPI(func(spec *APISpec) {
+			spec.Proxy.StripListenPath = false
+			spec.Proxy.ListenPath = "/listen_me"
+			spec.Proxy.TargetURL = testHttpAny + "/sent_to_me"
+		})
+
+		ts.Run(t, []test.TestCase{
+			{Path: "/listen_me/(abc,xyz)?arg=val", BodyMatch: `"Url":"/sent_to_me/listen_me/(abc,xyz)?arg=val"`},
+			{Path: "/listen_me/%28abc,xyz%29?arg=val", BodyMatch: `"Url":"/sent_to_me/listen_me/%28abc,xyz%29?arg=val"`},
+			{Path: "/listen_me/http%3A%2F%2Ftest.com?arg=val", BodyMatch: `"Url":"/sent_to_me/listen_me/http%3A%2F%2Ftest.com?arg=val`},
+		}...)
+	})
+
+	t.Run("With escaping, listen path and target URL are set, StripListenPath is ON", func(t *testing.T) {
+		config.Global.HttpServerOptions.SkipTargetPathEscaping = false
+
+		buildAndLoadAPI(func(spec *APISpec) {
+			spec.Proxy.StripListenPath = true
+			spec.Proxy.ListenPath = "/listen_me"
+			spec.Proxy.TargetURL = testHttpAny + "/sent_to_me"
+		})
+
+		ts.Run(t, []test.TestCase{
+			{Path: "/listen_me/(abc,xyz)?arg=val", BodyMatch: `"Url":"/sent_to_me/%28abc,xyz%29?arg=val"`},
+			{Path: "/listen_me/%28abc,xyz%29?arg=val", BodyMatch: `"Url":"/sent_to_me/%28abc,xyz%29?arg=val"`},
+			{Path: "/listen_me/http%3A%2F%2Ftest.com?arg=val", BodyMatch: `"Url":"/sent_to_me/http%3A%2F%2Ftest.com?arg=val`},
+		}...)
+	})
+
+	t.Run("Without escaping, listen path and target URL are set, StripListenPath is ON", func(t *testing.T) {
+		config.Global.HttpServerOptions.SkipTargetPathEscaping = true
+
+		buildAndLoadAPI(func(spec *APISpec) {
+			spec.Proxy.StripListenPath = true
+			spec.Proxy.ListenPath = "/listen_me"
+			spec.Proxy.TargetURL = testHttpAny + "/sent_to_me"
+		})
+
+		ts.Run(t, []test.TestCase{
+			{Path: "/listen_me/(abc,xyz)?arg=val", BodyMatch: `"Url":"/sent_to_me/(abc,xyz)?arg=val"`},
+			{Path: "/listen_me/%28abc,xyz%29?arg=val", BodyMatch: `"Url":"/sent_to_me/%28abc,xyz%29?arg=val"`},
+			{Path: "/listen_me/http%3A%2F%2Ftest.com?arg=val", BodyMatch: `"Url":"/sent_to_me/http%3A%2F%2Ftest.com?arg=val`},
+		}...)
+	})
+
+}
+
 func TestQuota(t *testing.T) {
 	ts := newTykTestServer()
 	defer ts.Close()
@@ -406,7 +605,7 @@ func TestAnalytics(t *testing.T) {
 }
 
 func TestListener(t *testing.T) {
-	// Trcik to get spec JSON, without loading API
+	// Trick to get spec JSON, without loading API
 	// Specs will be reseted when we do `newTykTestServer`
 	specs := buildAndLoadAPI()
 	specJSON, _ := json.Marshal(specs[0].APIDefinition)

handler_success.go

@@ -227,6 +227,7 @@ func (s *SuccessHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) *http
 	if s.Spec.Proxy.StripListenPath {
 		log.Debug("Stripping: ", s.Spec.Proxy.ListenPath)
 		r.URL.Path = strings.Replace(r.URL.Path, s.Spec.Proxy.ListenPath, "", 1)
+		r.URL.RawPath = strings.Replace(r.URL.RawPath, s.Spec.Proxy.ListenPath, "", 1)
 		log.Debug("Upstream Path is: ", r.URL.Path)
 	}
 
@@ -262,6 +263,7 @@ func (s *SuccessHandler) ServeHTTPWithCache(w http.ResponseWriter, r *http.Reque
 	// Make sure we get the correct target URL
 	if s.Spec.Proxy.StripListenPath {
 		r.URL.Path = strings.Replace(r.URL.Path, s.Spec.Proxy.ListenPath, "", 1)
+		r.URL.RawPath = strings.Replace(r.URL.RawPath, s.Spec.Proxy.ListenPath, "", 1)
 	}
 
 	var copiedRequest *http.Request

helpers_test.go

@@ -24,6 +24,8 @@ import (
 	"github.com/miekg/dns"
 	"github.com/satori/go.uuid"
 
+	"github.com/gorilla/mux"
+
 	"github.com/TykTechnologies/tyk/apidef"
 	"github.com/TykTechnologies/tyk/config"
 	"github.com/TykTechnologies/tyk/test"
@@ -92,7 +94,7 @@ const (
 	testHttpFailureAny = "http://" + testHttpFailure
 )
 
-func testHttpHandler() http.Handler {
+func testHttpHandler() *mux.Router {
 	var upgrader = websocket.Upgrader{
 		ReadBufferSize:  1024,
 		WriteBufferSize: 1024,
@@ -124,6 +126,7 @@ func testHttpHandler() http.Handler {
 			httpError(w, http.StatusInternalServerError)
 			return
 		}
+		r.URL.Opaque = r.URL.RawPath
 		err := json.NewEncoder(w).Encode(testHttpResponse{
 			Method:  r.Method,
 			Url:     r.URL.String(),
@@ -143,17 +146,20 @@ func testHttpHandler() http.Handler {
 			}
 		}
 	}
-	mux := http.NewServeMux()
-	mux.HandleFunc("/", handleMethod(""))
-	mux.HandleFunc("/get", handleMethod("GET"))
-	mux.HandleFunc("/post", handleMethod("POST"))
-	mux.HandleFunc("/ws", wsHandler)
-	mux.HandleFunc("/jwk.json", func(w http.ResponseWriter, r *http.Request) {
+
+	// use gorilla's mux as it allows to cancel URI cleaning
+	// (it is not configurable in standard http mux)
+	r := mux.NewRouter()
+	r.HandleFunc("/get", handleMethod("GET"))
+	r.HandleFunc("/post", handleMethod("POST"))
+	r.HandleFunc("/ws", wsHandler)
+	r.HandleFunc("/jwk.json", func(w http.ResponseWriter, r *http.Request) {
 		io.WriteString(w, jwkTestJson)
 	})
+	r.HandleFunc("/bundles/{rest:.*}", bundleHandleFunc)
+	r.HandleFunc("/{rest:.*}", handleMethod(""))
 
-	mux.HandleFunc("/bundles/", bundleHandleFunc)
-	return mux
+	return r
 }
 
 const jwkTestJson = `{

lint/schema.go

@@ -331,6 +331,9 @@ const confSchema = `{
 			"skip_url_cleaning": {
 				"type": "boolean"
 			},
+			"skip_target_path_escaping": {
+				"type": "boolean"
+			},
 			"ssl_insecure_skip_verify": {
 				"type": "boolean"
 			},

reverse_proxy.go

@@ -235,6 +235,9 @@ func TykNewSingleHostReverseProxy(target *url.URL, spec *APISpec) *ReverseProxy
 			req.URL.Scheme = targetToUse.Scheme
 			req.URL.Host = targetToUse.Host
 			req.URL.Path = singleJoiningSlash(targetToUse.Path, req.URL.Path)
+			if req.URL.RawPath != "" {
+				req.URL.RawPath = singleJoiningSlash(targetToUse.Path, req.URL.RawPath)
+			}
 		}
 		if !spec.Proxy.PreserveHostHeader {
 			req.Host = targetToUse.Host
@@ -249,6 +252,17 @@ func TykNewSingleHostReverseProxy(target *url.URL, spec *APISpec) *ReverseProxy
 			// this line, we would get the net/http default.
 			req.Header.Set("User-Agent", defaultUserAgent)
 		}
+
+		if config.Global.HttpServerOptions.SkipTargetPathEscaping {
+			// force RequestURI to skip escaping if API's proxy is set for this
+			// if we set opaque here it will force URL.RequestURI to skip escaping
+			if req.URL.RawPath != "" {
+				req.URL.Opaque = req.URL.RawPath
+			}
+		} else if req.URL.RawPath == req.URL.Path {
+			// this should force URL to do escaping
+			req.URL.RawPath = ""
+		}
 	}
 
 	proxy := &ReverseProxy{