feature: add multiple registry mirrors support (#479)

Signed-off-by: Jim Ma <majinjing3@gmail.com>

client/config/headers.go

@@ -21,4 +21,6 @@ const (
 	HeaderDragonflyPeer   = "X-Dragonfly-Peer"
 	HeaderDragonflyTask   = "X-Dragonfly-Task"
 	HeaderDragonflyBiz    = "X-Dragonfly-Biz"
+	// HeaderDragonflyRegistry is used for dynamic registry mirrors
+	HeaderDragonflyRegistry = "X-Dragonfly-Registry"
 )

client/config/peerhost.go

@@ -467,6 +467,10 @@ type RegistryMirror struct {
 	// Remote url for the registry mirror, default is https://index.docker.io
 	Remote *URL `yaml:"url" mapstructure:"url"`
 
+	// DynamicRemote indicates using header "X-Dragonfly-Registry" for remote instead of Remote
+	// if header "X-Dragonfly-Registry" does not exist, use Remote by default
+	DynamicRemote bool `yaml:"dynamic" mapstructure:"dynamic"`
+
 	// Optional certificates if the mirror uses self-signed certificates
 	Certs *CertPool `yaml:"certs" mapstructure:"certs"`
 

client/daemon/proxy/proxy.go

@@ -412,7 +412,7 @@ func (proxy *Proxy) newTransport(tlsConfig *tls.Config) http.RoundTripper {
 }
 
 func (proxy *Proxy) mirrorRegistry(w http.ResponseWriter, r *http.Request) {
-	reverseProxy := httputil.NewSingleHostReverseProxy(proxy.registry.Remote.URL)
+	reverseProxy := newReverseProxy(proxy.registry)
 	t, err := transport.New(
 		transport.WithPeerHost(proxy.peerHost),
 		transport.WithPeerTaskManager(proxy.peerTaskManager),

client/daemon/proxy/proxy_test.go

@@ -56,16 +56,17 @@ func (tc *testCase) WithRule(regx string, direct bool, useHTTPS bool, redirect s
 	return tc
 }
 
-func (tc *testCase) WithRegistryMirror(rawURL string, direct bool) *testCase {
+func (tc *testCase) WithRegistryMirror(rawURL string, direct bool, dynamic bool) *testCase {
 	if tc.Error != nil {
 		return tc
 	}
 
-	var url *url.URL
-	url, tc.Error = url.Parse(rawURL)
+	var u *url.URL
+	u, tc.Error = url.Parse(rawURL)
 	tc.RegistryMirror = &config.RegistryMirror{
-		Remote: &config.URL{URL: url},
-		Direct: direct,
+		Remote:        &config.URL{URL: u},
+		DynamicRemote: dynamic,
+		Direct:        direct,
 	}
 	return tc
 }
@@ -153,17 +154,17 @@ func TestMatch(t *testing.T) {
 		TestMirror(t)
 
 	newTestCase().
-		WithRegistryMirror("http://index.docker.io", false).
+		WithRegistryMirror("http://index.docker.io", false, false).
 		WithTest("http://h/a", true, false, "").
 		TestMirror(t)
 
 	newTestCase().
-		WithRegistryMirror("http://index.docker.io", false).
+		WithRegistryMirror("http://index.docker.io", false, false).
 		WithTest("http://index.docker.io/v2/blobs/sha256/xxx", false, false, "").
 		TestMirror(t)
 
 	newTestCase().
-		WithRegistryMirror("http://index.docker.io", true).
+		WithRegistryMirror("http://index.docker.io", true, false).
 		WithTest("http://index.docker.io/v2/blobs/sha256/xxx", true, false, "").
 		TestMirror(t)
 }

client/daemon/proxy/reverse_proxy.go

@@ -0,0 +1,98 @@
+/*
+ *     Copyright 2020 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package proxy
+
+import (
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"strings"
+
+	"d7y.io/dragonfly/v2/client/config"
+	logger "d7y.io/dragonfly/v2/internal/dflog"
+)
+
+func newReverseProxy(mirror *config.RegistryMirror) *httputil.ReverseProxy {
+	reverseProxy := httputil.NewSingleHostReverseProxy(mirror.Remote.URL)
+	if mirror.DynamicRemote {
+		reverseProxy.Director = newDynamicDirector(mirror.Remote.URL)
+	}
+	return reverseProxy
+}
+
+func newDynamicDirector(remote *url.URL) func(*http.Request) {
+	director := func(req *http.Request) {
+		var target = remote
+		targetQuery := target.RawQuery
+		reg := req.Header.Get(config.HeaderDragonflyRegistry)
+		if len(reg) > 0 {
+			regURL, err := url.Parse(reg)
+			if err == nil {
+				logger.Debugf("dynamic host url: %s", reg)
+				target = regURL
+			}
+		}
+		req.URL.Scheme = target.Scheme
+		req.URL.Host = target.Host
+		req.URL.Path, req.URL.RawPath = joinURLPath(target, req.URL)
+		if targetQuery == "" || req.URL.RawQuery == "" {
+			req.URL.RawQuery = targetQuery + req.URL.RawQuery
+		} else {
+			req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery
+		}
+		if _, ok := req.Header["User-Agent"]; !ok {
+			// explicitly disable User-Agent so it's not set to default value
+			req.Header.Set("User-Agent", "")
+		}
+	}
+	return director
+}
+
+// singleJoiningSlash is from net/http/httputil/reverseproxy.go
+func singleJoiningSlash(a, b string) string {
+	aslash := strings.HasSuffix(a, "/")
+	bslash := strings.HasPrefix(b, "/")
+	switch {
+	case aslash && bslash:
+		return a + b[1:]
+	case !aslash && !bslash:
+		return a + "/" + b
+	}
+	return a + b
+}
+
+// joinURLPath is from net/http/httputil/reverseproxy.go
+func joinURLPath(a, b *url.URL) (path, rawpath string) {
+	if a.RawPath == "" && b.RawPath == "" {
+		return singleJoiningSlash(a.Path, b.Path), ""
+	}
+	// Same as singleJoiningSlash, but uses EscapedPath to determine
+	// whether a slash should be added
+	apath := a.EscapedPath()
+	bpath := b.EscapedPath()
+
+	aslash := strings.HasSuffix(apath, "/")
+	bslash := strings.HasPrefix(bpath, "/")
+
+	switch {
+	case aslash && bslash:
+		return a.Path + b.Path[1:], apath + bpath[1:]
+	case !aslash && !bslash:
+		return a.Path + "/" + b.Path, apath + "/" + bpath
+	}
+	return a.Path + b.Path, apath + bpath
+}