feat(registry): add harbor chart proxy and inject tenant (#887)

1, add harbor chart upload and download proxy
2, add tke tenant for harbor registry and chart

Co-authored-by: kencchen <kencchen@tencent.com>

pkg/gateway/proxy/proxy.go

@@ -121,13 +121,17 @@ func componentPrefix() map[moduleName][]modulePath {
 				protected: false,
 			},
 			modulePath{
-				prefix:    harbor.PathPrefix,
+				prefix:    harbor.RegistryPrefix,
 				protected: false,
 			},
 			modulePath{
 				prefix:    harbor.AuthPrefix,
 				protected: false,
 			},
+			modulePath{
+				prefix:    harbor.ChartPrefix,
+				protected: false,
+			},
 			modulePath{
 				prefix:    fmt.Sprintf("%s/%s/", apiPrefix, registry.GroupName),
 				protected: true,

pkg/registry/apis/config/v1/types.go

@@ -36,7 +36,7 @@ type RegistryConfiguration struct {
 	// +optional
 	DomainSuffix string `json:"domainSuffix,omitempty"`
 	HarborEnabled bool  `json:"harborEnabled,omitempty"`
-	HarborCAFile string  `json:"string,omitempty"`
+	HarborCAFile string  `json:"harborCAFile,omitempty"`
 }
 
 type Storage struct {

pkg/registry/apiserver/apiserver.go

@@ -119,21 +119,20 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
 		if err := distribution.RegisterRoute(s.Handler.NonGoRestfulMux, distributionOpts); err != nil {
 			return nil, err
 		}
-	}
-
-
-	chartmuseumOpts := &chartmuseum.Options{
-		RegistryConfig:       c.ExtraConfig.RegistryConfig,
-		LoopbackClientConfig: c.GenericConfig.LoopbackClientConfig,
-		OIDCCAFile:           c.ExtraConfig.OIDCCAFile,
-		OIDCTokenReviewPath:  c.ExtraConfig.OIDCTokenReviewPath,
-		OIDCIssuerURL:        c.ExtraConfig.OIDCIssuerURL,
-		ExternalScheme:       c.ExtraConfig.ExternalScheme,
-		Authorizer:           c.GenericConfig.Authorization.Authorizer,
-	}
-	if err := chartmuseum.RegisterRoute(s.Handler.NonGoRestfulMux, chartmuseumOpts); err != nil {
-		return nil, err
-	}
+
+		chartmuseumOpts := &chartmuseum.Options{
+			RegistryConfig:       c.ExtraConfig.RegistryConfig,
+			LoopbackClientConfig: c.GenericConfig.LoopbackClientConfig,
+			OIDCCAFile:           c.ExtraConfig.OIDCCAFile,
+			OIDCTokenReviewPath:  c.ExtraConfig.OIDCTokenReviewPath,
+			OIDCIssuerURL:        c.ExtraConfig.OIDCIssuerURL,
+			ExternalScheme:       c.ExtraConfig.ExternalScheme,
+			Authorizer:           c.GenericConfig.Authorization.Authorizer,
+		}
+		if err := chartmuseum.RegisterRoute(s.Handler.NonGoRestfulMux, chartmuseumOpts); err != nil {
+			return nil, err
+		}
+	}	
 
 	// The order here is preserved in discovery.
 	restStorageProviders := []storage.RESTStorageProvider{

pkg/registry/harbor/harbor.go

@@ -23,6 +23,7 @@ import (
 	registryconfig "tkestack.io/tke/pkg/registry/apis/config"
 
 	"tkestack.io/tke/pkg/registry/harbor/handler"
+	"tkestack.io/tke/pkg/registry/harbor/tenant"
 
 	// import filesystem driver to store images
 	_ "github.com/docker/distribution/registry/storage/driver/filesystem"
@@ -34,9 +35,11 @@ import (
 	_ "tkestack.io/tke/pkg/registry/distribution/auth/token"
 )
 
-// PathPrefix defines the path prefix for accessing the docker registry v2 server.
-const PathPrefix = "/v2/"
+// RegistryPrefix defines the path prefix for accessing the docker registry v2 server.
+// ChartPrefix defines the path prefix for accessing the helm chart server
+const RegistryPrefix = "/v2/"
 const AuthPrefix = "/service/"
+const ChartPrefix = "/chartrepo/"
 
 type Options struct {
 	RegistryConfig *registryconfig.RegistryConfiguration
@@ -47,21 +50,27 @@ type Options struct {
 // go through the built-in authentication and authorization middleware of apiserver.
 func IgnoreAuthPathPrefixes() []string {
 	return []string{
-		PathPrefix,
+		RegistryPrefix,
 		AuthPrefix,
+		ChartPrefix,
 	}
 }
 
 // RegisterRoute to register the docker distribution server path prefix to apiserver.
 func RegisterRoute(m *mux.PathRecorderMux, opts *Options) error {
 
-	handler, err := handler.NewHandler("https://"+opts.RegistryConfig.DomainSuffix, opts.RegistryConfig.HarborCAFile, opts.ExternalHost)
+	httpURL := "https://" + opts.RegistryConfig.DomainSuffix
+
+	harborHandler, err := handler.NewHandler(httpURL, opts.RegistryConfig.HarborCAFile, opts.ExternalHost)
 	if err != nil {
 		return err
 	}
 
-	m.HandlePrefix(PathPrefix, handler)
-	m.HandlePrefix(AuthPrefix, handler)
+	wrappedHandler := tenant.WithTenant(harborHandler, RegistryPrefix, AuthPrefix, ChartPrefix, opts.RegistryConfig.DomainSuffix, opts.RegistryConfig.DefaultTenant)
+
+	m.HandlePrefix(RegistryPrefix, wrappedHandler)
+	m.HandlePrefix(AuthPrefix, wrappedHandler)
+	m.HandlePrefix(ChartPrefix, wrappedHandler)
 
 	return nil
 }

pkg/registry/harbor/tenant/tenant.go

@@ -0,0 +1,70 @@
+/*
+ * Tencent is pleased to support the open source community by making TKEStack
+ * available.
+ *
+ * Copyright (C) 2012-2019 Tencent. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+ * this file except in compliance with the License. You may obtain a copy of the
+ * License at
+ *
+ * https://opensource.org/licenses/Apache-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OF ANY KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tenant
+
+import (
+	"fmt"
+	"github.com/docker/distribution/registry/api/v2"
+	"github.com/gorilla/mux"
+	"net/http"
+	"strings"
+
+	utilregistryrequest "tkestack.io/tke/pkg/registry/util/request"
+)
+
+const CrossTenantNamespace = "library"
+
+// WithTenant adds an interceptor to the original http request handle and
+// converts the request for docker distribution to multi-tenant mode.
+func WithTenant(handler http.Handler, registryPrefix, authPrefix, chartPrefix, domainSuffix, defaultTenant string) http.Handler {
+	router := v2.Router()
+	return &tenant{handler, router, registryPrefix, authPrefix, chartPrefix, domainSuffix, defaultTenant}
+}
+
+type tenant struct {
+	handler        http.Handler
+	router         *mux.Router
+	registryPrefix string
+	authPrefix     string
+	chartPrefix    string
+	domainSuffix   string
+	defaultTenant  string
+}
+
+func (t *tenant) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	originalPath := r.URL.Path
+	tenant := utilregistryrequest.TenantID(r, t.domainSuffix, t.defaultTenant)
+	if strings.HasPrefix(originalPath, t.registryPrefix) && !strings.HasPrefix(originalPath, fmt.Sprintf("/v2/%s/", CrossTenantNamespace)) {
+		var match mux.RouteMatch
+		if matched := t.router.Match(r, &match); matched {
+			routeName := match.Route.GetName()
+			if routeName == v2.RouteNameManifest ||
+				routeName == v2.RouteNameTags ||
+				routeName == v2.RouteNameBlob ||
+				routeName == v2.RouteNameBlobUpload {
+				r.URL.Path = strings.Replace(originalPath, t.registryPrefix, fmt.Sprintf("%s%s-image-", t.registryPrefix, tenant), 1)
+			}
+		}
+	} else if strings.HasPrefix(originalPath, t.chartPrefix) {
+		r.URL.Path = strings.Replace(originalPath, t.chartPrefix, fmt.Sprintf("%s%s-chart-", t.chartPrefix, tenant), 1)
+	} else if strings.HasPrefix(originalPath, t.authPrefix) {
+		r.URL.RawQuery = strings.Replace(r.URL.RawQuery, "repository%3A", "repository%3A"+tenant+"-image-", -1)
+	}
+	t.handler.ServeHTTP(w, r)
+}