Add middlewares for permission checking for v2 API

When the registry shifts from token auth to basic auth, we'll use the middleware to check permission.
This commit add middlewares for populate the artifact info and check
permission based on request to /v2/* api via security context

Signed-off-by: Daniel Jiang <jiangd@vmware.com>

src/core/middlewares/contenttrust/handler.go

@@ -62,13 +62,13 @@ func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Reque
 	util.CopyResp(rec, rw)
 }
 
-func validate(req *http.Request) (bool, util.ImageInfo) {
-	var img util.ImageInfo
-	imgRaw := req.Context().Value(util.ImageInfoCtxKey)
+func validate(req *http.Request) (bool, util.ArtifactInfo) {
+	var img util.ArtifactInfo
+	imgRaw := req.Context().Value(util.ArtifactInfoCtxKey)
 	if imgRaw == nil || !config.WithNotary() {
 		return false, img
 	}
-	img, _ = req.Context().Value(util.ImageInfoCtxKey).(util.ImageInfo)
+	img, _ = req.Context().Value(util.ArtifactInfoCtxKey).(util.ArtifactInfo)
 	if img.Digest == "" {
 		return false, img
 	}
@@ -81,7 +81,7 @@ func validate(req *http.Request) (bool, util.ImageInfo) {
 	return true, img
 }
 
-func matchNotaryDigest(img util.ImageInfo) (bool, error) {
+func matchNotaryDigest(img util.ArtifactInfo) (bool, error) {
 	if NotaryEndpoint == "" {
 		NotaryEndpoint = config.InternalNotaryEndpoint()
 	}

src/core/middlewares/contenttrust/handler_test.go

@@ -49,8 +49,8 @@ func TestMain(m *testing.M) {
 func TestMatchNotaryDigest(t *testing.T) {
 	assert := assert.New(t)
 	// The data from common/utils/notary/helper_test.go
-	img1 := util.ImageInfo{Repository: "notary-demo/busybox", Reference: "1.0", ProjectName: "notary-demo", Digest: "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"}
-	img2 := util.ImageInfo{Repository: "notary-demo/busybox", Reference: "2.0", ProjectName: "notary-demo", Digest: "sha256:12345678"}
+	img1 := util.ArtifactInfo{Repository: "notary-demo/busybox", Reference: "1.0", ProjectName: "notary-demo", Digest: "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"}
+	img2 := util.ArtifactInfo{Repository: "notary-demo/busybox", Reference: "2.0", ProjectName: "notary-demo", Digest: "sha256:12345678"}
 
 	res1, err := matchNotaryDigest(img1)
 	assert.Nil(err, "Unexpected error: %v, image: %#v", err, img1)

src/core/middlewares/regtoken/handler.go

@@ -27,12 +27,12 @@ func New(next http.Handler) http.Handler {
 
 // ServeHTTP ...
 func (r *regTokenHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
-	imgRaw := req.Context().Value(util.ImageInfoCtxKey)
+	imgRaw := req.Context().Value(util.ArtifactInfoCtxKey)
 	if imgRaw == nil {
 		r.next.ServeHTTP(rw, req)
 		return
 	}
-	img, _ := req.Context().Value(util.ImageInfoCtxKey).(util.ImageInfo)
+	img, _ := req.Context().Value(util.ArtifactInfoCtxKey).(util.ArtifactInfo)
 	if img.Digest == "" {
 		r.next.ServeHTTP(rw, req)
 		return

src/core/middlewares/url/handler.go

@@ -20,10 +20,19 @@ import (
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/core/middlewares/util"
 	coreutils "github.com/goharbor/harbor/src/core/utils"
+	"github.com/opencontainers/go-digest"
 	"net/http"
+	"regexp"
 	"strings"
 )
 
+var (
+	urlPatterns = []*regexp.Regexp{
+		util.ManifestURLRe, util.TagListURLRe, util.BlobURLRe, util.BlobUploadURLRe,
+	}
+)
+
+// urlHandler extracts the artifact info from the url of request to V2 handler and propagates it to context
 type urlHandler struct {
 	next http.Handler
 }
@@ -37,38 +46,65 @@ func New(next http.Handler) http.Handler {
 
 // ServeHTTP ...
 func (uh urlHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
-	log.Debugf("in url handler, path: %s", req.URL.Path)
-	flag, repository, reference := util.MatchPullManifest(req)
-	if flag {
-		components := strings.SplitN(repository, "/", 2)
-		if len(components) < 2 {
-			http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Bad repository name: %s", repository)), http.StatusBadRequest)
-			return
-		}
+	path := req.URL.Path
+	log.Debugf("in url handler, path: %s", path)
+	m, ok := parse(path)
+	if !ok {
+		uh.next.ServeHTTP(rw, req)
+	}
+	repo := m[util.RepositorySubexp]
+	components := strings.SplitN(repo, "/", 2)
+	if len(components) < 2 {
+		http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Bad repository name: %s", repo)), http.StatusBadRequest)
+		return
+	}
+	art := util.ArtifactInfo{
+		Repository:  repo,
+		ProjectName: components[0],
+	}
+	if digest, ok := m[util.DigestSubexp]; ok {
+		art.Digest = digest
+	}
+	if ref, ok := m[util.ReferenceSubexp]; ok {
+		art.Reference = ref
+	}
 
-		client, err := coreutils.NewRepositoryClientForUI(util.TokenUsername, repository)
+	if util.ManifestURLRe.MatchString(path) && req.Method == http.MethodGet { // Request for pulling manifest
+		client, err := coreutils.NewRepositoryClientForUI(util.TokenUsername, art.Repository)
 		if err != nil {
 			log.Errorf("Error creating repository Client: %v", err)
 			http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError)
 			return
 		}
-		digest, _, err := client.ManifestExist(reference)
+		digest, _, err := client.ManifestExist(art.Reference)
 		if err != nil {
-			log.Errorf("Failed to get digest for reference: %s, error: %v", reference, err)
+			log.Errorf("Failed to get digest for reference: %s, error: %v", art.Reference, err)
 			http.Error(rw, util.MarshalError("PROJECT_POLICY_VIOLATION", fmt.Sprintf("Failed due to internal Error: %v", err)), http.StatusInternalServerError)
 			return
 		}
 
-		img := util.ImageInfo{
-			Repository:  repository,
-			Reference:   reference,
-			ProjectName: components[0],
-			Digest:      digest,
-		}
-
-		log.Debugf("image info of the request: %#v", img)
-		ctx := context.WithValue(req.Context(), util.ImageInfoCtxKey, img)
+		art.Digest = digest
+		log.Debugf("artifact info of the request: %#v", art)
+		ctx := context.WithValue(req.Context(), util.ArtifactInfoCtxKey, art)
 		req = req.WithContext(ctx)
 	}
 	uh.next.ServeHTTP(rw, req)
 }
+
+func parse(urlPath string) (map[string]string, bool) {
+	m := make(map[string]string)
+	match := false
+	for _, re := range urlPatterns {
+		l := re.FindStringSubmatch(urlPath)
+		if len(l) > 0 {
+			match = true
+			for i := 1; i < len(l); i++ {
+				m[re.SubexpNames()[i]] = l[i]
+			}
+		}
+	}
+	if digest.DigestRegexp.MatchString(m[util.ReferenceSubexp]) {
+		m[util.DigestSubexp] = m[util.ReferenceSubexp]
+	}
+	return m, match
+}

src/core/middlewares/url/handler_test.go

@@ -0,0 +1,101 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
+package url
+
+import (
+	"github.com/goharbor/harbor/src/core/middlewares/util"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	if result := m.Run(); result != 0 {
+		os.Exit(result)
+	}
+}
+
+func TestParseURL(t *testing.T) {
+	cases := []struct {
+		input  string
+		expect map[string]string
+		match  bool
+	}{
+		{
+			input:  "/api/projects",
+			expect: map[string]string{},
+			match:  false,
+		},
+		{
+			input:  "/v2/_catalog",
+			expect: map[string]string{},
+			match:  false,
+		},
+		{
+			input: "/v2/no-project-repo/tags/list",
+			expect: map[string]string{
+				util.RepositorySubexp: "no-project-repo",
+			},
+			match: true,
+		},
+		{
+			input: "/v2/development/golang/manifests/sha256:08e4a417ff4e3913d8723a05cc34055db01c2fd165b588e049c5bad16ce6094f",
+			expect: map[string]string{
+				util.RepositorySubexp: "development/golang",
+				util.ReferenceSubexp:  "sha256:08e4a417ff4e3913d8723a05cc34055db01c2fd165b588e049c5bad16ce6094f",
+				util.DigestSubexp:     "sha256:08e4a417ff4e3913d8723a05cc34055db01c2fd165b588e049c5bad16ce6094f",
+			},
+			match: true,
+		},
+		{
+			input:  "/v2/development/golang/manifests/shaxxx:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+			expect: map[string]string{},
+			match:  false,
+		},
+		{
+			input: "/v2/multi/sector/repository/blobs/sha256:08e4a417ff4e3913d8723a05cc34055db01c2fd165b588e049c5bad16ce6094f",
+			expect: map[string]string{
+				util.RepositorySubexp: "multi/sector/repository",
+				util.DigestSubexp:     "sha256:08e4a417ff4e3913d8723a05cc34055db01c2fd165b588e049c5bad16ce6094f",
+			},
+			match: true,
+		},
+		{
+			input:  "/v2/blobs/uploads",
+			expect: map[string]string{},
+			match:  false,
+		},
+		{
+			input: "/v2/library/ubuntu/blobs/uploads",
+			expect: map[string]string{
+				util.RepositorySubexp: "library/ubuntu",
+			},
+			match: true,
+		},
+		{
+			input: "/v2/library/centos/blobs/uploads/u-12345",
+			expect: map[string]string{
+				util.RepositorySubexp: "library/centos",
+			},
+			match: true,
+		},
+	}
+
+	for _, c := range cases {
+		e, m := parse(c.input)
+		assert.Equal(t, c.match, m)
+		assert.Equal(t, c.expect, e)
+	}
+}

src/core/middlewares/util/util.go

@@ -31,6 +31,8 @@ import (
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/schema1"
 	"github.com/docker/distribution/manifest/schema2"
+	"github.com/docker/distribution/reference"
+
 	"github.com/garyburd/redigo/redis"
 	"github.com/goharbor/harbor/src/common/dao"
 	"github.com/goharbor/harbor/src/common/models"
@@ -49,8 +51,15 @@ import (
 type contextKey string
 
 const (
-	// ImageInfoCtxKey the context key for image information
-	ImageInfoCtxKey = contextKey("ImageInfo")
+	// RepositorySubexp is the name for sub regex that maps to repository name in the url
+	RepositorySubexp = "repository"
+	// ReferenceSubexp is the name for sub regex that maps to reference (tag or digest) url
+	ReferenceSubexp = "reference"
+	// DigestSubexp is the name for sub regex that maps to digest in the url
+	DigestSubexp = "digest"
+
+	// ArtifactInfoCtxKey the context key for artifact information
+	ArtifactInfoCtxKey = contextKey("ArtifactInfo")
 	// ScannerPullCtxKey the context key for robot account to bypass the pull policy check.
 	ScannerPullCtxKey = contextKey("ScannerPullCheck")
 	// TokenUsername ...
@@ -73,7 +82,16 @@ const (
 )
 
 var (
-	manifestURLRe = regexp.MustCompile(`^/v2/((?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)+)manifests/([\w][\w.:-]{0,127})`)
+	// ManifestURLRe is the regular expression for matching request v2 handler to view/delete manifest
+	ManifestURLRe = regexp.MustCompile(fmt.Sprintf(`^/v2/(?P<%s>%s)/manifests/(?P<%s>%s|%s)$`, RepositorySubexp, reference.NameRegexp.String(), ReferenceSubexp, reference.TagRegexp.String(), digest.DigestRegexp.String()))
+	// TagListURLRe is the regular expression for matching request to v2 handler to list tags
+	TagListURLRe = regexp.MustCompile(fmt.Sprintf(`^/v2/(?P<%s>%s)/tags/list`, RepositorySubexp, reference.NameRegexp.String()))
+	// BlobURLRe is the regular expression for matching request to v2 handler to retrieve delete a blob
+	BlobURLRe = regexp.MustCompile(fmt.Sprintf(`^/v2/(?P<%s>%s)/blobs/(?P<%s>%s)$`, RepositorySubexp, reference.NameRegexp.String(), DigestSubexp, digest.DigestRegexp.String()))
+	// BlobUploadURLRe is the regular expression for matching the request to v2 handler to upload a blob, the upload uuid currently is not put into a group
+	BlobUploadURLRe = regexp.MustCompile(fmt.Sprintf(`^/v2/(?P<%s>%s)/blobs/uploads[/a-zA-Z0-9\-_\.=]*$`, RepositorySubexp, reference.NameRegexp.String()))
+	// CatalogURLRe is the regular expression for mathing the request to v2 handler to list catalog
+	CatalogURLRe = regexp.MustCompile(`^/v2/_catalog$`)
 )
 
 // ChartVersionInfo ...
@@ -91,8 +109,8 @@ func (info *ChartVersionInfo) MutexKey(suffix ...string) string {
 	return strings.Join(append(a, suffix...), ":")
 }
 
-// ImageInfo ...
-type ImageInfo struct {
+// ArtifactInfo ...
+type ArtifactInfo struct {
 	Repository  string
 	Reference   string
 	ProjectName string
@@ -281,7 +299,7 @@ func MarshalError(code, msg string) string {
 
 // MatchManifestURL ...
 func MatchManifestURL(req *http.Request) (bool, string, string) {
-	s := manifestURLRe.FindStringSubmatch(req.URL.Path)
+	s := ManifestURLRe.FindStringSubmatch(req.URL.Path)
 	if len(s) == 3 {
 		s[1] = strings.TrimSuffix(s[1], "/")
 		return true, s[1], s[2]
@@ -437,8 +455,8 @@ func ChartVersionInfoFromContext(ctx context.Context) (*ChartVersionInfo, bool)
 }
 
 // ImageInfoFromContext returns image info from context
-func ImageInfoFromContext(ctx context.Context) (*ImageInfo, bool) {
-	info, ok := ctx.Value(ImageInfoCtxKey).(*ImageInfo)
+func ImageInfoFromContext(ctx context.Context) (*ArtifactInfo, bool) {
+	info, ok := ctx.Value(ArtifactInfoCtxKey).(*ArtifactInfo)
 	return info, ok
 }
 
@@ -470,8 +488,8 @@ func NewChartVersionInfoContext(ctx context.Context, info *ChartVersionInfo) con
 }
 
 // NewImageInfoContext returns context with image info
-func NewImageInfoContext(ctx context.Context, info *ImageInfo) context.Context {
-	return context.WithValue(ctx, ImageInfoCtxKey, info)
+func NewImageInfoContext(ctx context.Context, info *ArtifactInfo) context.Context {
+	return context.WithValue(ctx, ArtifactInfoCtxKey, info)
 }
 
 // NewManifestInfoContext returns context with manifest info

src/core/middlewares/vulnerable/handler.go

@@ -108,17 +108,17 @@ func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request)
 	util.CopyResp(rec, rw)
 }
 
-func validate(req *http.Request) (bool, util.ImageInfo, vuln.Severity, models.CVEWhitelist) {
+func validate(req *http.Request) (bool, util.ArtifactInfo, vuln.Severity, models.CVEWhitelist) {
 	var vs vuln.Severity
 	var wl models.CVEWhitelist
-	var img util.ImageInfo
-	imgRaw := req.Context().Value(util.ImageInfoCtxKey)
+	var img util.ArtifactInfo
+	imgRaw := req.Context().Value(util.ArtifactInfoCtxKey)
 	if imgRaw == nil {
 		return false, img, vs, wl
 	}
 
 	// Expected artifact specified?
-	img, ok := imgRaw.(util.ImageInfo)
+	img, ok := imgRaw.(util.ArtifactInfo)
 	if !ok || len(img.Digest) == 0 {
 		return false, img, vs, wl
 	}