Merge pull request #10742 from bitsf/oci_tag_retention

requirement(oci) implement tag retention for oci
not include ChartClient yet

src/api/artifact/controller.go

@@ -588,7 +588,7 @@ func (c *controller) populateImmutableStatus(ctx context.Context, tag *Tag) {
 	_, repoName := utils.ParseRepository(repo.Name)
 	matched, err := c.immutableMtr.Match(repo.ProjectID, art.Candidate{
 		Repository:  repoName,
-		Tag:         tag.Name,
+		Tags:        []string{tag.Name},
 		NamespaceID: repo.ProjectID,
 	})
 	if err != nil {

src/core/api/repository.go

@@ -827,7 +827,7 @@ func isImmutable(projectID int64, repo string, tag string) bool {
 	_, repoName := utils.ParseRepository(repo)
 	matched, err := rule.NewRuleMatcher().Match(projectID, art.Candidate{
 		Repository:  repoName,
-		Tag:         tag,
+		Tags:        []string{tag},
 		NamespaceID: projectID,
 	})
 	if err != nil {

src/core/middlewares/interceptor/immutable/deletemf.go

@@ -47,7 +47,7 @@ func (dmf *delmfInterceptor) HandleRequest(req *http.Request) (err error) {
 		var matched bool
 		matched, err = rule.NewRuleMatcher().Match(dmf.mf.ProjectID, art.Candidate{
 			Repository:  repoName,
-			Tag:         af.Tag,
+			Tags:        []string{af.Tag},
 			NamespaceID: dmf.mf.ProjectID,
 		})
 		if err != nil {

src/core/middlewares/interceptor/immutable/pushmf.go

@@ -31,7 +31,7 @@ func (pmf *pushmfInterceptor) HandleRequest(req *http.Request) (err error) {
 	var matched bool
 	matched, err = rule.NewRuleMatcher().Match(pmf.mf.ProjectID, art.Candidate{
 		Repository:  repoName,
-		Tag:         pmf.mf.Tag,
+		Tags:        []string{pmf.mf.Tag},
 		NamespaceID: pmf.mf.ProjectID,
 	})
 	if err != nil {

src/pkg/art/candidate.go

@@ -73,8 +73,8 @@ type Candidate struct {
 	// Kind of the candidate
 	// "image" or "chart"
 	Kind string
-	// Tag info
-	Tag string
+	// Tags attached with the candidate
+	Tags []string
 	// Digest
 	Digest string
 	// Pushed time in seconds
@@ -90,16 +90,9 @@ type Candidate struct {
 // Hash code based on the candidate info for differentiation
 func (c *Candidate) Hash() string {
 	if c.Digest == "" {
-		log.Errorf("Lack Digest of Candidate for %s/%s:%s", c.Namespace, c.Repository, c.Tag)
+		log.Errorf("Lack Digest of Candidate for %s/%s", c.Namespace, c.Repository)
 	}
 	raw := fmt.Sprintf("%s:%s/%s:%s", c.Kind, c.Namespace, c.Repository, c.Digest)
 
 	return base64.StdEncoding.EncodeToString([]byte(raw))
 }
-
-// NameHash based on the candidate info for differentiation
-func (c *Candidate) NameHash() string {
-	raw := fmt.Sprintf("%s:%s/%s:%s", c.Kind, c.Namespace, c.Repository, c.Tag)
-
-	return base64.StdEncoding.EncodeToString([]byte(raw))
-}

src/pkg/art/result.go

@@ -23,12 +23,8 @@ type Result struct {
 
 // ImmutableError ...
 type ImmutableError struct {
-	IsShareDigest bool
 }
 
 func (e *ImmutableError) Error() string {
-	if e.IsShareDigest {
-		return "Same digest with other immutable tag"
-	}
 	return "Immutable tag"
 }

src/pkg/art/selectors/doublestar/selector.go

@@ -53,10 +53,21 @@ func (s *selector) Select(artifacts []*art.Candidate) (selected []*art.Candidate
 	for _, art := range artifacts {
 		switch s.decoration {
 		case Matches:
-			value = art.Tag
+			s, err := s.tagSelectMatch(art)
+			if err != nil {
+				return nil, err
+			}
+			if s {
+				selected = append(selected, art)
+			}
 		case Excludes:
-			value = art.Tag
-			excludes = true
+			s, err := s.tagSelectExclude(art)
+			if err != nil {
+				return nil, err
+			}
+			if s {
+				selected = append(selected, art)
+			}
 		case RepoMatches:
 			value = art.Repository
 		case RepoExcludes:
@@ -85,6 +96,32 @@ func (s *selector) Select(artifacts []*art.Candidate) (selected []*art.Candidate
 	return selected, nil
 }
 
+func (s *selector) tagSelectMatch(artifact *art.Candidate) (selected bool, err error) {
+	for _, t := range artifact.Tags {
+		matched, err := match(s.pattern, t)
+		if err != nil {
+			return false, err
+		}
+		if matched {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func (s *selector) tagSelectExclude(artifact *art.Candidate) (selected bool, err error) {
+	for _, t := range artifact.Tags {
+		matched, err := match(s.pattern, t)
+		if err != nil {
+			return false, err
+		}
+		if !matched {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
 // New is factory method for doublestar selector
 func New(decoration string, pattern string) art.Selector {
 	return &selector{

src/pkg/art/selectors/doublestar/selector_test.go

@@ -43,7 +43,7 @@ func (suite *RegExpSelectorTestSuite) SetupSuite() {
 			NamespaceID:  1,
 			Namespace:    "library",
 			Repository:   "harbor",
-			Tag:          "latest",
+			Tags:         []string{"latest"},
 			Kind:         art.Image,
 			PushedTime:   time.Now().Unix() - 3600,
 			PulledTime:   time.Now().Unix(),
@@ -54,7 +54,7 @@ func (suite *RegExpSelectorTestSuite) SetupSuite() {
 			NamespaceID:  2,
 			Namespace:    "retention",
 			Repository:   "redis",
-			Tag:          "4.0",
+			Tags:         []string{"4.0"},
 			Kind:         art.Image,
 			PushedTime:   time.Now().Unix() - 3600,
 			PulledTime:   time.Now().Unix(),
@@ -65,7 +65,7 @@ func (suite *RegExpSelectorTestSuite) SetupSuite() {
 			NamespaceID:  2,
 			Namespace:    "retention",
 			Repository:   "redis",
-			Tag:          "4.1",
+			Tags:         []string{"4.1"},
 			Kind:         art.Image,
 			PushedTime:   time.Now().Unix() - 3600,
 			PulledTime:   time.Now().Unix(),
@@ -239,7 +239,9 @@ func expect(expected []string, candidates []*art.Candidate) bool {
 	hash := make(map[string]bool)
 
 	for _, art := range candidates {
-		hash[fmt.Sprintf("%s:%s", art.Repository, art.Tag)] = true
+		for _, t := range art.Tags {
+			hash[fmt.Sprintf("%s:%s", art.Repository, t)] = true
+		}
 	}
 
 	for _, exp := range expected {

src/pkg/art/selectors/label/selector_test.go

@@ -43,7 +43,7 @@ func (suite *LabelSelectorTestSuite) SetupSuite() {
 			NamespaceID:  1,
 			Namespace:    "library",
 			Repository:   "harbor",
-			Tag:          "1.9",
+			Tags:         []string{"1.9"},
 			Kind:         art.Image,
 			PushedTime:   time.Now().Unix() - 3600,
 			PulledTime:   time.Now().Unix(),
@@ -54,7 +54,7 @@ func (suite *LabelSelectorTestSuite) SetupSuite() {
 			NamespaceID:  1,
 			Namespace:    "library",
 			Repository:   "harbor",
-			Tag:          "dev",
+			Tags:         []string{"dev"},
 			Kind:         art.Image,
 			PushedTime:   time.Now().Unix() - 3600,
 			PulledTime:   time.Now().Unix(),
@@ -135,7 +135,9 @@ func expect(expected []string, candidates []*art.Candidate) bool {
 	hash := make(map[string]bool)
 
 	for _, art := range candidates {
-		hash[fmt.Sprintf("%s:%s", art.Repository, art.Tag)] = true
+		for _, t := range art.Tags {
+			hash[fmt.Sprintf("%s:%s", art.Repository, t)] = true
+		}
 	}
 
 	for _, exp := range expected {

src/pkg/clients/core/client.go

@@ -16,10 +16,9 @@ package core
 
 import (
 	"fmt"
+	modelsv2 "github.com/goharbor/harbor/src/api/artifact"
 	"net/http"
 
-	"github.com/goharbor/harbor/src/common/models"
-
 	"github.com/goharbor/harbor/src/chartserver"
 	chttp "github.com/goharbor/harbor/src/common/http"
 	"github.com/goharbor/harbor/src/common/http/modifier"
@@ -29,15 +28,15 @@ import (
 // Currently, it contains only part of the whole method collection
 // and we should expand it when needed
 type Client interface {
-	ImageClient
+	ArtifactClient
 	ChartClient
 }
 
-// ImageClient defines the methods that an image client should implement
-type ImageClient interface {
-	ListAllImages(project, repository string) ([]*models.TagResp, error)
-	DeleteImage(project, repository, tag string) error
-	DeleteImageRepository(project, repository string) error
+// ArtifactClient defines the methods that an image client should implement
+type ArtifactClient interface {
+	ListAllArtifacts(project, repository string) ([]*modelsv2.Artifact, error)
+	DeleteArtifact(project, repository, digest string) error
+	DeleteArtifactRepository(project, repository string) error
 }
 
 // ChartClient defines the methods that a chart client should implement
@@ -61,5 +60,5 @@ type client struct {
 }
 
 func (c *client) buildURL(path string) string {
-	return fmt.Sprintf("%s/%s", c.url, path)
+	return fmt.Sprintf("%s%s", c.url, path)
 }

src/pkg/clients/core/image.go

@@ -16,25 +16,25 @@ package core
 
 import (
 	"fmt"
-
-	"github.com/goharbor/harbor/src/common/models"
+	modelsv2 "github.com/goharbor/harbor/src/api/artifact"
 )
 
-func (c *client) ListAllImages(project, repository string) ([]*models.TagResp, error) {
-	url := c.buildURL(fmt.Sprintf("/api/repositories/%s/%s/tags", project, repository))
-	var images []*models.TagResp
-	if err := c.httpclient.GetAndIteratePagination(url, &images); err != nil {
+func (c *client) ListAllArtifacts(project, repository string) ([]*modelsv2.Artifact, error) {
+	url := c.buildURL(fmt.Sprintf("/api/v2.0/projects/%s/repositories/%s/artifacts", project, repository)) // should query only tag
+	var arts []*modelsv2.Artifact
+	if err := c.httpclient.GetAndIteratePagination(url, &arts); err != nil {
 		return nil, err
 	}
-	return images, nil
+	return arts, nil
 }
 
-func (c *client) DeleteImage(project, repository, tag string) error {
-	url := c.buildURL(fmt.Sprintf("/api/repositories/%s/%s/tags/%s", project, repository, tag))
+func (c *client) DeleteArtifact(project, repository, digest string) error {
+	// /projects/{project_name}/repositories/{repository_name}/artifacts/{reference}
+	url := c.buildURL(fmt.Sprintf("/api/v2.0/projects/%s/repositories/%s/artifacts/%s", project, repository, digest))
 	return c.httpclient.Delete(url)
 }
 
-func (c *client) DeleteImageRepository(project, repository string) error {
+func (c *client) DeleteArtifactRepository(project, repository string) error {
 	url := c.buildURL(fmt.Sprintf("/api/repositories/%s/%s", project, repository))
 	return c.httpclient.Delete(url)
 }

src/pkg/immutabletag/match/rule/match_test.go

@@ -91,7 +91,7 @@ func (s *MatchTestSuite) TestImmuMatch() {
 		NamespaceID: 1,
 		Namespace:   "library",
 		Repository:  "redis",
-		Tag:         "release-1.10",
+		Tags:        []string{"release-1.10"},
 	}
 	isMatch, err := match.Match(1, c1)
 	s.require.Equal(isMatch, true)
@@ -101,7 +101,7 @@ func (s *MatchTestSuite) TestImmuMatch() {
 		NamespaceID: 1,
 		Namespace:   "library",
 		Repository:  "redis",
-		Tag:         "1.10",
+		Tags:        []string{"1.10"},
 		Kind:        art.Image,
 	}
 	isMatch, err = match.Match(1, c2)
@@ -112,7 +112,7 @@ func (s *MatchTestSuite) TestImmuMatch() {
 		NamespaceID: 1,
 		Namespace:   "immutable",
 		Repository:  "mysql",
-		Tag:         "9.4.8",
+		Tags:        []string{"9.4.8"},
 		Kind:        art.Image,
 	}
 	isMatch, err = match.Match(1, c3)
@@ -123,7 +123,7 @@ func (s *MatchTestSuite) TestImmuMatch() {
 		NamespaceID: 1,
 		Namespace:   "immutable",
 		Repository:  "hello",
-		Tag:         "world",
+		Tags:        []string{"world"},
 		Kind:        art.Image,
 	}
 	isMatch, err = match.Match(1, c4)

src/pkg/retention/dep/client.go

@@ -18,6 +18,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/goharbor/harbor/src/common/http/modifier/auth"
 	"github.com/goharbor/harbor/src/jobservice/config"
@@ -95,26 +96,41 @@ func (bc *basicClient) GetCandidates(repository *art.Repository) ([]*art.Candida
 	candidates := make([]*art.Candidate, 0)
 	switch repository.Kind {
 	case art.Image:
-		images, err := bc.coreClient.ListAllImages(repository.Namespace, repository.Name)
+		artifacts, err := bc.coreClient.ListAllArtifacts(repository.Namespace, repository.Name)
 		if err != nil {
 			return nil, err
 		}
-		for _, image := range images {
+		for _, artifact := range artifacts {
+			if artifact.Digest == "" {
+				return nil, fmt.Errorf("Lack Digest of Candidate for %s/%s", repository.Namespace, repository.Name)
+			}
 			labels := make([]string, 0)
-			for _, label := range image.Labels {
+			for _, label := range artifact.Labels {
 				labels = append(labels, label.Name)
 			}
+			tags := make([]string, 0)
+			var lastPulledTime time.Time
+			var lastPushedTime time.Time
+			for _, t := range artifact.Tags {
+				tags = append(tags, t.Name)
+				if t.PullTime.After(lastPulledTime) {
+					lastPulledTime = t.PullTime
+				}
+				if t.PushTime.After(lastPushedTime) {
+					lastPushedTime = t.PushTime
+				}
+			}
 			candidate := &art.Candidate{
 				Kind:         art.Image,
 				NamespaceID:  repository.NamespaceID,
 				Namespace:    repository.Namespace,
 				Repository:   repository.Name,
-				Tag:          image.Name,
-				Digest:       image.Digest,
+				Tags:         tags,
+				Digest:       artifact.Digest,
 				Labels:       labels,
-				CreationTime: image.Created.Unix(),
-				PulledTime:   image.PullTime.Unix(),
-				PushedTime:   image.PushTime.Unix(),
+				CreationTime: artifact.PushTime.Unix(),
+				PulledTime:   lastPulledTime.Unix(),
+				PushedTime:   lastPushedTime.Unix(),
 			}
 			candidates = append(candidates, candidate)
 		}
@@ -155,7 +171,7 @@ func (bc *basicClient) DeleteRepository(repo *art.Repository) error {
 	}
 	switch repo.Kind {
 	case art.Image:
-		return bc.coreClient.DeleteImageRepository(repo.Namespace, repo.Name)
+		return bc.coreClient.DeleteArtifactRepository(repo.Namespace, repo.Name)
 	/*
 		case art.Chart:
 			return bc.coreClient.DeleteChartRepository(repo.Namespace, repo.Name)
@@ -172,7 +188,7 @@ func (bc *basicClient) Delete(candidate *art.Candidate) error {
 	}
 	switch candidate.Kind {
 	case art.Image:
-		return bc.coreClient.DeleteImage(candidate.Namespace, candidate.Repository, candidate.Tag)
+		return bc.coreClient.DeleteArtifact(candidate.Namespace, candidate.Repository, candidate.Digest)
 	/*
 		case art.Chart:
 			return bc.coreClient.DeleteChart(candidate.Namespace, candidate.Repository, candidate.Tag)