From bcd2dd556ae191ccaad3e2349831b81df5ef76be Mon Sep 17 00:00:00 2001 From: Nathan Lowe Date: Sat, 12 Jan 2019 17:09:42 -0500 Subject: [PATCH 1/4] Initial boilerplate for tag retention policies Signed-off-by: Nathan Lowe --- src/common/models/retention.go | 107 ++++++++++++++++++++++++++++++++ src/core/api/repository.go | 26 ++++---- src/core/api/repository_test.go | 8 +-- 3 files changed, 124 insertions(+), 17 deletions(-) create mode 100644 src/common/models/retention.go diff --git a/src/common/models/retention.go b/src/common/models/retention.go new file mode 100644 index 00000000000..05df3777910 --- /dev/null +++ b/src/common/models/retention.go @@ -0,0 +1,107 @@ +// Copyright 2019 Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "time" + + "github.com/goharbor/harbor/src/core/api" +) + +// TagRecord represents all pertinent metadata about a tag +type TagRecord struct { + Project *Project + Repository *RepoRecord + Tag *api.TagResp +} + +// TagAction records when a filter takes an action upon a tag +type TagAction struct { + // The tag the action applies to + Target *TagRecord + // The filter that took this action + ActingFilter Filter +} + +// Filter is a tag filter in a Retention Policy Filter Chain +type Filter interface { + // Process takes tags from the input channel and writes them to one of the three output channels. + // Tags are written to toKeep if the tags should be explicitly kept by the Filter + // Tags are written to toDelete if the tags should be explicitly deleted by the Filter + // Tags are written to next if the retention policy does not apply to the provided tag + // or if the policy does not care if the tag is kept or deleted + // + // Filters do not own any of the provided channels and should **not** close them under any circumstance + Process(input <-chan *TagRecord, toKeep, toDelete chan<- *TagAction, next chan<- *TagRecord) error + + // InitializeFor re-initializes the filter for tags from the specified project and repository + // + // Filters that maintain per-project or per-repository tracking metadata should reset it when + // this method is called. + InitializeFor(project *Project, repo *RepoRecord) +} + +// FilterMetadata defines the metadata needed to construct various Filter instances +type FilterMetadata struct { + ID int64 + // The type of the filter to construct + Type string + // Parameters used to construct the filter + Options map[string]interface{} +} + +// RetentionScope identifies the scope of a specific retention policy +type RetentionScope int + +const ( + // RetentionScopeServer identifies a retention policy defined server-wide + RetentionScopeServer RetentionScope = iota + // RetentionScopeProject identifies a retention policy defined for a specific project + RetentionScopeProject + // RetentionScopeRepository identifies a retention policy defined for a specific repository + RetentionScopeRepository +) + +// FallThroughAction determines what action the policy should take when a tag has not +// been explicitly kept nor explicitly deleted by all filters in the filter chain +type FallThroughAction int + +const ( + // KeepExtraTags indicates that tags which are not explicitly kept or deleted are implicitly kept + KeepExtraTags FallThroughAction = iota + // DeleteExtraTags indicates that tags which are not explicitly kept or deleted are implicitly deleted + DeleteExtraTags +) + +// RetentionPolicy contains an ordered slice of FilterMetadata used to construct filter chains +// during tag retention procession +type RetentionPolicy struct { + ID int64 + Name string + Enabled bool + + Scope RetentionScope + FallThroughAction FallThroughAction + + ProjectID int64 + RepositoryID int64 + + // When a filter chain is constructed for this policy, these filters will + // be chained together in the order they appear in the slice + Filters []*FilterMetadata + + CreatedAt time.Time + UpdatedAt time.Time +} diff --git a/src/core/api/repository.go b/src/core/api/repository.go index 12659b48c3c..a33e51ebbae 100644 --- a/src/core/api/repository.go +++ b/src/core/api/repository.go @@ -78,7 +78,7 @@ func (r reposSorter) Less(i, j int) bool { return r[i].Index < r[j].Index } -type tagDetail struct { +type TagDetail struct { Digest string `json:"digest"` Name string `json:"name"` Size int64 `json:"size"` @@ -95,8 +95,8 @@ type cfg struct { Labels map[string]string `json:"labels"` } -type tagResp struct { - tagDetail +type TagResp struct { + TagDetail Signature *notary.Target `json:"signature"` ScanOverview *models.ImgScanOverview `json:"scan_overview,omitempty"` Labels []*models.Label `json:"labels"` @@ -607,7 +607,7 @@ func (ra *RepositoryAPI) GetTags() { // get config, signature and scan overview and assemble them into one // struct for each tag in tags func assembleTagsInParallel(client *registry.Repository, repository string, - tags []string, username string) []*tagResp { + tags []string, username string) []*TagResp { var err error signatures := map[string][]notary.Target{} if config.WithNotary() { @@ -618,13 +618,13 @@ func assembleTagsInParallel(client *registry.Repository, repository string, } } - c := make(chan *tagResp) + c := make(chan *TagResp) for _, tag := range tags { go assembleTag(c, client, repository, tag, config.WithClair(), config.WithNotary(), signatures) } - result := []*tagResp{} - var item *tagResp + result := []*TagResp{} + var item *TagResp for i := 0; i < len(tags); i++ { item = <-c if item == nil { @@ -635,10 +635,10 @@ func assembleTagsInParallel(client *registry.Repository, repository string, return result } -func assembleTag(c chan *tagResp, client *registry.Repository, +func assembleTag(c chan *TagResp, client *registry.Repository, repository, tag string, clairEnabled, notaryEnabled bool, signatures map[string][]notary.Target) { - item := &tagResp{} + item := &TagResp{} // labels image := fmt.Sprintf("%s:%s", repository, tag) labels, err := dao.GetLabelsOfResource(common.ResourceTypeImage, image) @@ -654,7 +654,7 @@ func assembleTag(c chan *tagResp, client *registry.Repository, log.Errorf("failed to get v2 manifest of %s:%s: %v", repository, tag, err) } if tagDetail != nil { - item.tagDetail = *tagDetail + item.TagDetail = *tagDetail } // scan overview @@ -677,8 +677,8 @@ func assembleTag(c chan *tagResp, client *registry.Repository, // getTagDetail returns the detail information for v2 manifest image // The information contains architecture, os, author, size, etc. -func getTagDetail(client *registry.Repository, tag string) (*tagDetail, error) { - detail := &tagDetail{ +func getTagDetail(client *registry.Repository, tag string) (*TagDetail, error) { + detail := &TagDetail{ Name: tag, } @@ -718,7 +718,7 @@ func getTagDetail(client *registry.Repository, tag string) (*tagDetail, error) { return detail, nil } -func populateAuthor(detail *tagDetail) { +func populateAuthor(detail *TagDetail) { // has author info already if len(detail.Author) > 0 { return diff --git a/src/core/api/repository_test.go b/src/core/api/repository_test.go index 34649d245f1..68d7bf4230c 100644 --- a/src/core/api/repository_test.go +++ b/src/core/api/repository_test.go @@ -96,7 +96,7 @@ func TestGetReposTags(t *testing.T) { t.Errorf("failed to get tags of repository %s: %v", repository, err) } else { assert.Equal(int(200), code, "httpStatusCode should be 200") - if tg, ok := tags.([]tagResp); ok { + if tg, ok := tags.([]TagResp); ok { assert.Equal(1, len(tg), fmt.Sprintf("there should be only one tag, but now %v", tg)) assert.Equal(tg[0].Name, "latest", "the tag should be latest") } else { @@ -207,18 +207,18 @@ func TestGetReposTop(t *testing.T) { func TestPopulateAuthor(t *testing.T) { author := "author" - detail := &tagDetail{ + detail := &TagDetail{ Author: author, } populateAuthor(detail) assert.Equal(t, author, detail.Author) - detail = &tagDetail{} + detail = &TagDetail{} populateAuthor(detail) assert.Equal(t, "", detail.Author) maintainer := "maintainer" - detail = &tagDetail{ + detail = &TagDetail{ Config: &cfg{ Labels: map[string]string{ "Maintainer": maintainer, From c0a7ce24bd02a613e5efa3e819eec8c188e9a67c Mon Sep 17 00:00:00 2001 From: Nathan Lowe Date: Sun, 20 Jan 2019 12:30:57 -0500 Subject: [PATCH 2/4] Fix import-cycle Instead of pulling in an api.TagResp, just include the fields we'll need for the initial filter implementations. We'll figure out how to wire them up later. Signed-off-by: Nathan Lowe --- src/common/models/retention.go | 8 +++++--- src/core/api/repository.go | 26 +++++++++++++------------- src/core/api/repository_test.go | 8 ++++---- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/common/models/retention.go b/src/common/models/retention.go index 05df3777910..3f8882f51e5 100644 --- a/src/common/models/retention.go +++ b/src/common/models/retention.go @@ -16,15 +16,17 @@ package models import ( "time" - - "github.com/goharbor/harbor/src/core/api" ) // TagRecord represents all pertinent metadata about a tag type TagRecord struct { Project *Project Repository *RepoRecord - Tag *api.TagResp + + Name string + CreatedAt time.Time + LastPullAt time.Time + Labels []*Label } // TagAction records when a filter takes an action upon a tag diff --git a/src/core/api/repository.go b/src/core/api/repository.go index a33e51ebbae..12659b48c3c 100644 --- a/src/core/api/repository.go +++ b/src/core/api/repository.go @@ -78,7 +78,7 @@ func (r reposSorter) Less(i, j int) bool { return r[i].Index < r[j].Index } -type TagDetail struct { +type tagDetail struct { Digest string `json:"digest"` Name string `json:"name"` Size int64 `json:"size"` @@ -95,8 +95,8 @@ type cfg struct { Labels map[string]string `json:"labels"` } -type TagResp struct { - TagDetail +type tagResp struct { + tagDetail Signature *notary.Target `json:"signature"` ScanOverview *models.ImgScanOverview `json:"scan_overview,omitempty"` Labels []*models.Label `json:"labels"` @@ -607,7 +607,7 @@ func (ra *RepositoryAPI) GetTags() { // get config, signature and scan overview and assemble them into one // struct for each tag in tags func assembleTagsInParallel(client *registry.Repository, repository string, - tags []string, username string) []*TagResp { + tags []string, username string) []*tagResp { var err error signatures := map[string][]notary.Target{} if config.WithNotary() { @@ -618,13 +618,13 @@ func assembleTagsInParallel(client *registry.Repository, repository string, } } - c := make(chan *TagResp) + c := make(chan *tagResp) for _, tag := range tags { go assembleTag(c, client, repository, tag, config.WithClair(), config.WithNotary(), signatures) } - result := []*TagResp{} - var item *TagResp + result := []*tagResp{} + var item *tagResp for i := 0; i < len(tags); i++ { item = <-c if item == nil { @@ -635,10 +635,10 @@ func assembleTagsInParallel(client *registry.Repository, repository string, return result } -func assembleTag(c chan *TagResp, client *registry.Repository, +func assembleTag(c chan *tagResp, client *registry.Repository, repository, tag string, clairEnabled, notaryEnabled bool, signatures map[string][]notary.Target) { - item := &TagResp{} + item := &tagResp{} // labels image := fmt.Sprintf("%s:%s", repository, tag) labels, err := dao.GetLabelsOfResource(common.ResourceTypeImage, image) @@ -654,7 +654,7 @@ func assembleTag(c chan *TagResp, client *registry.Repository, log.Errorf("failed to get v2 manifest of %s:%s: %v", repository, tag, err) } if tagDetail != nil { - item.TagDetail = *tagDetail + item.tagDetail = *tagDetail } // scan overview @@ -677,8 +677,8 @@ func assembleTag(c chan *TagResp, client *registry.Repository, // getTagDetail returns the detail information for v2 manifest image // The information contains architecture, os, author, size, etc. -func getTagDetail(client *registry.Repository, tag string) (*TagDetail, error) { - detail := &TagDetail{ +func getTagDetail(client *registry.Repository, tag string) (*tagDetail, error) { + detail := &tagDetail{ Name: tag, } @@ -718,7 +718,7 @@ func getTagDetail(client *registry.Repository, tag string) (*TagDetail, error) { return detail, nil } -func populateAuthor(detail *TagDetail) { +func populateAuthor(detail *tagDetail) { // has author info already if len(detail.Author) > 0 { return diff --git a/src/core/api/repository_test.go b/src/core/api/repository_test.go index 68d7bf4230c..34649d245f1 100644 --- a/src/core/api/repository_test.go +++ b/src/core/api/repository_test.go @@ -96,7 +96,7 @@ func TestGetReposTags(t *testing.T) { t.Errorf("failed to get tags of repository %s: %v", repository, err) } else { assert.Equal(int(200), code, "httpStatusCode should be 200") - if tg, ok := tags.([]TagResp); ok { + if tg, ok := tags.([]tagResp); ok { assert.Equal(1, len(tg), fmt.Sprintf("there should be only one tag, but now %v", tg)) assert.Equal(tg[0].Name, "latest", "the tag should be latest") } else { @@ -207,18 +207,18 @@ func TestGetReposTop(t *testing.T) { func TestPopulateAuthor(t *testing.T) { author := "author" - detail := &TagDetail{ + detail := &tagDetail{ Author: author, } populateAuthor(detail) assert.Equal(t, author, detail.Author) - detail = &TagDetail{} + detail = &tagDetail{} populateAuthor(detail) assert.Equal(t, "", detail.Author) maintainer := "maintainer" - detail = &TagDetail{ + detail = &tagDetail{ Config: &cfg{ Labels: map[string]string{ "Maintainer": maintainer, From 0219c674a4b6e73d3ef8c63c2eae7f321b25e2c1 Mon Sep 17 00:00:00 2001 From: Nathan Lowe Date: Sun, 20 Jan 2019 12:39:26 -0500 Subject: [PATCH 3/4] Simplify Filter.Process: The filter chain builder will associate actions with filters Signed-off-by: Nathan Lowe --- src/common/models/retention.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/common/models/retention.go b/src/common/models/retention.go index 3f8882f51e5..b4262495fff 100644 --- a/src/common/models/retention.go +++ b/src/common/models/retention.go @@ -29,14 +29,6 @@ type TagRecord struct { Labels []*Label } -// TagAction records when a filter takes an action upon a tag -type TagAction struct { - // The tag the action applies to - Target *TagRecord - // The filter that took this action - ActingFilter Filter -} - // Filter is a tag filter in a Retention Policy Filter Chain type Filter interface { // Process takes tags from the input channel and writes them to one of the three output channels. @@ -46,7 +38,7 @@ type Filter interface { // or if the policy does not care if the tag is kept or deleted // // Filters do not own any of the provided channels and should **not** close them under any circumstance - Process(input <-chan *TagRecord, toKeep, toDelete chan<- *TagAction, next chan<- *TagRecord) error + Process(input <-chan *TagRecord, toKeep, toDelete, next chan<- *TagRecord) error // InitializeFor re-initializes the filter for tags from the specified project and repository // From d23f2776df73808824a07f5fbd429ff02ee33fea Mon Sep 17 00:00:00 2001 From: Nathan Lowe Date: Sat, 16 Feb 2019 12:50:06 -0500 Subject: [PATCH 4/4] Move retention types and interfaces to their own namespace Signed-off-by: Nathan Lowe --- src/common/models/retention.go | 101 --------------------------------- src/common/retention/filter.go | 68 ++++++++++++++++++++++ src/common/retention/policy.go | 63 ++++++++++++++++++++ 3 files changed, 131 insertions(+), 101 deletions(-) delete mode 100644 src/common/models/retention.go create mode 100644 src/common/retention/filter.go create mode 100644 src/common/retention/policy.go diff --git a/src/common/models/retention.go b/src/common/models/retention.go deleted file mode 100644 index b4262495fff..00000000000 --- a/src/common/models/retention.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2019 Project Harbor Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package models - -import ( - "time" -) - -// TagRecord represents all pertinent metadata about a tag -type TagRecord struct { - Project *Project - Repository *RepoRecord - - Name string - CreatedAt time.Time - LastPullAt time.Time - Labels []*Label -} - -// Filter is a tag filter in a Retention Policy Filter Chain -type Filter interface { - // Process takes tags from the input channel and writes them to one of the three output channels. - // Tags are written to toKeep if the tags should be explicitly kept by the Filter - // Tags are written to toDelete if the tags should be explicitly deleted by the Filter - // Tags are written to next if the retention policy does not apply to the provided tag - // or if the policy does not care if the tag is kept or deleted - // - // Filters do not own any of the provided channels and should **not** close them under any circumstance - Process(input <-chan *TagRecord, toKeep, toDelete, next chan<- *TagRecord) error - - // InitializeFor re-initializes the filter for tags from the specified project and repository - // - // Filters that maintain per-project or per-repository tracking metadata should reset it when - // this method is called. - InitializeFor(project *Project, repo *RepoRecord) -} - -// FilterMetadata defines the metadata needed to construct various Filter instances -type FilterMetadata struct { - ID int64 - // The type of the filter to construct - Type string - // Parameters used to construct the filter - Options map[string]interface{} -} - -// RetentionScope identifies the scope of a specific retention policy -type RetentionScope int - -const ( - // RetentionScopeServer identifies a retention policy defined server-wide - RetentionScopeServer RetentionScope = iota - // RetentionScopeProject identifies a retention policy defined for a specific project - RetentionScopeProject - // RetentionScopeRepository identifies a retention policy defined for a specific repository - RetentionScopeRepository -) - -// FallThroughAction determines what action the policy should take when a tag has not -// been explicitly kept nor explicitly deleted by all filters in the filter chain -type FallThroughAction int - -const ( - // KeepExtraTags indicates that tags which are not explicitly kept or deleted are implicitly kept - KeepExtraTags FallThroughAction = iota - // DeleteExtraTags indicates that tags which are not explicitly kept or deleted are implicitly deleted - DeleteExtraTags -) - -// RetentionPolicy contains an ordered slice of FilterMetadata used to construct filter chains -// during tag retention procession -type RetentionPolicy struct { - ID int64 - Name string - Enabled bool - - Scope RetentionScope - FallThroughAction FallThroughAction - - ProjectID int64 - RepositoryID int64 - - // When a filter chain is constructed for this policy, these filters will - // be chained together in the order they appear in the slice - Filters []*FilterMetadata - - CreatedAt time.Time - UpdatedAt time.Time -} diff --git a/src/common/retention/filter.go b/src/common/retention/filter.go new file mode 100644 index 00000000000..b2458600fd3 --- /dev/null +++ b/src/common/retention/filter.go @@ -0,0 +1,68 @@ +// Copyright 2019 Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package retention + +import ( + "time" + + "github.com/goharbor/harbor/src/common/models" +) + +// FilterAction denotes the action a filter has taken for a given tag record +type FilterAction uint + +const ( + // FilterActionKeep explicitly marks the tag as kept + FilterActionKeep FilterAction = iota + + // FilterActionDelete explicitly marks the tag as deleted + FilterActionDelete + + // FilterActionNoDecision passes the tag onto the next filter in the chain + FilterActionNoDecision +) + +// FilterMetadata defines the metadata needed to construct various Filter instances +type FilterMetadata struct { + ID int64 + // The type of the filter to construct + Type string + // Parameters used to construct the filter + Options map[string]interface{} +} + +// TagRecord represents all pertinent metadata about a tag +type TagRecord struct { + Project *models.Project + Repository *models.RepoRecord + + Name string + CreatedAt time.Time + LastPullAt time.Time + Labels []*models.Label +} + +// Filter is a tag filter in a Retention Policy Filter Chain +type Filter interface { + // Process determines what to do for a given tag record + Process(tag *TagRecord) (FilterAction, error) + + // InitializeFor re-initializes the filter for tags from the specified project and repository + // + // Filters that maintain per-project or per-repository tracking metadata should reset it when + // this method is called. Every call to `Process` will be for the same project and repo until + // `InitializeFor` is called again. + InitializeFor(project *models.Project, repo *models.RepoRecord) +} diff --git a/src/common/retention/policy.go b/src/common/retention/policy.go new file mode 100644 index 00000000000..3330913cdaa --- /dev/null +++ b/src/common/retention/policy.go @@ -0,0 +1,63 @@ +// Copyright 2019 Project Harbor Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package retention + +import "time" + +// Scope identifies the scope of a specific retention policy +type Scope int + +const ( + // ScopeServer identifies a retention policy defined server-wide + ScopeServer Scope = iota + + // ScopeProject identifies a retention policy defined for a specific project + ScopeProject + + // ScopeRepository identifies a retention policy defined for a specific repository + ScopeRepository +) + +// FallThroughAction determines what action the policy should take when a tag has not +// been explicitly kept nor explicitly deleted by all filters in the filter chain +type FallThroughAction int + +const ( + // KeepExtraTags indicates that tags which are not explicitly kept or deleted are implicitly kept + KeepExtraTags FallThroughAction = iota + // DeleteExtraTags indicates that tags which are not explicitly kept or deleted are implicitly deleted + DeleteExtraTags +) + +// Policy contains an ordered slice of FilterMetadata used to construct filter chains +// during tag retention procession +type Policy struct { + ID int64 + Name string + Enabled bool + + Scope Scope + FallThroughAction FallThroughAction + + ProjectID int64 + RepositoryID int64 + + // When a filter chain is constructed for this policy, these filters will + // be chained together in the order they appear in the slice + Filters []*FilterMetadata + + CreatedAt time.Time + UpdatedAt time.Time +}