Skip to content

Commit

Permalink
Delete tag retention rule and tag immutable rule when deleting project (
Browse files Browse the repository at this point in the history
#19390)

fixes #18250

Signed-off-by: stonezdj <daojunz@vmware.com>
  • Loading branch information
stonezdj committed Oct 24, 2023
1 parent a1effcb commit 7b0beed
Show file tree
Hide file tree
Showing 12 changed files with 299 additions and 23 deletions.
7 changes: 4 additions & 3 deletions src/controller/event/handler/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,10 @@ func init() {
_ = notifier.Subscribe(event.TopicDeleteTag, &auditlog.Handler{})

// internal
_ = notifier.Subscribe(event.TopicPullArtifact, &internal.Handler{})
_ = notifier.Subscribe(event.TopicPushArtifact, &internal.Handler{})
_ = notifier.Subscribe(event.TopicDeleteArtifact, &internal.Handler{})
_ = notifier.Subscribe(event.TopicPullArtifact, &internal.ArtifactEventHandler{})
_ = notifier.Subscribe(event.TopicPushArtifact, &internal.ArtifactEventHandler{})
_ = notifier.Subscribe(event.TopicDeleteArtifact, &internal.ArtifactEventHandler{})
_ = notifier.Subscribe(event.TopicDeleteProject, &internal.ProjectEventHandler{})

_ = task.RegisterTaskStatusChangePostFunc(job.ReplicationVendorType, func(ctx context.Context, taskID int64, status string) error {
notification.AddEvent(ctx, &metadata.ReplicationMetaData{
Expand Down
28 changes: 14 additions & 14 deletions src/controller/event/handler/internal/artifact.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,8 @@ func init() {
}
}

// Handler preprocess artifact event data
type Handler struct {
// ArtifactEventHandler preprocess artifact event data
type ArtifactEventHandler struct {
// execMgr for managing executions
execMgr task.ExecutionManager
// reportMgr for managing scan reports
Expand All @@ -89,12 +89,12 @@ type Handler struct {
}

// Name ...
func (a *Handler) Name() string {
func (a *ArtifactEventHandler) Name() string {
return "InternalArtifact"
}

// Handle ...
func (a *Handler) Handle(ctx context.Context, value interface{}) error {
func (a *ArtifactEventHandler) Handle(ctx context.Context, value interface{}) error {
switch v := value.(type) {
case *event.PullArtifactEvent:
return a.onPull(ctx, v.ArtifactEvent)
Expand All @@ -109,11 +109,11 @@ func (a *Handler) Handle(ctx context.Context, value interface{}) error {
}

// IsStateful ...
func (a *Handler) IsStateful() bool {
func (a *ArtifactEventHandler) IsStateful() bool {
return false
}

func (a *Handler) onPull(ctx context.Context, event *event.ArtifactEvent) error {
func (a *ArtifactEventHandler) onPull(ctx context.Context, event *event.ArtifactEvent) error {
if config.ScannerSkipUpdatePullTime(ctx) && isScannerUser(ctx, event) {
return nil
}
Expand Down Expand Up @@ -159,7 +159,7 @@ func (a *Handler) onPull(ctx context.Context, event *event.ArtifactEvent) error
return nil
}

func (a *Handler) updatePullTimeInCache(ctx context.Context, event *event.ArtifactEvent) {
func (a *ArtifactEventHandler) updatePullTimeInCache(ctx context.Context, event *event.ArtifactEvent) {
var tagName string
if len(event.Tags) != 0 {
tagName = event.Tags[0]
Expand All @@ -173,14 +173,14 @@ func (a *Handler) updatePullTimeInCache(ctx context.Context, event *event.Artifa
a.pullTimeStore[key] = time.Now()
}

func (a *Handler) addPullCountInCache(ctx context.Context, event *event.ArtifactEvent) {
func (a *ArtifactEventHandler) addPullCountInCache(ctx context.Context, event *event.ArtifactEvent) {
a.pullCountLock.Lock()
defer a.pullCountLock.Unlock()

a.pullCountStore[event.Artifact.RepositoryID] = a.pullCountStore[event.Artifact.RepositoryID] + 1
}

func (a *Handler) syncFlushPullTime(ctx context.Context, artifactID int64, tagName string, time time.Time) {
func (a *ArtifactEventHandler) syncFlushPullTime(ctx context.Context, artifactID int64, tagName string, time time.Time) {
var tagID int64

if tagName != "" {
Expand All @@ -203,13 +203,13 @@ func (a *Handler) syncFlushPullTime(ctx context.Context, artifactID int64, tagNa
}
}

func (a *Handler) syncFlushPullCount(ctx context.Context, repositoryID int64, count uint64) {
func (a *ArtifactEventHandler) syncFlushPullCount(ctx context.Context, repositoryID int64, count uint64) {
if err := repository.Ctl.AddPullCount(ctx, repositoryID, count); err != nil {
log.Warningf("failed to add pull count repository %d, %v", repositoryID, err)
}
}

func (a *Handler) asyncFlushPullTime(ctx context.Context) {
func (a *ArtifactEventHandler) asyncFlushPullTime(ctx context.Context) {
for {
<-time.After(asyncFlushDuration)
a.pullTimeLock.Lock()
Expand All @@ -235,7 +235,7 @@ func (a *Handler) asyncFlushPullTime(ctx context.Context) {
}
}

func (a *Handler) asyncFlushPullCount(ctx context.Context) {
func (a *ArtifactEventHandler) asyncFlushPullCount(ctx context.Context) {
for {
<-time.After(asyncFlushDuration)
a.pullCountLock.Lock()
Expand All @@ -249,7 +249,7 @@ func (a *Handler) asyncFlushPullCount(ctx context.Context) {
}
}

func (a *Handler) onPush(ctx context.Context, event *event.ArtifactEvent) error {
func (a *ArtifactEventHandler) onPush(ctx context.Context, event *event.ArtifactEvent) error {
go func() {
if event.Operator != "" {
ctx = context.WithValue(ctx, operator.ContextKey{}, event.Operator)
Expand All @@ -263,7 +263,7 @@ func (a *Handler) onPush(ctx context.Context, event *event.ArtifactEvent) error
return nil
}

func (a *Handler) onDelete(ctx context.Context, event *event.ArtifactEvent) error {
func (a *ArtifactEventHandler) onDelete(ctx context.Context, event *event.ArtifactEvent) error {
execMgr := task.ExecMgr
reportMgr := report.Mgr
artMgr := pkg.ArtifactMgr
Expand Down
4 changes: 2 additions & 2 deletions src/controller/event/handler/internal/artifact_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ type ArtifactHandlerTestSuite struct {
suite.Suite

ctx context.Context
handler *Handler
handler *ArtifactEventHandler
projectManager project.Manager
scannerCtl scanner.Controller
reportMgr *reportMock.Manager
Expand All @@ -70,7 +70,7 @@ func (suite *ArtifactHandlerTestSuite) SetupSuite() {
suite.execMgr = &taskMock.ExecutionManager{}
suite.reportMgr = &reportMock.Manager{}
suite.artMgr = &artMock.Manager{}
suite.handler = &Handler{execMgr: suite.execMgr, reportMgr: suite.reportMgr, artMgr: suite.artMgr}
suite.handler = &ArtifactEventHandler{execMgr: suite.execMgr, reportMgr: suite.reportMgr, artMgr: suite.artMgr}

// mock artifact
_, err := pkg.ArtifactMgr.Create(suite.ctx, &artifact.Artifact{ID: 1, RepositoryID: 1})
Expand Down
64 changes: 64 additions & 0 deletions src/controller/event/handler/internal/project.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
"context"

"github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/controller/immutable"
"github.com/goharbor/harbor/src/controller/retention"
"github.com/goharbor/harbor/src/lib/log"
)

// ProjectEventHandler process project event data
type ProjectEventHandler struct {
}

// Name return the name of this handler
func (a *ProjectEventHandler) Name() string {
return "InternalProject"
}

// IsStateful return false
func (a *ProjectEventHandler) IsStateful() bool {
return false
}

func (a *ProjectEventHandler) onProjectDelete(ctx context.Context, event *event.DeleteProjectEvent) error {
log.Infof("delete project id: %d", event.ProjectID)
// delete tag immutable
err := immutable.Ctr.DeleteImmutableRuleByProject(ctx, event.ProjectID)
if err != nil {
log.Errorf("failed to delete immutable rule, error %v", err)
}
// delete tag retention
err = retention.Ctl.DeleteRetentionByProject(ctx, event.ProjectID)
if err != nil {
log.Errorf("failed to delete retention rule, error %v", err)
}
return nil
}

// Handle handle project event
func (a *ProjectEventHandler) Handle(ctx context.Context, value interface{}) error {
switch v := value.(type) {
case *event.DeleteProjectEvent:
return a.onProjectDelete(ctx, v)
default:
log.Errorf("Can not handler this event type! %#v", v)
}
return nil
}
124 changes: 124 additions & 0 deletions src/controller/event/handler/internal/project_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
"context"
"testing"

beegoorm "github.com/beego/beego/v2/client/orm"
"github.com/stretchr/testify/suite"

common_dao "github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/controller/immutable"
"github.com/goharbor/harbor/src/lib/config"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/pkg"
"github.com/goharbor/harbor/src/pkg/artifact"
immutableModel "github.com/goharbor/harbor/src/pkg/immutable/model"
"github.com/goharbor/harbor/src/pkg/project"
"github.com/goharbor/harbor/src/pkg/project/models"
"github.com/goharbor/harbor/src/pkg/repository/model"
"github.com/goharbor/harbor/src/pkg/tag"
tagmodel "github.com/goharbor/harbor/src/pkg/tag/model/tag"
)

// ProjectHandlerTestSuite is test suite for artifact handler.
type ProjectHandlerTestSuite struct {
suite.Suite

ctx context.Context
handler *ProjectEventHandler
}

// SetupSuite prepares for running ArtifactHandlerTestSuite.
func (suite *ProjectHandlerTestSuite) SetupSuite() {
common_dao.PrepareTestForPostgresSQL()
config.Init()
suite.ctx = orm.NewContext(context.TODO(), beegoorm.NewOrm())
suite.handler = &ProjectEventHandler{}

// mock artifact
_, err := pkg.ArtifactMgr.Create(suite.ctx, &artifact.Artifact{ID: 1, RepositoryID: 1})
suite.Nil(err)
// mock repository
_, err = pkg.RepositoryMgr.Create(suite.ctx, &model.RepoRecord{RepositoryID: 1})
suite.Nil(err)
// mock tag
_, err = tag.Mgr.Create(suite.ctx, &tagmodel.Tag{ID: 1, RepositoryID: 1, ArtifactID: 1, Name: "latest"})
suite.Nil(err)
}

// TearDownSuite cleans environment.
func (suite *ProjectHandlerTestSuite) TearDownSuite() {
// delete tag
err := tag.Mgr.Delete(suite.ctx, 1)
suite.Nil(err)
// delete artifact
err = pkg.ArtifactMgr.Delete(suite.ctx, 1)
suite.Nil(err)
// delete repository
err = pkg.RepositoryMgr.Delete(suite.ctx, 1)
suite.Nil(err)

}

func (suite *ProjectHandlerTestSuite) TestOnProjectDelete() {
// create project
projID, err := project.New().Create(suite.ctx, &models.Project{Name: "test-project", OwnerID: 1})
suite.Nil(err)

defer project.New().Delete(suite.ctx, projID)
immutableRule := &immutableModel.Metadata{
ProjectID: projID,
Priority: 1,
Action: "immutable",
Template: "immutable_template",
TagSelectors: []*immutableModel.Selector{
{
Kind: "doublestar",
Decoration: "matches",
Pattern: "release-**",
},
},
ScopeSelectors: map[string][]*immutableModel.Selector{
"repository": {
{
Kind: "doublestar",
Decoration: "repoMatches",
Pattern: "redis",
},
},
},
}
// create immutable rule
immutableID, err := immutable.Ctr.CreateImmutableRule(suite.ctx, immutableRule)
suite.Nil(err)

// emit delete project event
event := &event.DeleteProjectEvent{ProjectID: projID}
err = suite.handler.onProjectDelete(suite.ctx, event)
suite.Nil(err)

// check if immutable rule is deleted
_, err = immutable.Ctr.GetImmutableRule(suite.ctx, immutableID)
suite.NotNil(err)
}

// TestArtifactHandler tests ArtifactHandler.
func TestProjectEventHandler(t *testing.T) {
suite.Run(t, &ProjectHandlerTestSuite{})
}
16 changes: 16 additions & 0 deletions src/controller/immutable/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,29 @@ type Controller interface {

// Count count the immutable rules
Count(ctx context.Context, query *q.Query) (int64, error)

// DeleteImmutableRuleByProject delete immuatable rules with project id
DeleteImmutableRuleByProject(ctx context.Context, projectID int64) error
}

// DefaultAPIController ...
type DefaultAPIController struct {
manager immutable.Manager
}

func (r *DefaultAPIController) DeleteImmutableRuleByProject(ctx context.Context, projectID int64) error {
rules, err := r.ListImmutableRules(ctx, q.New(q.KeyWords{"ProjectID": projectID}))
if err != nil {
return err
}
for _, rule := range rules {
if err = r.DeleteImmutableRule(ctx, rule.ID); err != nil {
return err
}
}
return nil
}

// GetImmutableRule ...
func (r *DefaultAPIController) GetImmutableRule(ctx context.Context, id int64) (*model.Metadata, error) {
return r.manager.GetImmutableRule(ctx, id)
Expand Down
17 changes: 17 additions & 0 deletions src/controller/retention/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ type Controller interface {
GetRetentionExecTaskLog(ctx context.Context, taskID int64) ([]byte, error)

GetRetentionExecTask(ctx context.Context, taskID int64) (*retention.Task, error)
// DeleteRetentionByProject delete retetion rule by project id
DeleteRetentionByProject(ctx context.Context, projectID int64) error
}

var (
Expand Down Expand Up @@ -405,6 +407,21 @@ func (r *defaultController) UpdateTaskInfo(ctx context.Context, taskID int64, to
return r.taskMgr.UpdateExtraAttrs(ctx, taskID, t.ExtraAttrs)
}

func (r *defaultController) DeleteRetentionByProject(ctx context.Context, projectID int64) error {
policyIDs, err := r.manager.ListPolicyIDs(ctx,
q.New(q.KeyWords{"scope_level": "project",
"scope_reference": fmt.Sprintf("%d", projectID)}))
if err != nil {
return err
}
for _, policyID := range policyIDs {
if err := r.DeleteRetention(ctx, policyID); err != nil {
return err
}
}
return nil
}

// NewController ...
func NewController() Controller {
retentionMgr := retention.NewManager()
Expand Down

0 comments on commit 7b0beed

Please sign in to comment.