Skip to content

Commit

Permalink
Allow admin to associate missing LFS objects for repositories (#18143)
Browse files Browse the repository at this point in the history
This PR reworked the Find pointer files feature in Settings -> LFS page.

When a LFS object is missing from database but exists in LFS content store, admin can associate it to the repository by clicking the Associate button.

This PR is not perfect (because the LFS module itself should be improved too), it's just a nice-to-have feature to help users recover their LFS repositories (eg: database was lost / table was truncated)
  • Loading branch information
wxiaoguang committed Jan 1, 2022
1 parent 25a290e commit 385dc6a
Show file tree
Hide file tree
Showing 3 changed files with 57 additions and 26 deletions.
53 changes: 38 additions & 15 deletions models/lfs.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,13 @@ package models
import (
"context"
"errors"
"fmt"

"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/timeutil"

"xorm.io/builder"
Expand Down Expand Up @@ -145,6 +147,11 @@ func LFSObjectAccessible(user *user_model.User, oid string) (bool, error) {
return count > 0, err
}

// LFSObjectIsAssociated checks if a provided Oid is associated
func LFSObjectIsAssociated(oid string) (bool, error) {
return db.GetEngine(db.DefaultContext).Exist(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
}

// LFSAutoAssociate auto associates accessible LFSMetaObjects
func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int64) error {
ctx, committer, err := db.TxContext()
Expand All @@ -162,23 +169,39 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int6
oidMap[meta.Oid] = meta
}

cond := builder.NewCond()
if !user.IsAdmin {
cond = builder.In("`lfs_meta_object`.repository_id",
builder.Select("`repository`.id").From("repository").Where(accessibleRepositoryCondition(user)))
}
newMetas := make([]*LFSMetaObject, 0, len(metas))
if err := sess.Cols("oid").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas); err != nil {
return err
}
for i := range newMetas {
newMetas[i].Size = oidMap[newMetas[i].Oid].Size
newMetas[i].RepositoryID = repoID
}
if err := db.Insert(ctx, newMetas); err != nil {
return err
newMetas := make([]*LFSMetaObject, 0, len(metas))
cond := builder.In(
"`lfs_meta_object`.repository_id",
builder.Select("`repository`.id").From("repository").Where(accessibleRepositoryCondition(user)),
)
err = sess.Cols("oid").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas)
if err != nil {
return err
}
if len(newMetas) != len(oidMap) {
return fmt.Errorf("unable collect all LFS objects from database, expected %d, actually %d", len(oidMap), len(newMetas))
}
for i := range newMetas {
newMetas[i].Size = oidMap[newMetas[i].Oid].Size
newMetas[i].RepositoryID = repoID
}
if err = db.Insert(ctx, newMetas); err != nil {
return err
}
} else {
// admin can associate any LFS object to any repository, and we do not care about errors (eg: duplicated unique key),
// even if error occurs, it won't hurt users and won't make things worse
for i := range metas {
_, err = sess.Insert(&LFSMetaObject{
Pointer: lfs.Pointer{Oid: metas[i].Oid, Size: metas[i].Size},
RepositoryID: repoID,
})
if err != nil {
log.Warn("failed to insert LFS meta object into database, err=%v", err)
}
}
}

return committer.Commit()
}

Expand Down
28 changes: 18 additions & 10 deletions routers/web/repo/lfs.go
Original file line number Diff line number Diff line change
Expand Up @@ -421,12 +421,13 @@ func LFSPointerFiles(ctx *context.Context) {
var numAssociated, numNoExist, numAssociatable int

type pointerResult struct {
SHA string
Oid string
Size int64
InRepo bool
Exists bool
Accessible bool
SHA string
Oid string
Size int64
InRepo bool
Exists bool
Accessible bool
Associatable bool
}

results := []pointerResult{}
Expand Down Expand Up @@ -461,22 +462,29 @@ func LFSPointerFiles(ctx *context.Context) {
// Can we fix?
// OK well that's "simple"
// - we need to check whether current user has access to a repo that has access to the file
result.Accessible, err = models.LFSObjectAccessible(ctx.User, pointerBlob.Oid)
result.Associatable, err = models.LFSObjectAccessible(ctx.User, pointerBlob.Oid)
if err != nil {
return err
}
} else {
result.Accessible = true
if !result.Associatable {
associated, err := models.LFSObjectIsAssociated(pointerBlob.Oid)
if err != nil {
return err
}
result.Associatable = !associated
}
}
}

result.Accessible = result.InRepo || result.Associatable

if result.InRepo {
numAssociated++
}
if !result.Exists {
numNoExist++
}
if !result.InRepo && result.Accessible {
if result.Associatable {
numAssociatable++
}

Expand Down
2 changes: 1 addition & 1 deletion templates/repo/settings/lfs_pointers.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
<form class="ui form" method="post" action="{{$.Link}}/associate">
{{.CsrfTokenHtml}}
{{range .Pointers}}
{{if and (not .InRepo) .Exists .Accessible}}
{{if .Associatable}}
<input type="hidden" name="oid" value="{{.Oid}} {{.Size}}"/>
{{end}}
{{end}}
Expand Down

0 comments on commit 385dc6a

Please sign in to comment.