From 5e4642d7e940e142e4c10eda8f3274f0d5dcdac4 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 10:50:33 +0000 Subject: [PATCH 01/69] Implemented LFS client. --- modules/lfs/client.go | 105 +++++++++++++++++++++++++++++++++ modules/lfs/server.go | 49 ++------------- modules/lfs/shared.go | 71 ++++++++++++++++++++++ modules/lfs/transferadapter.go | 47 +++++++++++++++ 4 files changed, 227 insertions(+), 45 deletions(-) create mode 100644 modules/lfs/client.go create mode 100644 modules/lfs/shared.go create mode 100644 modules/lfs/transferadapter.go diff --git a/modules/lfs/client.go b/modules/lfs/client.go new file mode 100644 index 000000000000..fb862cc8061c --- /dev/null +++ b/modules/lfs/client.go @@ -0,0 +1,105 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" +) + +type Client struct { + client *http.Client + transfers map[string]TransferAdapter +} + +func NewClient(hc *http.Client) *Client { + client := &Client{hc, make(map[string]TransferAdapter)} + + basic := &BasicTransferAdapter{hc} + + client.transfers[basic.Name()] = basic + + return client +} + +func (c *Client) transferNames() []string { + keys := make([]string, len(c.transfers)) + + i := 0 + for k := range c.transfers { + keys[i] = k + i++ + } + + return keys +} + +func (c *Client) batch(repositoryUrl, operation string, objects []*LfsObject) (*BatchResponse, error) { + url := fmt.Sprintf("%s.git/info/lfs/objects/batch", strings.TrimSuffix(repositoryUrl, ".git")) + + request := &BatchRequest{operation, c.transferNames(), nil, objects} + + payload := new(bytes.Buffer) + json.NewEncoder(payload).Encode(request) + + req, err := http.NewRequest("POST", url, payload) + if err != nil { + return nil, err + } + req.Header.Set("Content-type", metaMediaType) + req.Header.Set("Accept", metaMediaType) + + res, err := c.client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return nil, errors.New(fmt.Sprintf("Unexpected servers response: %s", res.Status)) + } + + var response BatchResponse + err = json.NewDecoder(res.Body).Decode(&response) + if err != nil { + return nil, err + } + + if len(response.Transfer) == 0 { + response.Transfer = "basic" + } + + return &response, nil +} + +func (c *Client) Download(repositoryUrl, oid string, size int64) (io.ReadCloser, error) { + var objects []*LfsObject + objects = append(objects, &LfsObject{oid, size}) + + result, err := c.batch(repositoryUrl, "download", objects) + if err != nil { + return nil, err + } + + transferAdapter, ok := c.transfers[result.Transfer] + if !ok { + return nil, fmt.Errorf("Transferadapter not found: %s", result.Transfer) + } + + if len(result.Objects) == 0 { + return nil, errors.New("No objects in result") + } + + content, err := transferAdapter.Download(result.Objects[0]) + if err != nil { + return nil, err + } + return content, nil +} diff --git a/modules/lfs/server.go b/modules/lfs/server.go index be21a4de8291..060ae53f388f 100644 --- a/modules/lfs/server.go +++ b/modules/lfs/server.go @@ -14,7 +14,6 @@ import ( "regexp" "strconv" "strings" - "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" @@ -25,10 +24,6 @@ import ( "github.com/dgrijalva/jwt-go" ) -const ( - metaMediaType = "application/vnd.git-lfs+json" -) - // RequestVars contain variables from the HTTP request. Variables from routing, json body decoding, and // some headers are stored. type RequestVars struct { @@ -40,35 +35,6 @@ type RequestVars struct { Authorization string } -// BatchVars contains multiple RequestVars processed in one batch operation. -// https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md -type BatchVars struct { - Transfers []string `json:"transfers,omitempty"` - Operation string `json:"operation"` - Objects []*RequestVars `json:"objects"` -} - -// BatchResponse contains multiple object metadata Representation structures -// for use with the batch API. -type BatchResponse struct { - Transfer string `json:"transfer,omitempty"` - Objects []*Representation `json:"objects"` -} - -// Representation is object metadata as seen by clients of the lfs server. -type Representation struct { - Oid string `json:"oid"` - Size int64 `json:"size"` - Actions map[string]*link `json:"actions"` - Error *ObjectError `json:"error,omitempty"` -} - -// ObjectError defines the JSON structure returned to the client in case of an error -type ObjectError struct { - Code int `json:"code"` - Message string `json:"message"` -} - // Claims is a JWT Token Claims type Claims struct { RepoID int64 @@ -87,13 +53,6 @@ func (v *RequestVars) VerifyLink() string { return setting.AppURL + path.Join(v.User, v.Repo+".git", "info/lfs/verify") } -// link provides a structure used to build a hypermedia representation of an HTTP link. -type link struct { - Href string `json:"href"` - Header map[string]string `json:"header,omitempty"` - ExpiresAt time.Time `json:"expires_at,omitempty"` -} - var oidRegExp = regexp.MustCompile(`^[A-Fa-f0-9]+$`) func isOidValid(oid string) bool { @@ -528,19 +487,19 @@ func unpack(ctx *context.Context) *RequestVars { } if r.Method == "POST" { // Maybe also check if +json - var p RequestVars + var o LfsObject bodyReader := r.Body defer bodyReader.Close() dec := json.NewDecoder(bodyReader) - err := dec.Decode(&p) + err := dec.Decode(&o) if err != nil { // The error is logged as a WARN here because this may represent misbehaviour rather than a true error log.Warn("Unable to decode POST request vars for LFS OID[%s] in %s/%s: Error: %v", rv.Oid, rv.User, rv.Repo, err) return rv } - rv.Oid = p.Oid - rv.Size = p.Size + rv.Oid = o.Oid + rv.Size = o.Size } return rv diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go new file mode 100644 index 000000000000..b078104d285d --- /dev/null +++ b/modules/lfs/shared.go @@ -0,0 +1,71 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "time" +) + +const ( + metaMediaType = "application/vnd.git-lfs+json" +) + +// BatchResponse contains multiple object metadata Representation structures +// for use with the batch API. +type BatchResponse struct { + Transfer string `json:"transfer,omitempty"` + Objects []*Representation `json:"objects"` +} + +// Representation is object metadata as seen by clients of the lfs server. +type Representation struct { + Oid string `json:"oid"` + Size int64 `json:"size"` + Actions map[string]*link `json:"actions"` + Error *ObjectError `json:"error,omitempty"` +} + +// ObjectError defines the JSON structure returned to the client in case of an error +type ObjectError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// link provides a structure used to build a hypermedia representation of an HTTP link. +type link struct { + Href string `json:"href"` + Header map[string]string `json:"header,omitempty"` + ExpiresAt time.Time `json:"expires_at,omitempty"` +} + +type Pointer struct { + Oid string `json:"oid"` + Size int64 `json:"size"` +} + +// BatchVars contains multiple RequestVars processed in one batch operation. +// https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md +type BatchVars struct { + Transfers []string `json:"transfers,omitempty"` + Operation string `json:"operation"` + Objects []*RequestVars `json:"objects"` +} + +// TODO replace BatchVars in Server +type BatchRequest struct { + Operation string `json:"operation"` + Transfers []string `json:"transfers,omitempty"` + Ref *Reference `json:"ref,omitempty"` + Objects []*LfsObject `json:"objects"` +} + +type Reference struct { + Name string `json:"name"` +} + +type LfsObject struct { + Oid string `json:"oid"` + Size int64 `json:"size"` +} \ No newline at end of file diff --git a/modules/lfs/transferadapter.go b/modules/lfs/transferadapter.go new file mode 100644 index 000000000000..cff70b9fca8a --- /dev/null +++ b/modules/lfs/transferadapter.go @@ -0,0 +1,47 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "errors" + "io" + "net/http" +) + +type TransferAdapter interface { + Name() string + Download(r *Representation) (io.ReadCloser, error) + //Upload(reader io.Reader) error + } + +type BasicTransferAdapter struct { + client *http.Client +} + +func (a *BasicTransferAdapter) Name() string { + return "basic" +} + +func (a *BasicTransferAdapter) Download(r *Representation) (io.ReadCloser, error) { + download, ok := r.Actions["download"] + if !ok { + return nil, errors.New("Action 'download' not found") + } + + req, err := http.NewRequest("GET", download.Href, nil) + if err != nil { + return nil, err + } + for key, value := range download.Header { + req.Header.Set(key, value) + } + + res, err := a.client.Do(req) + if err != nil { + return nil, err + } + + return res.Body, nil +} From 6a236f1abb1523a4f80f7b9e76f641c67df1f1e9 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 10:52:22 +0000 Subject: [PATCH 02/69] Implemented scanning for pointer files. --- modules/lfs/pointer_scanner.go | 80 ++++++++++++++++++++++++++ modules/lfs/pointer_scanner_nogogit.go | 15 +++++ 2 files changed, 95 insertions(+) create mode 100644 modules/lfs/pointer_scanner.go create mode 100644 modules/lfs/pointer_scanner_nogogit.go diff --git a/modules/lfs/pointer_scanner.go b/modules/lfs/pointer_scanner.go new file mode 100644 index 000000000000..3514c554babc --- /dev/null +++ b/modules/lfs/pointer_scanner.go @@ -0,0 +1,80 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package lfs + +import ( + "strconv" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + + "github.com/go-git/go-git/v5/plumbing/object" +) + +const blobSizeCutoff = 1024 + +// TODO Combine with methods in pointers.go +// TryReadPointer will return a pointer if the provided byte slice is a pointer file or nil otherwise. +func TryReadPointer(buf []byte) *Pointer { + headString := string(buf) + if !strings.HasPrefix(headString, models.LFSMetaFileIdentifier) { + return nil + } + + splitLines := strings.Split(headString, "\n") + if len(splitLines) < 3 { + return nil + } + + oid := strings.TrimPrefix(splitLines[1], models.LFSMetaFileOidPrefix) + size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) + if len(oid) != 64 || err != nil { + return nil + } + + return &Pointer{Oid: oid, Size: size} +} + +func SearchPointerFiles(repo *git.Repository) ([]*Pointer, error) { + gitRepo := repo.GoGitRepo() + + blobs, err := gitRepo.BlobObjects() + if err != nil { + return nil, err + } + + var pointers []*Pointer + + err = blobs.ForEach(func(blob *object.Blob) error { + if blob.Size > blobSizeCutoff { + return nil + } + + reader, err := blob.Reader() + if err != nil { + return nil + } + defer reader.Close() + + buf := make([]byte, blobSizeCutoff) + n, _ := reader.Read(buf) + buf = buf[:n] + + pointer := TryReadPointer(buf) + if pointer != nil { + pointers = append(pointers, pointer) + } + + return nil + }) + if err != nil { + return nil, err + } + + return pointers, nil +} \ No newline at end of file diff --git a/modules/lfs/pointer_scanner_nogogit.go b/modules/lfs/pointer_scanner_nogogit.go new file mode 100644 index 000000000000..e639ca875f0a --- /dev/null +++ b/modules/lfs/pointer_scanner_nogogit.go @@ -0,0 +1,15 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package lfs + +import ( + "code.gitea.io/gitea/modules/git" +) + +func SearchPointerFiles(repo *git.Repository) ([]*Pointer, error) { + return []*Pointer, nil +} \ No newline at end of file From ac8127b13c644002ad790c69718344195b741b74 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 10:54:30 +0000 Subject: [PATCH 03/69] Implemented downloading of lfs files. --- modules/lfs/repository.go | 55 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 modules/lfs/repository.go diff --git a/modules/lfs/repository.go b/modules/lfs/repository.go new file mode 100644 index 000000000000..da76e131b249 --- /dev/null +++ b/modules/lfs/repository.go @@ -0,0 +1,55 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/storage" +) + +func StoreMissingLfsObjectsInRepository(client *Client, repository *context.Repository) error { + contentStore := &ContentStore{ObjectStorage: storage.LFS} + + pointers, err := SearchPointerFiles(repository.GitRepo) + if err != nil { + return err + } + + for _, pointer := range pointers { + _, err := repository.Repository.GetLFSMetaObjectByOid(pointer.Oid) + if err != models.ErrLFSObjectNotExist { + continue + } + + // TODO What to do if file is too big / quota + + meta := &models.LFSMetaObject{Oid: pointer.Oid, Size: pointer.Size, RepositoryID: repository.Repository.ID} + + meta, err = models.NewLFSMetaObject(meta) + + exist, err := contentStore.Exists(meta) + if err != nil { + return err + } + if !exist { + lfsBaseUrl := "" // TODO + stream, err := client.Download(lfsBaseUrl, pointer.Oid, pointer.Size) + if err != nil { + return err + } + defer stream.Close() + + if err := contentStore.Put(meta, stream); err != nil { + if _, err2 := repository.Repository.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil { + return err2 + } + return err + } + } + } + + return nil +} \ No newline at end of file From 01899420ec1e2ebeddff83d24e216560cf25351f Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 15:40:25 +0000 Subject: [PATCH 04/69] Moved model-dependent code into services. --- cmd/serv.go | 2 +- modules/repofiles/update.go | 2 +- modules/repofiles/upload.go | 2 +- routers/repo/download.go | 2 +- routers/repo/lfs.go | 2 +- routers/repo/view.go | 2 +- routers/routes/web.go | 2 +- {modules => services}/lfs/content_store.go | 0 {modules => services}/lfs/locks.go | 0 {modules => services}/lfs/pointers.go | 0 {modules => services}/lfs/server.go | 46 ++++++++++++++++++++++ services/pull/lfs.go | 2 +- 12 files changed, 54 insertions(+), 8 deletions(-) rename {modules => services}/lfs/content_store.go (100%) rename {modules => services}/lfs/locks.go (100%) rename {modules => services}/lfs/pointers.go (100%) rename {modules => services}/lfs/server.go (93%) diff --git a/cmd/serv.go b/cmd/serv.go index 1e66cb511100..1c5a02b0bea2 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -18,11 +18,11 @@ import ( "time" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/pprof" "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/lfs" "github.com/dgrijalva/jwt-go" "github.com/kballard/go-shellquote" diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index 0ee1ada34c1c..8f834b8b0a8f 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -14,12 +14,12 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/services/lfs" stdcharset "golang.org/x/net/html/charset" "golang.org/x/text/transform" diff --git a/modules/repofiles/upload.go b/modules/repofiles/upload.go index c261e188c123..aa750999c4c7 100644 --- a/modules/repofiles/upload.go +++ b/modules/repofiles/upload.go @@ -11,9 +11,9 @@ import ( "strings" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/services/lfs" ) // UploadRepoFileOptions contains the uploaded repository file options diff --git a/routers/repo/download.go b/routers/repo/download.go index 50f893690b1a..23e0c02d1ca7 100644 --- a/routers/repo/download.go +++ b/routers/repo/download.go @@ -15,8 +15,8 @@ import ( "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/services/lfs" ) // ServeData download file from io.Reader diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index fb0e3b10eae9..85ab0468744e 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -22,10 +22,10 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git/pipeline" - "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/services/lfs" ) const ( diff --git a/routers/repo/view.go b/routers/repo/view.go index e50e4613b7a9..d97300f8ab86 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -24,10 +24,10 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/highlight" - "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/services/lfs" ) const ( diff --git a/routers/routes/web.go b/routers/routes/web.go index 1f860a623949..b6726b287b0b 100644 --- a/routers/routes/web.go +++ b/routers/routes/web.go @@ -16,7 +16,6 @@ import ( "code.gitea.io/gitea/modules/context" auth "code.gitea.io/gitea/modules/forms" "code.gitea.io/gitea/modules/httpcache" - "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/metrics" "code.gitea.io/gitea/modules/public" @@ -36,6 +35,7 @@ import ( "code.gitea.io/gitea/routers/repo" "code.gitea.io/gitea/routers/user" userSetting "code.gitea.io/gitea/routers/user/setting" + "code.gitea.io/gitea/services/lfs" "code.gitea.io/gitea/services/mailer" // to registers all internal adapters diff --git a/modules/lfs/content_store.go b/services/lfs/content_store.go similarity index 100% rename from modules/lfs/content_store.go rename to services/lfs/content_store.go diff --git a/modules/lfs/locks.go b/services/lfs/locks.go similarity index 100% rename from modules/lfs/locks.go rename to services/lfs/locks.go diff --git a/modules/lfs/pointers.go b/services/lfs/pointers.go similarity index 100% rename from modules/lfs/pointers.go rename to services/lfs/pointers.go diff --git a/modules/lfs/server.go b/services/lfs/server.go similarity index 93% rename from modules/lfs/server.go rename to services/lfs/server.go index 060ae53f388f..dd1c43bec5d0 100644 --- a/modules/lfs/server.go +++ b/services/lfs/server.go @@ -14,6 +14,7 @@ import ( "regexp" "strconv" "strings" + "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" @@ -35,6 +36,51 @@ type RequestVars struct { Authorization string } +// BatchVars contains multiple RequestVars processed in one batch operation. +// https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md +type BatchVars struct { + Transfers []string `json:"transfers,omitempty"` + Operation string `json:"operation"` + Objects []*RequestVars `json:"objects"` +} + +const ( + metaMediaType = "application/vnd.git-lfs+json" +) + +// BatchResponse contains multiple object metadata Representation structures +// for use with the batch API. +type BatchResponse struct { + Transfer string `json:"transfer,omitempty"` + Objects []*Representation `json:"objects"` +} + +// Representation is object metadata as seen by clients of the lfs server. +type Representation struct { + Oid string `json:"oid"` + Size int64 `json:"size"` + Actions map[string]*link `json:"actions"` + Error *ObjectError `json:"error,omitempty"` +} + +// ObjectError defines the JSON structure returned to the client in case of an error +type ObjectError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// link provides a structure used to build a hypermedia representation of an HTTP link. +type link struct { + Href string `json:"href"` + Header map[string]string `json:"header,omitempty"` + ExpiresAt time.Time `json:"expires_at,omitempty"` +} + +type LfsObject struct { + Oid string `json:"oid"` + Size int64 `json:"size"` +} + // Claims is a JWT Token Claims type Claims struct { RepoID int64 diff --git a/services/pull/lfs.go b/services/pull/lfs.go index a1981b825369..a43862741d24 100644 --- a/services/pull/lfs.go +++ b/services/pull/lfs.go @@ -13,8 +13,8 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git/pipeline" - "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/services/lfs" ) // LFSPush pushes lfs objects referred to in new commits in the head repository from the base repository From 9aa898fc24fe86c15ce8a096e546c6130cf2ae98 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 16:19:47 +0000 Subject: [PATCH 05/69] Removed models dependency. Added TryReadPointerFromBuffer. --- modules/lfs/pointer_scanner.go | 38 +++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/modules/lfs/pointer_scanner.go b/modules/lfs/pointer_scanner.go index 3514c554babc..b15b5ccdc72e 100644 --- a/modules/lfs/pointer_scanner.go +++ b/modules/lfs/pointer_scanner.go @@ -7,22 +7,40 @@ package lfs import ( + "io" "strconv" "strings" - "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" "github.com/go-git/go-git/v5/plumbing/object" ) -const blobSizeCutoff = 1024 +const ( + blobSizeCutoff = 1024 -// TODO Combine with methods in pointers.go -// TryReadPointer will return a pointer if the provided byte slice is a pointer file or nil otherwise. -func TryReadPointer(buf []byte) *Pointer { + // TODO remove duplicate from models + + // LFSMetaFileIdentifier is the string appearing at the first line of LFS pointer files. + // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md + LFSMetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" + + // LFSMetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash. + LFSMetaFileOidPrefix = "oid sha256:" +) + +func TryReadPointer(reader io.Reader) *Pointer { + buf := make([]byte, blobSizeCutoff) + n, _ := reader.Read(buf) + buf = buf[:n] + + return TryReadPointerFromBuffer(buf) +} + +// TryReadPointerFromBuffer will return a pointer if the provided byte slice is a pointer file or nil otherwise. +func TryReadPointerFromBuffer(buf []byte) *Pointer { headString := string(buf) - if !strings.HasPrefix(headString, models.LFSMetaFileIdentifier) { + if !strings.HasPrefix(headString, LFSMetaFileIdentifier) { return nil } @@ -31,7 +49,7 @@ func TryReadPointer(buf []byte) *Pointer { return nil } - oid := strings.TrimPrefix(splitLines[1], models.LFSMetaFileOidPrefix) + oid := strings.TrimPrefix(splitLines[1], LFSMetaFileOidPrefix) size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) if len(oid) != 64 || err != nil { return nil @@ -61,11 +79,7 @@ func SearchPointerFiles(repo *git.Repository) ([]*Pointer, error) { } defer reader.Close() - buf := make([]byte, blobSizeCutoff) - n, _ := reader.Read(buf) - buf = buf[:n] - - pointer := TryReadPointer(buf) + pointer := TryReadPointer(reader) if pointer != nil { pointers = append(pointers, pointer) } From 4fd390f9d53b66a7a8dfa60dabcc5c261aede0b4 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 16:21:22 +0000 Subject: [PATCH 06/69] Migrated code from service to module. --- modules/repofiles/update.go | 36 +++++----- routers/repo/download.go | 5 +- routers/repo/lfs.go | 5 +- routers/repo/view.go | 120 +++++++++++++++++----------------- services/lfs/content_store.go | 6 ++ services/lfs/pointers.go | 71 -------------------- services/pull/lfs.go | 20 ++++-- 7 files changed, 105 insertions(+), 158 deletions(-) delete mode 100644 services/lfs/pointers.go diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index 8f834b8b0a8f..d23660d44149 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" + lfs_module "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" @@ -70,30 +71,29 @@ func detectEncodingAndBOM(entry *git.TreeEntry, repo *models.Repository) (string buf = buf[:n] if setting.LFS.StartServer { - meta := lfs.IsPointerFile(&buf) - if meta != nil { - meta, err = repo.GetLFSMetaObjectByOid(meta.Oid) + pointer := lfs_module.TryReadPointerFromBuffer(buf) + if pointer != nil { + meta, err := repo.GetLFSMetaObjectByOid(pointer.Oid) if err != nil && err != models.ErrLFSObjectNotExist { // return default return "UTF-8", false } - } - if meta != nil { - dataRc, err := lfs.ReadMetaObject(meta) - if err != nil { - // return default - return "UTF-8", false - } - defer dataRc.Close() - buf = make([]byte, 1024) - n, err = dataRc.Read(buf) - if err != nil { - // return default - return "UTF-8", false + if meta != nil { + dataRc, err := lfs.ReadMetaObject(meta) + if err != nil { + // return default + return "UTF-8", false + } + defer dataRc.Close() + buf = make([]byte, 1024) + n, err = dataRc.Read(buf) + if err != nil { + // return default + return "UTF-8", false + } + buf = buf[:n] } - buf = buf[:n] } - } encoding, err := charset.DetectEncoding(buf) diff --git a/routers/repo/download.go b/routers/repo/download.go index 23e0c02d1ca7..508aa582d129 100644 --- a/routers/repo/download.go +++ b/routers/repo/download.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" + lfs_module "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/services/lfs" ) @@ -96,8 +97,8 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error { } }() - if meta, _ := lfs.ReadPointerFile(dataRc); meta != nil { - meta, _ = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid) + if pointer := lfs_module.TryReadPointer(dataRc); pointer != nil { + meta, _ := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid) if meta == nil { return ServeBlob(ctx, blob) } diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index 85ab0468744e..971103324aab 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -22,6 +22,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git/pipeline" + lfs_module "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" @@ -526,7 +527,7 @@ func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg } pointerBuf = pointerBuf[:size] // Now we need to check if the pointerBuf is an LFS pointer - pointer := lfs.IsPointerFile(&pointerBuf) + pointer := lfs_module.TryReadPointerFromBuffer(pointerBuf) if pointer == nil { continue } @@ -547,7 +548,7 @@ func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg result.InRepo = true } - result.Exists, err = contentStore.Exists(pointer) + result.Exists, err = contentStore.Exists(&models.LFSMetaObject{Oid: pointer.Oid, Size: pointer.Size}) if err != nil { _ = catFileBatchReader.CloseWithError(err) break diff --git a/routers/repo/view.go b/routers/repo/view.go index d97300f8ab86..2ddf21fdfffa 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -24,6 +24,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/highlight" + lfs_module "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" @@ -273,43 +274,42 @@ func renderDirectory(ctx *context.Context, treeLink string) { // FIXME: what happens when README file is an image? if isTextFile && setting.LFS.StartServer { - meta := lfs.IsPointerFile(&buf) - if meta != nil { - meta, err = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid) + pointer := lfs_module.TryReadPointerFromBuffer(buf) + if pointer != nil { + meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid) if err != nil && err != models.ErrLFSObjectNotExist { ctx.ServerError("GetLFSMetaObject", err) return } - } - - if meta != nil { - ctx.Data["IsLFSFile"] = true - isLFSFile = true - - // OK read the lfs object - var err error - dataRc, err = lfs.ReadMetaObject(meta) - if err != nil { - ctx.ServerError("ReadMetaObject", err) - return - } - defer dataRc.Close() - - buf = make([]byte, 1024) - n, err = dataRc.Read(buf) - if err != nil { - ctx.ServerError("Data", err) - return + if meta != nil { + ctx.Data["IsLFSFile"] = true + isLFSFile = true + + // OK read the lfs object + var err error + dataRc, err = lfs.ReadMetaObject(meta) + if err != nil { + ctx.ServerError("ReadMetaObject", err) + return + } + defer dataRc.Close() + + buf = make([]byte, 1024) + n, err = dataRc.Read(buf) + if err != nil { + ctx.ServerError("Data", err) + return + } + buf = buf[:n] + + isTextFile = base.IsTextFile(buf) + ctx.Data["IsTextFile"] = isTextFile + + fileSize = meta.Size + ctx.Data["FileSize"] = meta.Size + filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name)) + ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, filenameBase64) } - buf = buf[:n] - - isTextFile = base.IsTextFile(buf) - ctx.Data["IsTextFile"] = isTextFile - - fileSize = meta.Size - ctx.Data["FileSize"] = meta.Size - filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name)) - ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, filenameBase64) } } @@ -399,39 +399,39 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st //Check for LFS meta file if isTextFile && setting.LFS.StartServer { - meta := lfs.IsPointerFile(&buf) - if meta != nil { - meta, err = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid) + pointer := lfs_module.TryReadPointerFromBuffer(buf) + if pointer != nil { + meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid) if err != nil && err != models.ErrLFSObjectNotExist { ctx.ServerError("GetLFSMetaObject", err) return } - } - if meta != nil { - isLFSFile = true - - // OK read the lfs object - var err error - dataRc, err = lfs.ReadMetaObject(meta) - if err != nil { - ctx.ServerError("ReadMetaObject", err) - return - } - defer dataRc.Close() - - buf = make([]byte, 1024) - n, err = dataRc.Read(buf) - // Error EOF don't mean there is an error, it just means we read to - // the end - if err != nil && err != io.EOF { - ctx.ServerError("Data", err) - return + if meta != nil { + isLFSFile = true + + // OK read the lfs object + var err error + dataRc, err = lfs.ReadMetaObject(meta) + if err != nil { + ctx.ServerError("ReadMetaObject", err) + return + } + defer dataRc.Close() + + buf = make([]byte, 1024) + n, err = dataRc.Read(buf) + // Error EOF don't mean there is an error, it just means we read to + // the end + if err != nil && err != io.EOF { + ctx.ServerError("Data", err) + return + } + buf = buf[:n] + + isTextFile = base.IsTextFile(buf) + fileSize = meta.Size + ctx.Data["RawFileLink"] = fmt.Sprintf("%s/media/%s/%s", ctx.Repo.RepoLink, ctx.Repo.BranchNameSubURL(), ctx.Repo.TreePath) } - buf = buf[:n] - - isTextFile = base.IsTextFile(buf) - fileSize = meta.Size - ctx.Data["RawFileLink"] = fmt.Sprintf("%s/media/%s/%s", ctx.Repo.RepoLink, ctx.Repo.BranchNameSubURL(), ctx.Repo.TreePath) } } diff --git a/services/lfs/content_store.go b/services/lfs/content_store.go index 247191a1bf84..f89c65317c89 100644 --- a/services/lfs/content_store.go +++ b/services/lfs/content_store.go @@ -118,3 +118,9 @@ func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) { return true, nil } + +// ReadMetaObject will read a models.LFSMetaObject and return a reader +func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) { + contentStore := &ContentStore{ObjectStorage: storage.LFS} + return contentStore.Get(meta, 0) +} diff --git a/services/lfs/pointers.go b/services/lfs/pointers.go deleted file mode 100644 index c6fbf090e516..000000000000 --- a/services/lfs/pointers.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package lfs - -import ( - "io" - "strconv" - "strings" - - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/base" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/storage" -) - -// ReadPointerFile will return a partially filled LFSMetaObject if the provided reader is a pointer file -func ReadPointerFile(reader io.Reader) (*models.LFSMetaObject, *[]byte) { - if !setting.LFS.StartServer { - return nil, nil - } - - buf := make([]byte, 1024) - n, _ := reader.Read(buf) - buf = buf[:n] - - if isTextFile := base.IsTextFile(buf); !isTextFile { - return nil, nil - } - - return IsPointerFile(&buf), &buf -} - -// IsPointerFile will return a partially filled LFSMetaObject if the provided byte slice is a pointer file -func IsPointerFile(buf *[]byte) *models.LFSMetaObject { - if !setting.LFS.StartServer { - return nil - } - - headString := string(*buf) - if !strings.HasPrefix(headString, models.LFSMetaFileIdentifier) { - return nil - } - - splitLines := strings.Split(headString, "\n") - if len(splitLines) < 3 { - return nil - } - - oid := strings.TrimPrefix(splitLines[1], models.LFSMetaFileOidPrefix) - size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) - if len(oid) != 64 || err != nil { - return nil - } - - contentStore := &ContentStore{ObjectStorage: storage.LFS} - meta := &models.LFSMetaObject{Oid: oid, Size: size} - exist, err := contentStore.Exists(meta) - if err != nil || !exist { - return nil - } - - return meta -} - -// ReadMetaObject will read a models.LFSMetaObject and return a reader -func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) { - contentStore := &ContentStore{ObjectStorage: storage.LFS} - return contentStore.Get(meta, 0) -} diff --git a/services/pull/lfs.go b/services/pull/lfs.go index a43862741d24..3461b2fce78a 100644 --- a/services/pull/lfs.go +++ b/services/pull/lfs.go @@ -13,7 +13,9 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git/pipeline" + lfs_module "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/services/lfs" ) @@ -101,14 +103,22 @@ func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg } pointerBuf = pointerBuf[:size] // Now we need to check if the pointerBuf is an LFS pointer - pointer := lfs.IsPointerFile(&pointerBuf) + pointer := lfs_module.TryReadPointerFromBuffer(pointerBuf) if pointer == nil { continue } + + contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS} + meta := &models.LFSMetaObject{Oid: pointer.Oid, Size: pointer.Size} + exist, _ := contentStore.Exists(meta) + if !exist { + continue + } + // Then we need to check that this pointer is in the db - if _, err := pr.HeadRepo.GetLFSMetaObjectByOid(pointer.Oid); err != nil { + if _, err := pr.HeadRepo.GetLFSMetaObjectByOid(meta.Oid); err != nil { if err == models.ErrLFSObjectNotExist { - log.Warn("During merge of: %d in %-v, there is a pointer to LFS Oid: %s which although present in the LFS store is not associated with the head repo %-v", pr.Index, pr.BaseRepo, pointer.Oid, pr.HeadRepo) + log.Warn("During merge of: %d in %-v, there is a pointer to LFS Oid: %s which although present in the LFS store is not associated with the head repo %-v", pr.Index, pr.BaseRepo, meta.Oid, pr.HeadRepo) continue } _ = catFileBatchReader.CloseWithError(err) @@ -117,8 +127,8 @@ func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg // OK we have a pointer that is associated with the head repo // and is actually a file in the LFS // Therefore it should be associated with the base repo - pointer.RepositoryID = pr.BaseRepoID - if _, err := models.NewLFSMetaObject(pointer); err != nil { + meta.RepositoryID = pr.BaseRepoID + if _, err := models.NewLFSMetaObject(meta); err != nil { _ = catFileBatchReader.CloseWithError(err) break } From f94c5991672876e4e23a0a482346d667596f460a Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 16:31:35 +0000 Subject: [PATCH 07/69] Centralised storage creation. --- integrations/lfs_getobject_test.go | 3 +-- modules/repofiles/update.go | 3 +-- modules/repofiles/upload.go | 3 +-- routers/repo/lfs.go | 2 +- services/lfs/content_store.go | 7 ++++++- services/lfs/server.go | 11 +++++------ services/pull/lfs.go | 3 +-- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/integrations/lfs_getobject_test.go b/integrations/lfs_getobject_test.go index f364349ef140..e5e7cd15fb5e 100644 --- a/integrations/lfs_getobject_test.go +++ b/integrations/lfs_getobject_test.go @@ -18,7 +18,6 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/routers/routes" gzipp "github.com/klauspost/compress/gzip" @@ -50,7 +49,7 @@ func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string lfsID++ lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject) assert.NoError(t, err) - contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS} + contentStore := lfs.NewContetStore() exist, err := contentStore.Exists(lfsMetaObject) assert.NoError(t, err) if !exist { diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index d23660d44149..3fa1343eebc6 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -18,7 +18,6 @@ import ( "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/services/lfs" @@ -432,7 +431,7 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up if err != nil { return nil, err } - contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS} + contentStore := lfs.NewContetStore() exist, err := contentStore.Exists(lfsMetaObject) if err != nil { return nil, err diff --git a/modules/repofiles/upload.go b/modules/repofiles/upload.go index aa750999c4c7..a9391c98f6f3 100644 --- a/modules/repofiles/upload.go +++ b/modules/repofiles/upload.go @@ -12,7 +12,6 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/services/lfs" ) @@ -165,7 +164,7 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep // OK now we can insert the data into the store - there's no way to clean up the store // once it's in there, it's in there. - contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS} + contentStore := lfs.NewContetStore() for _, uploadInfo := range infos { if uploadInfo.lfsMetaObject == nil { continue diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index 971103324aab..ed83e693d023 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -494,7 +494,7 @@ type pointerResult struct { func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) { defer wg.Done() defer catFileBatchReader.Close() - contentStore := lfs.ContentStore{ObjectStorage: storage.LFS} + contentStore := lfs.NewContetStore() bufferedReader := bufio.NewReader(catFileBatchReader) buf := make([]byte, 1025) diff --git a/services/lfs/content_store.go b/services/lfs/content_store.go index f89c65317c89..ffe7bbf7dfd5 100644 --- a/services/lfs/content_store.go +++ b/services/lfs/content_store.go @@ -42,6 +42,11 @@ type ContentStore struct { storage.ObjectStorage } +func NewContetStore() *ContentStore { + contentStore := &ContentStore{ObjectStorage: storage.LFS} + return contentStore +} + // Get takes a Meta object and retrieves the content from the store, returning // it as an io.Reader. If fromByte > 0, the reader starts from that byte func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) { @@ -121,6 +126,6 @@ func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) { // ReadMetaObject will read a models.LFSMetaObject and return a reader func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) { - contentStore := &ContentStore{ObjectStorage: storage.LFS} + contentStore := NewContetStore() return contentStore.Get(meta, 0) } diff --git a/services/lfs/server.go b/services/lfs/server.go index dd1c43bec5d0..9c8cf092539b 100644 --- a/services/lfs/server.go +++ b/services/lfs/server.go @@ -20,7 +20,6 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/storage" "github.com/dgrijalva/jwt-go" ) @@ -192,7 +191,7 @@ func getContentHandler(ctx *context.Context) { } } - contentStore := &ContentStore{ObjectStorage: storage.LFS} + contentStore := NewContetStore() content, err := contentStore.Get(meta, fromByte) if err != nil { if IsErrRangeNotSatisfiable(err) { @@ -297,7 +296,7 @@ func PostHandler(ctx *context.Context) { ctx.Resp.Header().Set("Content-Type", metaMediaType) sentStatus := 202 - contentStore := &ContentStore{ObjectStorage: storage.LFS} + contentStore := NewContetStore() exist, err := contentStore.Exists(meta) if err != nil { log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", rv.Oid, rv.User, rv.Repo, err) @@ -358,7 +357,7 @@ func BatchHandler(ctx *context.Context) { return } - contentStore := &ContentStore{ObjectStorage: storage.LFS} + contentStore := NewContetStore() meta, err := repository.GetLFSMetaObjectByOid(object.Oid) if err == nil { // Object is found and exists @@ -416,7 +415,7 @@ func PutHandler(ctx *context.Context) { return } - contentStore := &ContentStore{ObjectStorage: storage.LFS} + contentStore := NewContetStore() defer ctx.Req.Body.Close() if err := contentStore.Put(meta, ctx.Req.Body); err != nil { // Put will log the error itself @@ -457,7 +456,7 @@ func VerifyHandler(ctx *context.Context) { return } - contentStore := &ContentStore{ObjectStorage: storage.LFS} + contentStore := NewContetStore() ok, err := contentStore.Verify(meta) if err != nil { // Error will be logged in Verify diff --git a/services/pull/lfs.go b/services/pull/lfs.go index 3461b2fce78a..14e194fb1638 100644 --- a/services/pull/lfs.go +++ b/services/pull/lfs.go @@ -15,7 +15,6 @@ import ( "code.gitea.io/gitea/modules/git/pipeline" lfs_module "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/services/lfs" ) @@ -108,7 +107,7 @@ func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg continue } - contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS} + contentStore := lfs.NewContetStore() meta := &models.LFSMetaObject{Oid: pointer.Oid, Size: pointer.Size} exist, _ := contentStore.Exists(meta) if !exist { From 85621c984361b5e34491d291bd6a4559f5e5ece2 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 16:56:12 +0000 Subject: [PATCH 08/69] Removed dependency from models. --- models/lfs.go | 6 +++++ modules/repofiles/update.go | 6 ++--- modules/repofiles/upload.go | 4 +-- routers/repo/download.go | 2 +- routers/repo/lfs.go | 4 +-- routers/repo/view.go | 4 +-- services/lfs/content_store.go | 51 ++++++++++++++++++++--------------- services/lfs/server.go | 12 ++++----- services/pull/lfs.go | 8 +++--- 9 files changed, 56 insertions(+), 41 deletions(-) diff --git a/models/lfs.go b/models/lfs.go index 274b32a73675..b108caa4b341 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -12,6 +12,7 @@ import ( "io" "path" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" @@ -41,6 +42,11 @@ func (m *LFSMetaObject) Pointer() string { return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size) } +func (m *LFSMetaObject) AsPointer() *lfs.Pointer { + pointer := &lfs.Pointer{Oid: m.Oid, Size: m.Size} + return pointer +} + // LFSTokenResponse defines the JSON structure in which the JWT token is stored. // This structure is fetched via SSH and passed by the Git LFS client to the server // endpoint for authorization. diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index 3fa1343eebc6..1447fd2da3b3 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -78,7 +78,7 @@ func detectEncodingAndBOM(entry *git.TreeEntry, repo *models.Repository) (string return "UTF-8", false } if meta != nil { - dataRc, err := lfs.ReadMetaObject(meta) + dataRc, err := lfs.ReadMetaObject(pointer) if err != nil { // return default return "UTF-8", false @@ -432,12 +432,12 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up return nil, err } contentStore := lfs.NewContetStore() - exist, err := contentStore.Exists(lfsMetaObject) + exist, err := contentStore.Exists(lfsMetaObject.AsPointer()) if err != nil { return nil, err } if !exist { - if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil { + if err := contentStore.Put(lfsMetaObject.AsPointer(), strings.NewReader(opts.Content)); err != nil { if _, err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil { return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err) } diff --git a/modules/repofiles/upload.go b/modules/repofiles/upload.go index a9391c98f6f3..42d83b3d433d 100644 --- a/modules/repofiles/upload.go +++ b/modules/repofiles/upload.go @@ -169,7 +169,7 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep if uploadInfo.lfsMetaObject == nil { continue } - exist, err := contentStore.Exists(uploadInfo.lfsMetaObject) + exist, err := contentStore.Exists(uploadInfo.lfsMetaObject.AsPointer()) if err != nil { return cleanUpAfterFailure(&infos, t, err) } @@ -181,7 +181,7 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep defer file.Close() // FIXME: Put regenerates the hash and copies the file over. // I guess this strictly ensures the soundness of the store but this is inefficient. - if err := contentStore.Put(uploadInfo.lfsMetaObject, file); err != nil { + if err := contentStore.Put(uploadInfo.lfsMetaObject.AsPointer(), file); err != nil { // OK Now we need to cleanup // Can't clean up the store, once uploaded there they're there. return cleanUpAfterFailure(&infos, t, err) diff --git a/routers/repo/download.go b/routers/repo/download.go index 508aa582d129..08661b40b139 100644 --- a/routers/repo/download.go +++ b/routers/repo/download.go @@ -102,7 +102,7 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error { if meta == nil { return ServeBlob(ctx, blob) } - lfsDataRc, err := lfs.ReadMetaObject(meta) + lfsDataRc, err := lfs.ReadMetaObject(meta.AsPointer()) if err != nil { return err } diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index ed83e693d023..5d4eb5608fe1 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -266,7 +266,7 @@ func LFSFileGet(ctx *context.Context) { return } ctx.Data["LFSFile"] = meta - dataRc, err := lfs.ReadMetaObject(meta) + dataRc, err := lfs.ReadMetaObject(meta.AsPointer()) if err != nil { ctx.ServerError("LFSFileGet", err) return @@ -548,7 +548,7 @@ func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg result.InRepo = true } - result.Exists, err = contentStore.Exists(&models.LFSMetaObject{Oid: pointer.Oid, Size: pointer.Size}) + result.Exists, err = contentStore.Exists(pointer) if err != nil { _ = catFileBatchReader.CloseWithError(err) break diff --git a/routers/repo/view.go b/routers/repo/view.go index 2ddf21fdfffa..6fa1feb6a901 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -287,7 +287,7 @@ func renderDirectory(ctx *context.Context, treeLink string) { // OK read the lfs object var err error - dataRc, err = lfs.ReadMetaObject(meta) + dataRc, err = lfs.ReadMetaObject(pointer) if err != nil { ctx.ServerError("ReadMetaObject", err) return @@ -411,7 +411,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st // OK read the lfs object var err error - dataRc, err = lfs.ReadMetaObject(meta) + dataRc, err = lfs.ReadMetaObject(pointer) if err != nil { ctx.ServerError("ReadMetaObject", err) return diff --git a/services/lfs/content_store.go b/services/lfs/content_store.go index ffe7bbf7dfd5..00768d2739cf 100644 --- a/services/lfs/content_store.go +++ b/services/lfs/content_store.go @@ -11,8 +11,9 @@ import ( "fmt" "io" "os" + "path" - "code.gitea.io/gitea/models" + lfs_module "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" ) @@ -47,50 +48,58 @@ func NewContetStore() *ContentStore { return contentStore } +func relativePath(p *lfs_module.Pointer) string { + if len(p.Oid) < 5 { + return p.Oid + } + + return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid[4:]) +} + // Get takes a Meta object and retrieves the content from the store, returning // it as an io.Reader. If fromByte > 0, the reader starts from that byte -func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) { - f, err := s.Open(meta.RelativePath()) +func (s *ContentStore) Get(pointer *lfs_module.Pointer, fromByte int64) (io.ReadCloser, error) { + f, err := s.Open(relativePath(pointer)) if err != nil { - log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", meta.Oid, err) + log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", pointer.Oid, err) return nil, err } if fromByte > 0 { - if fromByte >= meta.Size { + if fromByte >= pointer.Size { return nil, ErrRangeNotSatisfiable{ FromByte: fromByte, } } _, err = f.Seek(fromByte, io.SeekStart) if err != nil { - log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", meta.Oid, fromByte, err) + log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", pointer.Oid, fromByte, err) } } return f, err } // Put takes a Meta object and an io.Reader and writes the content to the store. -func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error { +func (s *ContentStore) Put(pointer *lfs_module.Pointer, r io.Reader) error { hash := sha256.New() rd := io.TeeReader(r, hash) - p := meta.RelativePath() + p := relativePath(pointer) written, err := s.Save(p, rd) if err != nil { - log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, p, err) + log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", pointer.Oid, p, err) return err } - if written != meta.Size { + if written != pointer.Size { if err := s.Delete(p); err != nil { - log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err) + log.Error("Cleaning the LFS OID[%s] failed: %v", pointer.Oid, err) } return errSizeMismatch } shaStr := hex.EncodeToString(hash.Sum(nil)) - if shaStr != meta.Oid { + if shaStr != pointer.Oid { if err := s.Delete(p); err != nil { - log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err) + log.Error("Cleaning the LFS OID[%s] failed: %v", pointer.Oid, err) } return errHashMismatch } @@ -99,8 +108,8 @@ func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error { } // Exists returns true if the object exists in the content store. -func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) { - _, err := s.ObjectStorage.Stat(meta.RelativePath()) +func (s *ContentStore) Exists(pointer *lfs_module.Pointer) (bool, error) { + _, err := s.ObjectStorage.Stat(relativePath(pointer)) if err != nil { if os.IsNotExist(err) { return false, nil @@ -111,13 +120,13 @@ func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) { } // Verify returns true if the object exists in the content store and size is correct. -func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) { - p := meta.RelativePath() +func (s *ContentStore) Verify(pointer *lfs_module.Pointer) (bool, error) { + p := relativePath(pointer) fi, err := s.ObjectStorage.Stat(p) - if os.IsNotExist(err) || (err == nil && fi.Size() != meta.Size) { + if os.IsNotExist(err) || (err == nil && fi.Size() != pointer.Size) { return false, nil } else if err != nil { - log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, meta.Oid, err) + log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, pointer.Oid, err) return false, err } @@ -125,7 +134,7 @@ func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) { } // ReadMetaObject will read a models.LFSMetaObject and return a reader -func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) { +func ReadMetaObject(pointer *lfs_module.Pointer) (io.ReadCloser, error) { contentStore := NewContetStore() - return contentStore.Get(meta, 0) + return contentStore.Get(pointer, 0) } diff --git a/services/lfs/server.go b/services/lfs/server.go index 9c8cf092539b..cd5e6b84dd02 100644 --- a/services/lfs/server.go +++ b/services/lfs/server.go @@ -192,7 +192,7 @@ func getContentHandler(ctx *context.Context) { } contentStore := NewContetStore() - content, err := contentStore.Get(meta, fromByte) + content, err := contentStore.Get(meta.AsPointer(), fromByte) if err != nil { if IsErrRangeNotSatisfiable(err) { writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable) @@ -297,7 +297,7 @@ func PostHandler(ctx *context.Context) { sentStatus := 202 contentStore := NewContetStore() - exist, err := contentStore.Exists(meta) + exist, err := contentStore.Exists(meta.AsPointer()) if err != nil { log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", rv.Oid, rv.User, rv.Repo, err) writeStatus(ctx, 500) @@ -361,7 +361,7 @@ func BatchHandler(ctx *context.Context) { meta, err := repository.GetLFSMetaObjectByOid(object.Oid) if err == nil { // Object is found and exists - exist, err := contentStore.Exists(meta) + exist, err := contentStore.Exists(meta.AsPointer()) if err != nil { log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err) writeStatus(ctx, 500) @@ -382,7 +382,7 @@ func BatchHandler(ctx *context.Context) { // Object is not found meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID}) if err == nil { - exist, err := contentStore.Exists(meta) + exist, err := contentStore.Exists(meta.AsPointer()) if err != nil { log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err) writeStatus(ctx, 500) @@ -417,7 +417,7 @@ func PutHandler(ctx *context.Context) { contentStore := NewContetStore() defer ctx.Req.Body.Close() - if err := contentStore.Put(meta, ctx.Req.Body); err != nil { + if err := contentStore.Put(meta.AsPointer(), ctx.Req.Body); err != nil { // Put will log the error itself ctx.Resp.WriteHeader(500) if err == errSizeMismatch || err == errHashMismatch { @@ -457,7 +457,7 @@ func VerifyHandler(ctx *context.Context) { } contentStore := NewContetStore() - ok, err := contentStore.Verify(meta) + ok, err := contentStore.Verify(meta.AsPointer()) if err != nil { // Error will be logged in Verify ctx.Resp.WriteHeader(500) diff --git a/services/pull/lfs.go b/services/pull/lfs.go index 14e194fb1638..ab3a9cb8f938 100644 --- a/services/pull/lfs.go +++ b/services/pull/lfs.go @@ -108,16 +108,15 @@ func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg } contentStore := lfs.NewContetStore() - meta := &models.LFSMetaObject{Oid: pointer.Oid, Size: pointer.Size} - exist, _ := contentStore.Exists(meta) + exist, _ := contentStore.Exists(pointer) if !exist { continue } // Then we need to check that this pointer is in the db - if _, err := pr.HeadRepo.GetLFSMetaObjectByOid(meta.Oid); err != nil { + if _, err := pr.HeadRepo.GetLFSMetaObjectByOid(pointer.Oid); err != nil { if err == models.ErrLFSObjectNotExist { - log.Warn("During merge of: %d in %-v, there is a pointer to LFS Oid: %s which although present in the LFS store is not associated with the head repo %-v", pr.Index, pr.BaseRepo, meta.Oid, pr.HeadRepo) + log.Warn("During merge of: %d in %-v, there is a pointer to LFS Oid: %s which although present in the LFS store is not associated with the head repo %-v", pr.Index, pr.BaseRepo, pointer.Oid, pr.HeadRepo) continue } _ = catFileBatchReader.CloseWithError(err) @@ -126,6 +125,7 @@ func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg // OK we have a pointer that is associated with the head repo // and is actually a file in the LFS // Therefore it should be associated with the base repo + meta := &models.LFSMetaObject{Oid: pointer.Oid, Size: pointer.Size} meta.RepositoryID = pr.BaseRepoID if _, err := models.NewLFSMetaObject(meta); err != nil { _ = catFileBatchReader.CloseWithError(err) From ae60e34018287b97647284576bfa96a62f6ad67d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 18:13:00 +0100 Subject: [PATCH 09/69] Moved ContentStore into modules. --- {services => modules}/lfs/content_store.go | 27 ++++++++-------------- modules/repofiles/update.go | 5 ++-- modules/repofiles/upload.go | 2 +- routers/repo/download.go | 5 ++-- routers/repo/lfs.go | 5 ++-- routers/repo/view.go | 7 +++--- services/lfs/server.go | 17 ++++++++------ services/pull/lfs.go | 5 ++-- 8 files changed, 32 insertions(+), 41 deletions(-) rename {services => modules}/lfs/content_store.go (78%) diff --git a/services/lfs/content_store.go b/modules/lfs/content_store.go similarity index 78% rename from services/lfs/content_store.go rename to modules/lfs/content_store.go index 00768d2739cf..ffcad3ae951b 100644 --- a/services/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -13,14 +13,13 @@ import ( "os" "path" - lfs_module "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" ) var ( - errHashMismatch = errors.New("Content hash does not match OID") - errSizeMismatch = errors.New("Content size does not match") + ErrHashMismatch = errors.New("Content hash does not match OID") + ErrSizeMismatch = errors.New("Content size does not match") ) // ErrRangeNotSatisfiable represents an error which request range is not satisfiable. @@ -32,12 +31,6 @@ func (err ErrRangeNotSatisfiable) Error() string { return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte) } -// IsErrRangeNotSatisfiable returns true if the error is an ErrRangeNotSatisfiable -func IsErrRangeNotSatisfiable(err error) bool { - _, ok := err.(ErrRangeNotSatisfiable) - return ok -} - // ContentStore provides a simple file system based storage. type ContentStore struct { storage.ObjectStorage @@ -48,7 +41,7 @@ func NewContetStore() *ContentStore { return contentStore } -func relativePath(p *lfs_module.Pointer) string { +func relativePath(p *Pointer) string { if len(p.Oid) < 5 { return p.Oid } @@ -58,7 +51,7 @@ func relativePath(p *lfs_module.Pointer) string { // Get takes a Meta object and retrieves the content from the store, returning // it as an io.Reader. If fromByte > 0, the reader starts from that byte -func (s *ContentStore) Get(pointer *lfs_module.Pointer, fromByte int64) (io.ReadCloser, error) { +func (s *ContentStore) Get(pointer *Pointer, fromByte int64) (io.ReadCloser, error) { f, err := s.Open(relativePath(pointer)) if err != nil { log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", pointer.Oid, err) @@ -79,7 +72,7 @@ func (s *ContentStore) Get(pointer *lfs_module.Pointer, fromByte int64) (io.Read } // Put takes a Meta object and an io.Reader and writes the content to the store. -func (s *ContentStore) Put(pointer *lfs_module.Pointer, r io.Reader) error { +func (s *ContentStore) Put(pointer *Pointer, r io.Reader) error { hash := sha256.New() rd := io.TeeReader(r, hash) p := relativePath(pointer) @@ -93,7 +86,7 @@ func (s *ContentStore) Put(pointer *lfs_module.Pointer, r io.Reader) error { if err := s.Delete(p); err != nil { log.Error("Cleaning the LFS OID[%s] failed: %v", pointer.Oid, err) } - return errSizeMismatch + return ErrSizeMismatch } shaStr := hex.EncodeToString(hash.Sum(nil)) @@ -101,14 +94,14 @@ func (s *ContentStore) Put(pointer *lfs_module.Pointer, r io.Reader) error { if err := s.Delete(p); err != nil { log.Error("Cleaning the LFS OID[%s] failed: %v", pointer.Oid, err) } - return errHashMismatch + return ErrHashMismatch } return nil } // Exists returns true if the object exists in the content store. -func (s *ContentStore) Exists(pointer *lfs_module.Pointer) (bool, error) { +func (s *ContentStore) Exists(pointer *Pointer) (bool, error) { _, err := s.ObjectStorage.Stat(relativePath(pointer)) if err != nil { if os.IsNotExist(err) { @@ -120,7 +113,7 @@ func (s *ContentStore) Exists(pointer *lfs_module.Pointer) (bool, error) { } // Verify returns true if the object exists in the content store and size is correct. -func (s *ContentStore) Verify(pointer *lfs_module.Pointer) (bool, error) { +func (s *ContentStore) Verify(pointer *Pointer) (bool, error) { p := relativePath(pointer) fi, err := s.ObjectStorage.Stat(p) if os.IsNotExist(err) || (err == nil && fi.Size() != pointer.Size) { @@ -134,7 +127,7 @@ func (s *ContentStore) Verify(pointer *lfs_module.Pointer) (bool, error) { } // ReadMetaObject will read a models.LFSMetaObject and return a reader -func ReadMetaObject(pointer *lfs_module.Pointer) (io.ReadCloser, error) { +func ReadMetaObject(pointer *Pointer) (io.ReadCloser, error) { contentStore := NewContetStore() return contentStore.Get(pointer, 0) } diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index 1447fd2da3b3..4ccdbc066f2f 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -14,12 +14,11 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" - lfs_module "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/services/lfs" stdcharset "golang.org/x/net/html/charset" "golang.org/x/text/transform" @@ -70,7 +69,7 @@ func detectEncodingAndBOM(entry *git.TreeEntry, repo *models.Repository) (string buf = buf[:n] if setting.LFS.StartServer { - pointer := lfs_module.TryReadPointerFromBuffer(buf) + pointer := lfs.TryReadPointerFromBuffer(buf) if pointer != nil { meta, err := repo.GetLFSMetaObjectByOid(pointer.Oid) if err != nil && err != models.ErrLFSObjectNotExist { diff --git a/modules/repofiles/upload.go b/modules/repofiles/upload.go index 42d83b3d433d..4e6b4b8a85ea 100644 --- a/modules/repofiles/upload.go +++ b/modules/repofiles/upload.go @@ -11,8 +11,8 @@ import ( "strings" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/services/lfs" ) // UploadRepoFileOptions contains the uploaded repository file options diff --git a/routers/repo/download.go b/routers/repo/download.go index 08661b40b139..50a90a8d50ac 100644 --- a/routers/repo/download.go +++ b/routers/repo/download.go @@ -15,9 +15,8 @@ import ( "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" - lfs_module "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/services/lfs" ) // ServeData download file from io.Reader @@ -97,7 +96,7 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error { } }() - if pointer := lfs_module.TryReadPointer(dataRc); pointer != nil { + if pointer := lfs.TryReadPointer(dataRc); pointer != nil { meta, _ := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid) if meta == nil { return ServeBlob(ctx, blob) diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index 5d4eb5608fe1..24cf866719ba 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -22,11 +22,10 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/git/pipeline" - lfs_module "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" - "code.gitea.io/gitea/services/lfs" ) const ( @@ -527,7 +526,7 @@ func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg } pointerBuf = pointerBuf[:size] // Now we need to check if the pointerBuf is an LFS pointer - pointer := lfs_module.TryReadPointerFromBuffer(pointerBuf) + pointer := lfs.TryReadPointerFromBuffer(pointerBuf) if pointer == nil { continue } diff --git a/routers/repo/view.go b/routers/repo/view.go index 6fa1feb6a901..be5127d07064 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -24,11 +24,10 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/highlight" - lfs_module "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/services/lfs" ) const ( @@ -274,7 +273,7 @@ func renderDirectory(ctx *context.Context, treeLink string) { // FIXME: what happens when README file is an image? if isTextFile && setting.LFS.StartServer { - pointer := lfs_module.TryReadPointerFromBuffer(buf) + pointer := lfs.TryReadPointerFromBuffer(buf) if pointer != nil { meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid) if err != nil && err != models.ErrLFSObjectNotExist { @@ -399,7 +398,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st //Check for LFS meta file if isTextFile && setting.LFS.StartServer { - pointer := lfs_module.TryReadPointerFromBuffer(buf) + pointer := lfs.TryReadPointerFromBuffer(buf) if pointer != nil { meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid) if err != nil && err != models.ErrLFSObjectNotExist { diff --git a/services/lfs/server.go b/services/lfs/server.go index cd5e6b84dd02..e0b12d8755a9 100644 --- a/services/lfs/server.go +++ b/services/lfs/server.go @@ -7,6 +7,7 @@ package lfs import ( "encoding/base64" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -18,6 +19,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" + lfs_module "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -191,10 +193,11 @@ func getContentHandler(ctx *context.Context) { } } - contentStore := NewContetStore() + contentStore := lfs_module.NewContetStore() content, err := contentStore.Get(meta.AsPointer(), fromByte) if err != nil { - if IsErrRangeNotSatisfiable(err) { + var rerr *lfs_module.ErrRangeNotSatisfiable + if errors.As(err, &rerr) { writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable) } else { // Errors are logged in contentStore.Get @@ -296,7 +299,7 @@ func PostHandler(ctx *context.Context) { ctx.Resp.Header().Set("Content-Type", metaMediaType) sentStatus := 202 - contentStore := NewContetStore() + contentStore := lfs_module.NewContetStore() exist, err := contentStore.Exists(meta.AsPointer()) if err != nil { log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", rv.Oid, rv.User, rv.Repo, err) @@ -357,7 +360,7 @@ func BatchHandler(ctx *context.Context) { return } - contentStore := NewContetStore() + contentStore := lfs_module.NewContetStore() meta, err := repository.GetLFSMetaObjectByOid(object.Oid) if err == nil { // Object is found and exists @@ -415,12 +418,12 @@ func PutHandler(ctx *context.Context) { return } - contentStore := NewContetStore() + contentStore := lfs_module.NewContetStore() defer ctx.Req.Body.Close() if err := contentStore.Put(meta.AsPointer(), ctx.Req.Body); err != nil { // Put will log the error itself ctx.Resp.WriteHeader(500) - if err == errSizeMismatch || err == errHashMismatch { + if err == lfs_module.ErrSizeMismatch || err == lfs_module.ErrHashMismatch { fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err) } else { fmt.Fprintf(ctx.Resp, `{"message":"Internal Server Error"}`) @@ -456,7 +459,7 @@ func VerifyHandler(ctx *context.Context) { return } - contentStore := NewContetStore() + contentStore := lfs_module.NewContetStore() ok, err := contentStore.Verify(meta.AsPointer()) if err != nil { // Error will be logged in Verify diff --git a/services/pull/lfs.go b/services/pull/lfs.go index ab3a9cb8f938..0499de199461 100644 --- a/services/pull/lfs.go +++ b/services/pull/lfs.go @@ -13,9 +13,8 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git/pipeline" - lfs_module "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/services/lfs" ) // LFSPush pushes lfs objects referred to in new commits in the head repository from the base repository @@ -102,7 +101,7 @@ func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg } pointerBuf = pointerBuf[:size] // Now we need to check if the pointerBuf is an LFS pointer - pointer := lfs_module.TryReadPointerFromBuffer(pointerBuf) + pointer := lfs.TryReadPointerFromBuffer(pointerBuf) if pointer == nil { continue } From 37fa1c22a8b52322a199b6bd652229ed6fa25919 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 18:20:00 +0000 Subject: [PATCH 10/69] Share structs between server and client. --- modules/lfs/client.go | 10 +- modules/lfs/shared.go | 70 +++++------ modules/lfs/transferadapter.go | 4 +- services/lfs/locks.go | 13 +- services/lfs/server.go | 222 +++++++++++++-------------------- 5 files changed, 130 insertions(+), 189 deletions(-) diff --git a/modules/lfs/client.go b/modules/lfs/client.go index fb862cc8061c..06fbdc5de8c9 100644 --- a/modules/lfs/client.go +++ b/modules/lfs/client.go @@ -41,7 +41,7 @@ func (c *Client) transferNames() []string { return keys } -func (c *Client) batch(repositoryUrl, operation string, objects []*LfsObject) (*BatchResponse, error) { +func (c *Client) batch(repositoryUrl, operation string, objects []*Pointer) (*BatchResponse, error) { url := fmt.Sprintf("%s.git/info/lfs/objects/batch", strings.TrimSuffix(repositoryUrl, ".git")) request := &BatchRequest{operation, c.transferNames(), nil, objects} @@ -53,8 +53,8 @@ func (c *Client) batch(repositoryUrl, operation string, objects []*LfsObject) (* if err != nil { return nil, err } - req.Header.Set("Content-type", metaMediaType) - req.Header.Set("Accept", metaMediaType) + req.Header.Set("Content-type", MediaType) + req.Header.Set("Accept", MediaType) res, err := c.client.Do(req) if err != nil { @@ -80,8 +80,8 @@ func (c *Client) batch(repositoryUrl, operation string, objects []*LfsObject) (* } func (c *Client) Download(repositoryUrl, oid string, size int64) (io.ReadCloser, error) { - var objects []*LfsObject - objects = append(objects, &LfsObject{oid, size}) + var objects []*Pointer + objects = append(objects, &Pointer{oid, size}) result, err := c.batch(repositoryUrl, "download", objects) if err != nil { diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go index b078104d285d..0ba1d8669fca 100644 --- a/modules/lfs/shared.go +++ b/modules/lfs/shared.go @@ -9,63 +9,53 @@ import ( ) const ( - metaMediaType = "application/vnd.git-lfs+json" + MediaType = "application/vnd.git-lfs+json" ) +// BatchRequest contains multiple requests processed in one batch operation. +// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#requests +type BatchRequest struct { + Operation string `json:"operation"` + Transfers []string `json:"transfers,omitempty"` + Ref *Reference `json:"ref,omitempty"` + Objects []*Pointer `json:"objects"` +} + +// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#ref-property +type Reference struct { + Name string `json:"name"` +} + +type Pointer struct { + Oid string `json:"oid"` + Size int64 `json:"size"` +} + // BatchResponse contains multiple object metadata Representation structures // for use with the batch API. +// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#successful-responses type BatchResponse struct { Transfer string `json:"transfer,omitempty"` - Objects []*Representation `json:"objects"` + Objects []*ObjectResponse `json:"objects"` } // Representation is object metadata as seen by clients of the lfs server. -type Representation struct { +type ObjectResponse struct { Oid string `json:"oid"` Size int64 `json:"size"` - Actions map[string]*link `json:"actions"` + Actions map[string]*Link `json:"actions"` Error *ObjectError `json:"error,omitempty"` } -// ObjectError defines the JSON structure returned to the client in case of an error -type ObjectError struct { - Code int `json:"code"` - Message string `json:"message"` -} - -// link provides a structure used to build a hypermedia representation of an HTTP link. -type link struct { +// Link provides a structure used to build a hypermedia representation of an HTTP link. +type Link struct { Href string `json:"href"` Header map[string]string `json:"header,omitempty"` ExpiresAt time.Time `json:"expires_at,omitempty"` } -type Pointer struct { - Oid string `json:"oid"` - Size int64 `json:"size"` -} - -// BatchVars contains multiple RequestVars processed in one batch operation. -// https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md -type BatchVars struct { - Transfers []string `json:"transfers,omitempty"` - Operation string `json:"operation"` - Objects []*RequestVars `json:"objects"` -} - -// TODO replace BatchVars in Server -type BatchRequest struct { - Operation string `json:"operation"` - Transfers []string `json:"transfers,omitempty"` - Ref *Reference `json:"ref,omitempty"` - Objects []*LfsObject `json:"objects"` -} - -type Reference struct { - Name string `json:"name"` +// ObjectError defines the JSON structure returned to the client in case of an error +type ObjectError struct { + Code int `json:"code"` + Message string `json:"message"` } - -type LfsObject struct { - Oid string `json:"oid"` - Size int64 `json:"size"` -} \ No newline at end of file diff --git a/modules/lfs/transferadapter.go b/modules/lfs/transferadapter.go index cff70b9fca8a..6936efd60839 100644 --- a/modules/lfs/transferadapter.go +++ b/modules/lfs/transferadapter.go @@ -12,7 +12,7 @@ import ( type TransferAdapter interface { Name() string - Download(r *Representation) (io.ReadCloser, error) + Download(r *ObjectResponse) (io.ReadCloser, error) //Upload(reader io.Reader) error } @@ -24,7 +24,7 @@ func (a *BasicTransferAdapter) Name() string { return "basic" } -func (a *BasicTransferAdapter) Download(r *Representation) (io.ReadCloser, error) { +func (a *BasicTransferAdapter) Download(r *ObjectResponse) (io.ReadCloser, error) { download, ok := r.Actions["download"] if !ok { return nil, errors.New("Action 'download' not found") diff --git a/services/lfs/locks.go b/services/lfs/locks.go index cf62492c7e1e..c918bf1ddc95 100644 --- a/services/lfs/locks.go +++ b/services/lfs/locks.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/convert" + lfs_module "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" @@ -25,7 +26,7 @@ func checkIsValidRequest(ctx *context.Context) bool { return false } if !MetaMatcher(ctx.Req) { - log.Info("Attempt access LOCKs without accepting the correct media type: %s", metaMediaType) + log.Info("Attempt access LOCKs without accepting the correct media type: %s", lfs_module.MediaType) writeStatus(ctx, 400) return false } @@ -71,9 +72,9 @@ func GetListLockHandler(ctx *context.Context) { // Status is written in checkIsValidRequest return } - ctx.Resp.Header().Set("Content-Type", metaMediaType) + ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) - rv := unpack(ctx) + rv, _ := unpack(ctx) repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo) if err != nil { @@ -158,7 +159,7 @@ func PostLockHandler(ctx *context.Context) { // Status is written in checkIsValidRequest return } - ctx.Resp.Header().Set("Content-Type", metaMediaType) + ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) userName := ctx.Params("username") repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git") @@ -226,7 +227,7 @@ func VerifyLockHandler(ctx *context.Context) { // Status is written in checkIsValidRequest return } - ctx.Resp.Header().Set("Content-Type", metaMediaType) + ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) userName := ctx.Params("username") repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git") @@ -293,7 +294,7 @@ func UnLockHandler(ctx *context.Context) { // Status is written in checkIsValidRequest return } - ctx.Resp.Header().Set("Content-Type", metaMediaType) + ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) userName := ctx.Params("username") repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git") diff --git a/services/lfs/server.go b/services/lfs/server.go index e0b12d8755a9..6cee5ca77a08 100644 --- a/services/lfs/server.go +++ b/services/lfs/server.go @@ -15,7 +15,6 @@ import ( "regexp" "strconv" "strings" - "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" @@ -26,62 +25,13 @@ import ( "github.com/dgrijalva/jwt-go" ) -// RequestVars contain variables from the HTTP request. Variables from routing, json body decoding, and -// some headers are stored. -type RequestVars struct { - Oid string - Size int64 +// requestContext contain variables from the HTTP request. +type requestContext struct { User string - Password string Repo string Authorization string } -// BatchVars contains multiple RequestVars processed in one batch operation. -// https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md -type BatchVars struct { - Transfers []string `json:"transfers,omitempty"` - Operation string `json:"operation"` - Objects []*RequestVars `json:"objects"` -} - -const ( - metaMediaType = "application/vnd.git-lfs+json" -) - -// BatchResponse contains multiple object metadata Representation structures -// for use with the batch API. -type BatchResponse struct { - Transfer string `json:"transfer,omitempty"` - Objects []*Representation `json:"objects"` -} - -// Representation is object metadata as seen by clients of the lfs server. -type Representation struct { - Oid string `json:"oid"` - Size int64 `json:"size"` - Actions map[string]*link `json:"actions"` - Error *ObjectError `json:"error,omitempty"` -} - -// ObjectError defines the JSON structure returned to the client in case of an error -type ObjectError struct { - Code int `json:"code"` - Message string `json:"message"` -} - -// link provides a structure used to build a hypermedia representation of an HTTP link. -type link struct { - Href string `json:"href"` - Header map[string]string `json:"header,omitempty"` - ExpiresAt time.Time `json:"expires_at,omitempty"` -} - -type LfsObject struct { - Oid string `json:"oid"` - Size int64 `json:"size"` -} - // Claims is a JWT Token Claims type Claims struct { RepoID int64 @@ -91,13 +41,13 @@ type Claims struct { } // ObjectLink builds a URL linking to the object. -func (v *RequestVars) ObjectLink() string { - return setting.AppURL + path.Join(v.User, v.Repo+".git", "info/lfs/objects", v.Oid) +func (rc *requestContext) ObjectLink(oid string) string { + return setting.AppURL + path.Join(rc.User, rc.Repo+".git", "info/lfs/objects", oid) } // VerifyLink builds a URL for verifying the object. -func (v *RequestVars) VerifyLink() string { - return setting.AppURL + path.Join(v.User, v.Repo+".git", "info/lfs/verify") +func (rc *requestContext) VerifyLink() string { + return setting.AppURL + path.Join(rc.User, rc.Repo+".git", "info/lfs/verify") } var oidRegExp = regexp.MustCompile(`^[A-Fa-f0-9]+$`) @@ -131,28 +81,28 @@ func ObjectOidHandler(ctx *context.Context) { writeStatus(ctx, 404) } -func getAuthenticatedRepoAndMeta(ctx *context.Context, rv *RequestVars, requireWrite bool) (*models.LFSMetaObject, *models.Repository) { - if !isOidValid(rv.Oid) { - log.Info("Attempt to access invalid LFS OID[%s] in %s/%s", rv.Oid, rv.User, rv.Repo) +func getAuthenticatedRepoAndMeta(ctx *context.Context, rc *requestContext, p *lfs_module.Pointer, requireWrite bool) (*models.LFSMetaObject, *models.Repository) { + if !isOidValid(p.Oid) { + log.Info("Attempt to access invalid LFS OID[%s] in %s/%s", p.Oid, rc.User, rc.Repo) writeStatus(ctx, 404) return nil, nil } - repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo) + repository, err := models.GetRepositoryByOwnerAndName(rc.User, rc.Repo) if err != nil { - log.Error("Unable to get repository: %s/%s Error: %v", rv.User, rv.Repo, err) + log.Error("Unable to get repository: %s/%s Error: %v", rc.User, rc.Repo, err) writeStatus(ctx, 404) return nil, nil } - if !authenticate(ctx, repository, rv.Authorization, requireWrite) { + if !authenticate(ctx, repository, rc.Authorization, requireWrite) { requireAuth(ctx) return nil, nil } - meta, err := repository.GetLFSMetaObjectByOid(rv.Oid) + meta, err := repository.GetLFSMetaObjectByOid(p.Oid) if err != nil { - log.Error("Unable to get LFS OID[%s] Error: %v", rv.Oid, err) + log.Error("Unable to get LFS OID[%s] Error: %v", p.Oid, err) writeStatus(ctx, 404) return nil, nil } @@ -162,9 +112,9 @@ func getAuthenticatedRepoAndMeta(ctx *context.Context, rv *RequestVars, requireW // getContentHandler gets the content from the content store func getContentHandler(ctx *context.Context) { - rv := unpack(ctx) + rc, p := unpack(ctx) - meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, false) + meta, _ := getAuthenticatedRepoAndMeta(ctx, rc, p, false) if meta == nil { // Status already written in getAuthenticatedRepoAndMeta return @@ -229,19 +179,19 @@ func getContentHandler(ctx *context.Context) { // getMetaHandler retrieves metadata about the object func getMetaHandler(ctx *context.Context) { - rv := unpack(ctx) + rc, p := unpack(ctx) - meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, false) + meta, _ := getAuthenticatedRepoAndMeta(ctx, rc, p, false) if meta == nil { // Status already written in getAuthenticatedRepoAndMeta return } - ctx.Resp.Header().Set("Content-Type", metaMediaType) + ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) if ctx.Req.Method == "GET" { enc := json.NewEncoder(ctx.Resp) - if err := enc.Encode(Represent(rv, meta, true, false)); err != nil { + if err := enc.Encode(represent(rc, meta, true, false)); err != nil { log.Error("Failed to encode representation as json. Error: %v", err) } } @@ -258,51 +208,51 @@ func PostHandler(ctx *context.Context) { } if !MetaMatcher(ctx.Req) { - log.Info("Attempt to POST without accepting the correct media type: %s", metaMediaType) + log.Info("Attempt to POST without accepting the correct media type: %s", lfs_module.MediaType) writeStatus(ctx, 400) return } - rv := unpack(ctx) + rc, p := unpack(ctx) - repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo) + repository, err := models.GetRepositoryByOwnerAndName(rc.User, rc.Repo) if err != nil { - log.Error("Unable to get repository: %s/%s Error: %v", rv.User, rv.Repo, err) + log.Error("Unable to get repository: %s/%s Error: %v", rc.User, rc.Repo, err) writeStatus(ctx, 404) return } - if !authenticate(ctx, repository, rv.Authorization, true) { + if !authenticate(ctx, repository, rc.Authorization, true) { requireAuth(ctx) return } - if !isOidValid(rv.Oid) { - log.Info("Invalid LFS OID[%s] attempt to POST in %s/%s", rv.Oid, rv.User, rv.Repo) + if !isOidValid(p.Oid) { + log.Info("Invalid LFS OID[%s] attempt to POST in %s/%s", p.Oid, rc.User, rc.Repo) writeStatus(ctx, 404) return } - if setting.LFS.MaxFileSize > 0 && rv.Size > setting.LFS.MaxFileSize { - log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", rv.Oid, rv.Size, rv.User, rv.Repo, setting.LFS.MaxFileSize) + if setting.LFS.MaxFileSize > 0 && p.Size > setting.LFS.MaxFileSize { + log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", p.Oid, p.Size, rc.User, rc.Repo, setting.LFS.MaxFileSize) writeStatus(ctx, 413) return } - meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Oid: rv.Oid, Size: rv.Size, RepositoryID: repository.ID}) + meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Oid: p.Oid, Size: p.Size, RepositoryID: repository.ID}) if err != nil { - log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", rv.Oid, rv.Size, rv.User, rv.Repo, err) + log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", p.Oid, p.Size, rc.User, rc.Repo, err) writeStatus(ctx, 404) return } - ctx.Resp.Header().Set("Content-Type", metaMediaType) + ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) sentStatus := 202 contentStore := lfs_module.NewContetStore() - exist, err := contentStore.Exists(meta.AsPointer()) + exist, err := contentStore.Exists(p) if err != nil { - log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", rv.Oid, rv.User, rv.Repo, err) + log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", p.Oid, rc.User, rc.Repo, err) writeStatus(ctx, 500) return } @@ -312,7 +262,7 @@ func PostHandler(ctx *context.Context) { ctx.Resp.WriteHeader(sentStatus) enc := json.NewEncoder(ctx.Resp) - if err := enc.Encode(Represent(rv, meta, meta.Existing, true)); err != nil { + if err := enc.Encode(represent(rc, meta, meta.Existing, true)); err != nil { log.Error("Failed to encode representation as json. Error: %v", err) } logRequest(ctx.Req, sentStatus) @@ -327,25 +277,31 @@ func BatchHandler(ctx *context.Context) { } if !MetaMatcher(ctx.Req) { - log.Info("Attempt to BATCH without accepting the correct media type: %s", metaMediaType) + log.Info("Attempt to BATCH without accepting the correct media type: %s", lfs_module.MediaType) writeStatus(ctx, 400) return } bv := unpackbatch(ctx) - var responseObjects []*Representation + reqCtx := &requestContext{ + User: ctx.Params("username"), + Repo: strings.TrimSuffix(ctx.Params("reponame"), ".git"), + Authorization: ctx.Req.Header.Get("Authorization"), + } + + var responseObjects []*lfs_module.ObjectResponse // Create a response object for _, object := range bv.Objects { if !isOidValid(object.Oid) { - log.Info("Invalid LFS OID[%s] attempt to BATCH in %s/%s", object.Oid, object.User, object.Repo) + log.Info("Invalid LFS OID[%s] attempt to BATCH in %s/%s", object.Oid, reqCtx.User, reqCtx.Repo) continue } - repository, err := models.GetRepositoryByOwnerAndName(object.User, object.Repo) + repository, err := models.GetRepositoryByOwnerAndName(reqCtx.User, reqCtx.Repo) if err != nil { - log.Error("Unable to get repository: %s/%s Error: %v", object.User, object.Repo, err) + log.Error("Unable to get repository: %s/%s Error: %v", reqCtx.User, reqCtx.Repo, err) writeStatus(ctx, 404) return } @@ -355,7 +311,7 @@ func BatchHandler(ctx *context.Context) { requireWrite = true } - if !authenticate(ctx, repository, object.Authorization, requireWrite) { + if !authenticate(ctx, repository, reqCtx.Authorization, requireWrite) { requireAuth(ctx) return } @@ -366,18 +322,18 @@ func BatchHandler(ctx *context.Context) { if err == nil { // Object is found and exists exist, err := contentStore.Exists(meta.AsPointer()) if err != nil { - log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err) + log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, reqCtx.User, reqCtx.Repo, err) writeStatus(ctx, 500) return } if exist { - responseObjects = append(responseObjects, Represent(object, meta, true, false)) + responseObjects = append(responseObjects, represent(reqCtx, meta, true, false)) continue } } if requireWrite && setting.LFS.MaxFileSize > 0 && object.Size > setting.LFS.MaxFileSize { - log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", object.Oid, object.Size, object.User, object.Repo, setting.LFS.MaxFileSize) + log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", object.Oid, object.Size, reqCtx.User, reqCtx.Repo, setting.LFS.MaxFileSize) writeStatus(ctx, 413) return } @@ -387,19 +343,19 @@ func BatchHandler(ctx *context.Context) { if err == nil { exist, err := contentStore.Exists(meta.AsPointer()) if err != nil { - log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err) + log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, reqCtx.User, reqCtx.Repo, err) writeStatus(ctx, 500) return } - responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !exist)) + responseObjects = append(responseObjects, represent(reqCtx, meta, meta.Existing, !exist)) } else { - log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, object.User, object.Repo, err) + log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, reqCtx.User, reqCtx.Repo, err) } } - ctx.Resp.Header().Set("Content-Type", metaMediaType) + ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) - respobj := &BatchResponse{Objects: responseObjects} + respobj := &lfs_module.BatchResponse{Objects: responseObjects} enc := json.NewEncoder(ctx.Resp) if err := enc.Encode(respobj); err != nil { @@ -410,9 +366,9 @@ func BatchHandler(ctx *context.Context) { // PutHandler receives data from the client and puts it into the content store func PutHandler(ctx *context.Context) { - rv := unpack(ctx) + rc, p := unpack(ctx) - meta, repository := getAuthenticatedRepoAndMeta(ctx, rv, true) + meta, repository := getAuthenticatedRepoAndMeta(ctx, rc, p, true) if meta == nil { // Status already written in getAuthenticatedRepoAndMeta return @@ -428,8 +384,8 @@ func PutHandler(ctx *context.Context) { } else { fmt.Fprintf(ctx.Resp, `{"message":"Internal Server Error"}`) } - if _, err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil { - log.Error("Whilst removing metaobject for LFS OID[%s] due to preceding error there was another Error: %v", rv.Oid, err) + if _, err = repository.RemoveLFSMetaObjectByOid(p.Oid); err != nil { + log.Error("Whilst removing metaobject for LFS OID[%s] due to preceding error there was another Error: %v", p.Oid, err) } return } @@ -446,14 +402,14 @@ func VerifyHandler(ctx *context.Context) { } if !MetaMatcher(ctx.Req) { - log.Info("Attempt to VERIFY without accepting the correct media type: %s", metaMediaType) + log.Info("Attempt to VERIFY without accepting the correct media type: %s", lfs_module.MediaType) writeStatus(ctx, 400) return } - rv := unpack(ctx) + rc, p := unpack(ctx) - meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, true) + meta, _ := getAuthenticatedRepoAndMeta(ctx, rc, p, true) if meta == nil { // Status already written in getAuthenticatedRepoAndMeta return @@ -475,30 +431,30 @@ func VerifyHandler(ctx *context.Context) { logRequest(ctx.Req, 200) } -// Represent takes a RequestVars and Meta and turns it into a Representation suitable +// represent takes a requestContext and Meta and turns it into a ObjectResponse suitable // for json encoding -func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload bool) *Representation { - rep := &Representation{ +func represent(rc *requestContext, meta *models.LFSMetaObject, download, upload bool) *lfs_module.ObjectResponse { + rep := &lfs_module.ObjectResponse{ Oid: meta.Oid, Size: meta.Size, - Actions: make(map[string]*link), + Actions: make(map[string]*lfs_module.Link), } header := make(map[string]string) - if rv.Authorization == "" { + if rc.Authorization == "" { //https://github.com/github/git-lfs/issues/1088 header["Authorization"] = "Authorization: Basic dummy" } else { - header["Authorization"] = rv.Authorization + header["Authorization"] = rc.Authorization } if download { - rep.Actions["download"] = &link{Href: rv.ObjectLink(), Header: header} + rep.Actions["download"] = &lfs_module.Link{Href: rc.ObjectLink(meta.Oid), Header: header} } if upload { - rep.Actions["upload"] = &link{Href: rv.ObjectLink(), Header: header} + rep.Actions["upload"] = &lfs_module.Link{Href: rc.ObjectLink(meta.Oid), Header: header} } if upload && !download { @@ -509,55 +465,55 @@ func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload boo } // This is only needed to workaround https://github.com/git-lfs/git-lfs/issues/3662 - verifyHeader["Accept"] = metaMediaType + verifyHeader["Accept"] = lfs_module.MediaType - rep.Actions["verify"] = &link{Href: rv.VerifyLink(), Header: verifyHeader} + rep.Actions["verify"] = &lfs_module.Link{Href: rc.VerifyLink(), Header: verifyHeader} } return rep } // MetaMatcher provides a mux.MatcherFunc that only allows requests that contain -// an Accept header with the metaMediaType +// an Accept header with the lfs_module.MediaType func MetaMatcher(r *http.Request) bool { mediaParts := strings.Split(r.Header.Get("Accept"), ";") mt := mediaParts[0] - return mt == metaMediaType + return mt == lfs_module.MediaType } -func unpack(ctx *context.Context) *RequestVars { +func unpack(ctx *context.Context) (*requestContext, *lfs_module.Pointer) { r := ctx.Req - rv := &RequestVars{ + rc := &requestContext{ User: ctx.Params("username"), Repo: strings.TrimSuffix(ctx.Params("reponame"), ".git"), - Oid: ctx.Params("oid"), Authorization: r.Header.Get("Authorization"), } + p := &lfs_module.Pointer{Oid: ctx.Params("oid")} if r.Method == "POST" { // Maybe also check if +json - var o LfsObject + var p2 lfs_module.Pointer bodyReader := r.Body defer bodyReader.Close() dec := json.NewDecoder(bodyReader) - err := dec.Decode(&o) + err := dec.Decode(&p2) if err != nil { // The error is logged as a WARN here because this may represent misbehaviour rather than a true error - log.Warn("Unable to decode POST request vars for LFS OID[%s] in %s/%s: Error: %v", rv.Oid, rv.User, rv.Repo, err) - return rv + log.Warn("Unable to decode POST request vars for LFS OID[%s] in %s/%s: Error: %v", p.Oid, rc.User, rc.Repo, err) + return rc, p } - rv.Oid = o.Oid - rv.Size = o.Size + p.Oid = p2.Oid + p.Size = p2.Size } - return rv + return rc, p } // TODO cheap hack, unify with unpack -func unpackbatch(ctx *context.Context) *BatchVars { +func unpackbatch(ctx *context.Context) *lfs_module.BatchRequest { r := ctx.Req - var bv BatchVars + var bv lfs_module.BatchRequest bodyReader := r.Body defer bodyReader.Close() @@ -569,12 +525,6 @@ func unpackbatch(ctx *context.Context) *BatchVars { return &bv } - for i := 0; i < len(bv.Objects); i++ { - bv.Objects[i].User = ctx.Params("username") - bv.Objects[i].Repo = strings.TrimSuffix(ctx.Params("reponame"), ".git") - bv.Objects[i].Authorization = r.Header.Get("Authorization") - } - return &bv } From c6ce58c4a87c8195bb51f3851789489d88639404 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 18:22:00 +0000 Subject: [PATCH 11/69] Moved method to services. --- modules/lfs/repository.go | 55 -------------------------------- services/lfs/repository.go | 64 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 55 deletions(-) delete mode 100644 modules/lfs/repository.go create mode 100644 services/lfs/repository.go diff --git a/modules/lfs/repository.go b/modules/lfs/repository.go deleted file mode 100644 index da76e131b249..000000000000 --- a/modules/lfs/repository.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package lfs - -import ( - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/storage" -) - -func StoreMissingLfsObjectsInRepository(client *Client, repository *context.Repository) error { - contentStore := &ContentStore{ObjectStorage: storage.LFS} - - pointers, err := SearchPointerFiles(repository.GitRepo) - if err != nil { - return err - } - - for _, pointer := range pointers { - _, err := repository.Repository.GetLFSMetaObjectByOid(pointer.Oid) - if err != models.ErrLFSObjectNotExist { - continue - } - - // TODO What to do if file is too big / quota - - meta := &models.LFSMetaObject{Oid: pointer.Oid, Size: pointer.Size, RepositoryID: repository.Repository.ID} - - meta, err = models.NewLFSMetaObject(meta) - - exist, err := contentStore.Exists(meta) - if err != nil { - return err - } - if !exist { - lfsBaseUrl := "" // TODO - stream, err := client.Download(lfsBaseUrl, pointer.Oid, pointer.Size) - if err != nil { - return err - } - defer stream.Close() - - if err := contentStore.Put(meta, stream); err != nil { - if _, err2 := repository.Repository.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil { - return err2 - } - return err - } - } - } - - return nil -} \ No newline at end of file diff --git a/services/lfs/repository.go b/services/lfs/repository.go new file mode 100644 index 000000000000..dbdda6e7563d --- /dev/null +++ b/services/lfs/repository.go @@ -0,0 +1,64 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "net/http" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + lfs_module "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" +) + +func StoreMissingLfsObjectsInRepository(repo *models.Repository, gitRepo *git.Repository, lfsAddr string) error { + client := lfs_module.NewClient(&http.Client{}) + contentStore := lfs_module.NewContetStore() + + pointers, err := lfs_module.SearchPointerFiles(gitRepo) + if err != nil { + return err + } + + for _, pointer := range pointers { + meta := &models.LFSMetaObject{Oid: pointer.Oid, Size: pointer.Size, RepositoryID: repo.ID} + meta, err = models.NewLFSMetaObject(meta) + if err != nil { + return err + } + if meta.Existing { + continue + } + + log.Trace("LFS OID[%s] not present in repository %v", pointer.Oid, repo) + + exist, err := contentStore.Exists(pointer) + if err != nil { + return err + } + if !exist { + if setting.LFS.MaxFileSize > 0 && pointer.Size > setting.LFS.MaxFileSize { + log.Info("LFS OID[%s] download denied because of LFS_MAX_FILE_SIZE=%d < size %d", pointer.Oid, setting.LFS.MaxFileSize, pointer.Size) + continue + } + + stream, err := client.Download(lfsAddr, pointer.Oid, pointer.Size) + if err != nil { + return err + } + defer stream.Close() + + if err := contentStore.Put(pointer, stream); err != nil { + if _, err2 := repo.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil { + return err2 + } + return err + } + } + } + + return nil +} \ No newline at end of file From 7a86be84e2b0cc68b78cdc3b8a0f1f50db693c2c Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 18:29:00 +0000 Subject: [PATCH 12/69] Implemented lfs download on clone. --- modules/repository/repo.go | 56 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/modules/repository/repo.go b/modules/repository/repo.go index ede714673ab1..90ebb77a6f54 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -7,12 +7,14 @@ package repository import ( "context" "fmt" + "net/http" "path" "strings" "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" migration "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/setting" @@ -120,6 +122,10 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. log.Error("Failed to synchronize tags to releases for repository: %v", err) } } + + if err = storeMissingLfsObjectsInRepository(repo, gitRepo, opts.CloneAddr); err != nil { + log.Error("Failed to store missing LFS objects for repository: %v", err) + } } if err = repo.UpdateSize(models.DefaultDBContext()); err != nil { @@ -300,3 +306,53 @@ func PushUpdateAddTag(repo *models.Repository, gitRepo *git.Repository, tagName return models.SaveOrUpdateTag(repo, &rel) } + +// TODO use /services/lfs.StoreMissingLfsObjectsInRepository after migrating code to /services +func storeMissingLfsObjectsInRepository(repo *models.Repository, gitRepo *git.Repository, lfsAddr string) error { + client := lfs.NewClient(&http.Client{}) + contentStore := lfs.NewContetStore() + + pointers, err := lfs.SearchPointerFiles(gitRepo) + if err != nil { + return err + } + + for _, pointer := range pointers { + meta := &models.LFSMetaObject{Oid: pointer.Oid, Size: pointer.Size, RepositoryID: repo.ID} + meta, err = models.NewLFSMetaObject(meta) + if err != nil { + return err + } + if meta.Existing { + continue + } + + log.Trace("LFS OID[%s] not present in repository %v", pointer.Oid, repo) + + exist, err := contentStore.Exists(pointer) + if err != nil { + return err + } + if !exist { + if setting.LFS.MaxFileSize > 0 && pointer.Size > setting.LFS.MaxFileSize { + log.Info("LFS OID[%s] download denied because of LFS_MAX_FILE_SIZE=%d < size %d", pointer.Oid, setting.LFS.MaxFileSize, pointer.Size) + continue + } + + stream, err := client.Download(lfsAddr, pointer.Oid, pointer.Size) + if err != nil { + return err + } + defer stream.Close() + + if err := contentStore.Put(pointer, stream); err != nil { + if _, err2 := repo.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil { + return err2 + } + return err + } + } + } + + return nil +} \ No newline at end of file From 68e34cd70c5f90e2d4d3f4068cfbb71418b56841 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 18:34:00 +0000 Subject: [PATCH 13/69] Implemented LFS sync on clone and mirror update. --- modules/repository/repo.go | 6 ++-- services/lfs/repository.go | 64 -------------------------------------- services/mirror/mirror.go | 9 ++++-- 3 files changed, 10 insertions(+), 69 deletions(-) delete mode 100644 services/lfs/repository.go diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 90ebb77a6f54..a40d23234abe 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -123,7 +123,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. } } - if err = storeMissingLfsObjectsInRepository(repo, gitRepo, opts.CloneAddr); err != nil { + if err = StoreMissingLfsObjectsInRepository(repo, gitRepo, opts.CloneAddr); err != nil { log.Error("Failed to store missing LFS objects for repository: %v", err) } } @@ -307,8 +307,8 @@ func PushUpdateAddTag(repo *models.Repository, gitRepo *git.Repository, tagName return models.SaveOrUpdateTag(repo, &rel) } -// TODO use /services/lfs.StoreMissingLfsObjectsInRepository after migrating code to /services -func storeMissingLfsObjectsInRepository(repo *models.Repository, gitRepo *git.Repository, lfsAddr string) error { +// StoreMissingLfsObjectsInRepository downloads missing LFS objects +func StoreMissingLfsObjectsInRepository(repo *models.Repository, gitRepo *git.Repository, lfsAddr string) error { client := lfs.NewClient(&http.Client{}) contentStore := lfs.NewContetStore() diff --git a/services/lfs/repository.go b/services/lfs/repository.go deleted file mode 100644 index dbdda6e7563d..000000000000 --- a/services/lfs/repository.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package lfs - -import ( - "net/http" - - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/git" - lfs_module "code.gitea.io/gitea/modules/lfs" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" -) - -func StoreMissingLfsObjectsInRepository(repo *models.Repository, gitRepo *git.Repository, lfsAddr string) error { - client := lfs_module.NewClient(&http.Client{}) - contentStore := lfs_module.NewContetStore() - - pointers, err := lfs_module.SearchPointerFiles(gitRepo) - if err != nil { - return err - } - - for _, pointer := range pointers { - meta := &models.LFSMetaObject{Oid: pointer.Oid, Size: pointer.Size, RepositoryID: repo.ID} - meta, err = models.NewLFSMetaObject(meta) - if err != nil { - return err - } - if meta.Existing { - continue - } - - log.Trace("LFS OID[%s] not present in repository %v", pointer.Oid, repo) - - exist, err := contentStore.Exists(pointer) - if err != nil { - return err - } - if !exist { - if setting.LFS.MaxFileSize > 0 && pointer.Size > setting.LFS.MaxFileSize { - log.Info("LFS OID[%s] download denied because of LFS_MAX_FILE_SIZE=%d < size %d", pointer.Oid, setting.LFS.MaxFileSize, pointer.Size) - continue - } - - stream, err := client.Download(lfsAddr, pointer.Oid, pointer.Size) - if err != nil { - return err - } - defer stream.Close() - - if err := contentStore.Put(pointer, stream); err != nil { - if _, err2 := repo.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil { - return err2 - } - return err - } - } - } - - return nil -} \ No newline at end of file diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index e4981b8c00e6..d94d0b1e02ca 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -253,13 +253,18 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { log.Error("OpenRepository: %v", err) return nil, false } + defer gitRepo.Close() log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo) if err = repo_module.SyncReleasesWithTags(m.Repo, gitRepo); err != nil { - gitRepo.Close() log.Error("Failed to synchronize tags to releases for repository: %v", err) } - gitRepo.Close() + + log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) + lfsAddr, _ := remoteAddress(m.Repo.RepoPath()) + if err = repo_module.StoreMissingLfsObjectsInRepository(m.Repo, gitRepo, lfsAddr); err != nil { // TODO support custom endpoint + log.Error("Failed to synchronize LFS objects for repository: %v", err) + } log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo) if err := m.Repo.UpdateSize(models.DefaultDBContext()); err != nil { From 4775b8060d72ac9bba26ec345df6ceb5885cfb48 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 18:37:00 +0000 Subject: [PATCH 14/69] Added form fields. --- modules/forms/repo_form.go | 2 ++ modules/migrations/base/options.go | 2 ++ modules/migrations/gitea_uploader.go | 2 ++ modules/repository/repo.go | 6 ++++-- modules/structs/repo.go | 2 ++ routers/api/v1/repo/migrate.go | 2 ++ routers/repo/migrate.go | 4 ++++ templates/swagger/v1_json.tmpl | 16 ++++++++++++++++ 8 files changed, 34 insertions(+), 2 deletions(-) diff --git a/modules/forms/repo_form.go b/modules/forms/repo_form.go index ac968a1dd5b6..28c5b74641f4 100644 --- a/modules/forms/repo_form.go +++ b/modules/forms/repo_form.go @@ -73,6 +73,8 @@ type MigrateRepoForm struct { // required: true RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"` Mirror bool `json:"mirror"` + LFS bool `json:"lfs"` + LFSEndpoint string `json:"lfs_endpoint"` Private bool `json:"private"` Description string `json:"description" binding:"MaxSize(255)"` Wiki bool `json:"wiki"` diff --git a/modules/migrations/base/options.go b/modules/migrations/base/options.go index 168f9848c813..f1d9f81e575b 100644 --- a/modules/migrations/base/options.go +++ b/modules/migrations/base/options.go @@ -20,6 +20,8 @@ type MigrateOptions struct { // required: true RepoName string `json:"repo_name" binding:"Required"` Mirror bool `json:"mirror"` + LFS bool `json:"lfs"` + LFSEndpoint string `json:"lfs_endpoint"` Private bool `json:"private"` Description string `json:"description"` OriginalURL string diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go index 3be49b5c6ced..dc6dba112548 100644 --- a/modules/migrations/gitea_uploader.go +++ b/modules/migrations/gitea_uploader.go @@ -117,6 +117,8 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate OriginalURL: repo.OriginalURL, GitServiceType: opts.GitServiceType, Mirror: repo.IsMirror, + LFS: opts.LFS, + LFSEndpoint: opts.LFSEndpoint, CloneAddr: repo.CloneURL, Private: repo.IsPrivate, Wiki: opts.Wiki, diff --git a/modules/repository/repo.go b/modules/repository/repo.go index a40d23234abe..c60a4303b566 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -123,8 +123,10 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. } } - if err = StoreMissingLfsObjectsInRepository(repo, gitRepo, opts.CloneAddr); err != nil { - log.Error("Failed to store missing LFS objects for repository: %v", err) + if opts.LFS { + if err = StoreMissingLfsObjectsInRepository(repo, gitRepo, opts.LFSEndpoint); err != nil { + log.Error("Failed to store missing LFS objects for repository: %v", err) + } } } diff --git a/modules/structs/repo.go b/modules/structs/repo.go index d588813b2188..fbb9944a6075 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -253,6 +253,8 @@ type MigrateRepoOptions struct { AuthToken string `json:"auth_token"` Mirror bool `json:"mirror"` + LFS bool `json:"lfs"` + LFSEndpoint string `json:"lfs_endpoint"` Private bool `json:"private"` Description string `json:"description" binding:"MaxSize(255)"` Wiki bool `json:"wiki"` diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index 61cd12b991cb..ddfdb1711259 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -134,6 +134,8 @@ func Migrate(ctx *context.APIContext) { Description: form.Description, Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror, + LFS: form.LFS, + LFSEndpoint: form.LFSEndpoint, AuthUsername: form.AuthUsername, AuthPassword: form.AuthPassword, AuthToken: form.AuthToken, diff --git a/routers/repo/migrate.go b/routers/repo/migrate.go index 89452de0fabf..fa6def10dc67 100644 --- a/routers/repo/migrate.go +++ b/routers/repo/migrate.go @@ -47,6 +47,8 @@ func Migrate(ctx *context.Context) { ctx.Data["IsForcedPrivate"] = setting.Repository.ForcePrivate ctx.Data["DisableMirrors"] = setting.Repository.DisableMirrors ctx.Data["mirror"] = ctx.Query("mirror") == "1" + ctx.Data["LFS"] = ctx.Query("lfs") == "1" + ctx.Data["LFSEndpoint"] = ctx.Query("lfs_endpoint") ctx.Data["wiki"] = ctx.Query("wiki") == "1" ctx.Data["milestones"] = ctx.Query("milestones") == "1" ctx.Data["labels"] = ctx.Query("labels") == "1" @@ -172,6 +174,8 @@ func MigratePost(ctx *context.Context) { Description: form.Description, Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror && !setting.Repository.DisableMirrors, + LFS: form.LFS, + LFSEndpoint: form.LFSEndpoint, AuthUsername: form.AuthUsername, AuthPassword: form.AuthPassword, AuthToken: form.AuthToken, diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 5a3be37b4aec..3a9dc8d3b4c5 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -14458,6 +14458,14 @@ "type": "boolean", "x-go-name": "Labels" }, + "lfs": { + "type": "boolean", + "x-go-name": "LFS" + }, + "lfs_endpoint": { + "type": "string", + "x-go-name": "LFSEndpoint" + }, "milestones": { "type": "boolean", "x-go-name": "Milestones" @@ -14537,6 +14545,14 @@ "type": "boolean", "x-go-name": "Labels" }, + "lfs": { + "type": "boolean", + "x-go-name": "LFS" + }, + "lfs_endpoint": { + "type": "string", + "x-go-name": "LFSEndpoint" + }, "milestones": { "type": "boolean", "x-go-name": "Milestones" From e458d6452109da49ef7f308c5ce8dfac693b3770 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 18:40:00 +0000 Subject: [PATCH 15/69] Updated templates. --- options/locale/locale_en-US.ini | 3 ++- templates/repo/migrate/git.tmpl | 14 +------------- templates/repo/migrate/gitea.tmpl | 16 ++-------------- templates/repo/migrate/github.tmpl | 16 ++-------------- templates/repo/migrate/gitlab.tmpl | 16 ++-------------- templates/repo/migrate/gogs.tmpl | 16 ++-------------- templates/repo/migrate/options.tmpl | 26 ++++++++++++++++++++++++++ web_src/js/features/migration.js | 4 ++++ 8 files changed, 41 insertions(+), 70 deletions(-) create mode 100644 templates/repo/migrate/options.tmpl diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index a4b677e43baf..bf071999be0e 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -754,6 +754,8 @@ migrate_options = Migration Options migrate_service = Migration Service migrate_options_mirror_helper = This repository will be a mirror migrate_options_mirror_disabled = Your site administrator has disabled new mirrors. +migrate_options_lfs = Migrate LFS files +migrate_options_lfs_endpoint = Migration will attempt to use your Git remote to determine the LFS server. You can also configure a custom LFS server. migrate_items = Migration Items migrate_items_wiki = Wiki migrate_items_milestones = Milestones @@ -769,7 +771,6 @@ migrate.clone_local_path = or a local server path migrate.permission_denied = You are not allowed to import local repositories. migrate.invalid_local_path = "The local path is invalid. It does not exist or is not a directory." migrate.failed = Migration failed: %v -migrate.lfs_mirror_unsupported = Mirroring LFS objects is not supported - use 'git lfs fetch --all' and 'git lfs push --all' instead. migrate.migrate_items_options = Access Token is required to migrate additional items migrated_from = Migrated from %[2]s migrated_from_fake = Migrated From %[1]s diff --git a/templates/repo/migrate/git.tmpl b/templates/repo/migrate/git.tmpl index 7fb8f3d585d9..0145777504e9 100644 --- a/templates/repo/migrate/git.tmpl +++ b/templates/repo/migrate/git.tmpl @@ -15,7 +15,6 @@ {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} - {{if .LFSActive}}
{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
@@ -28,18 +27,7 @@
-
- -
- {{if .DisableMirrors}} - - - {{else}} - - - {{end}} -
-
+ {{template "repo/migrate/options" .}}
diff --git a/templates/repo/migrate/gitea.tmpl b/templates/repo/migrate/gitea.tmpl index 4ad6e6024fae..d8dd69a7acc2 100644 --- a/templates/repo/migrate/gitea.tmpl +++ b/templates/repo/migrate/gitea.tmpl @@ -15,28 +15,16 @@ {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} - {{if .LFSActive}}
{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
- {{svg "octicon-question"}} + {{svg "octicon-question"}}
-
- -
- {{if .DisableMirrors}} - - - {{else}} - - - {{end}} -
-
+ {{template "repo/migrate/options" .}} {{.i18n.Tr "repo.migrate.migrate_items_options"}}
diff --git a/templates/repo/migrate/github.tmpl b/templates/repo/migrate/github.tmpl index c31444aaef26..eb918e717c41 100644 --- a/templates/repo/migrate/github.tmpl +++ b/templates/repo/migrate/github.tmpl @@ -15,28 +15,16 @@ {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} - {{if .LFSActive}}
{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
- {{svg "octicon-question"}} + {{svg "octicon-question"}}
-
- -
- {{if .DisableMirrors}} - - - {{else}} - - - {{end}} -
-
+ {{template "repo/migrate/options" .}} {{.i18n.Tr "repo.migrate.migrate_items_options"}}
diff --git a/templates/repo/migrate/gitlab.tmpl b/templates/repo/migrate/gitlab.tmpl index 823bf25de2f2..ca9ef1551f37 100644 --- a/templates/repo/migrate/gitlab.tmpl +++ b/templates/repo/migrate/gitlab.tmpl @@ -15,28 +15,16 @@ {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} - {{if .LFSActive}}
{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
- {{svg "octicon-question"}} + {{svg "octicon-question"}}
-
- -
- {{if .DisableMirrors}} - - - {{else}} - - - {{end}} -
-
+ {{template "repo/migrate/options" .}} {{.i18n.Tr "repo.migrate.migrate_items_options"}}
diff --git a/templates/repo/migrate/gogs.tmpl b/templates/repo/migrate/gogs.tmpl index dd33fd013893..d22f3afa68dc 100644 --- a/templates/repo/migrate/gogs.tmpl +++ b/templates/repo/migrate/gogs.tmpl @@ -15,28 +15,16 @@ {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} - {{if .LFSActive}}
{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
- +
-
- -
- {{if .DisableMirrors}} - - - {{else}} - - - {{end}} -
-
+ {{template "repo/migrate/options" .}} {{.i18n.Tr "repo.migrate.migrate_items_options"}}
diff --git a/templates/repo/migrate/options.tmpl b/templates/repo/migrate/options.tmpl new file mode 100644 index 000000000000..3990ba0b8ce9 --- /dev/null +++ b/templates/repo/migrate/options.tmpl @@ -0,0 +1,26 @@ +
+ +
+ {{if .DisableMirrors}} + + + {{else}} + + + {{end}} +
+
+{{if .LFSActive}} +
+ +
+ + +
+
+{{.i18n.Tr "repo.migrate_options_lfs_endpoint"}} +
+ + +
+{{end}} \ No newline at end of file diff --git a/web_src/js/features/migration.js b/web_src/js/features/migration.js index 09ab49b3e1e4..d707027167ec 100644 --- a/web_src/js/features/migration.js +++ b/web_src/js/features/migration.js @@ -3,6 +3,8 @@ const $user = $('#auth_username'); const $pass = $('#auth_password'); const $token = $('#auth_token'); const $mirror = $('#mirror'); +const $lfs = $('#lfs'); +const $lfsEndpoint = $('#lfs_endpoint'); const $items = $('#migrate_items').find('input[type=checkbox]'); export default function initMigration() { @@ -12,12 +14,14 @@ export default function initMigration() { $pass.on('keyup', () => {checkItems(false)}); $token.on('keyup', () => {checkItems(true)}); $mirror.on('change', () => {checkItems(true)}); + $lfs.on('change', () => {$lfsEndpoint.attr('disabled', !$lfs.is(':checked'))}); const $cloneAddr = $('#clone_addr'); $cloneAddr.on('change', () => { const $repoName = $('#repo_name'); if ($cloneAddr.val().length > 0 && $repoName.val().length === 0) { // Only modify if repo_name input is blank $repoName.val($cloneAddr.val().match(/^(.*\/)?((.+?)(\.git)?)$/)[3]); + $lfsEndpoint.attr('placeholder', $cloneAddr.val()); } }); } From 4d2abc3d8cbcf6d5384ce7b8795c8292f4e52b94 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 18:42:00 +0000 Subject: [PATCH 16/69] Fixed condition. --- web_src/js/features/migration.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/web_src/js/features/migration.js b/web_src/js/features/migration.js index d707027167ec..597d32728249 100644 --- a/web_src/js/features/migration.js +++ b/web_src/js/features/migration.js @@ -19,8 +19,10 @@ export default function initMigration() { const $cloneAddr = $('#clone_addr'); $cloneAddr.on('change', () => { const $repoName = $('#repo_name'); - if ($cloneAddr.val().length > 0 && $repoName.val().length === 0) { // Only modify if repo_name input is blank - $repoName.val($cloneAddr.val().match(/^(.*\/)?((.+?)(\.git)?)$/)[3]); + if ($cloneAddr.val().length > 0) { + if ($repoName.val().length === 0) { // Only modify if repo_name input is blank + $repoName.val($cloneAddr.val().match(/^(.*\/)?((.+?)(\.git)?)$/)[3]); + } $lfsEndpoint.attr('placeholder', $cloneAddr.val()); } }); From a5e64dd43a370f1b08486fd49ed7fd758789e73e Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 18 Feb 2021 18:44:00 +0000 Subject: [PATCH 17/69] Use alternate endpoint. --- modules/repository/repo.go | 7 +++++-- routers/api/v1/repo/migrate.go | 7 ++++++- routers/repo/migrate.go | 10 +++++++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/modules/repository/repo.go b/modules/repository/repo.go index c60a4303b566..1af0bd18d98a 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -329,7 +329,7 @@ func StoreMissingLfsObjectsInRepository(repo *models.Repository, gitRepo *git.Re continue } - log.Trace("LFS OID[%s] not present in repository %v", pointer.Oid, repo) + log.Trace("LFS OID[%s] not present in repository %s", pointer.Oid, repo.FullName()) exist, err := contentStore.Exists(pointer) if err != nil { @@ -343,7 +343,8 @@ func StoreMissingLfsObjectsInRepository(repo *models.Repository, gitRepo *git.Re stream, err := client.Download(lfsAddr, pointer.Oid, pointer.Size) if err != nil { - return err + log.Error("LFS OID[%s] failed to download: %v", err) + continue } defer stream.Close() @@ -353,6 +354,8 @@ func StoreMissingLfsObjectsInRepository(repo *models.Repository, gitRepo *git.Re } return err } + } else { + log.Trace("LFS OID[%s] already present in content store", pointer.Oid) } } diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index ddfdb1711259..9cc4880b9f86 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -128,6 +128,11 @@ func Migrate(ctx *context.APIContext) { return } + lfsEndpoint := remoteAddr + if len(form.LFSEndpoint) > 0 { + lfsEndpoint = form.LFSEndpoint + } + var opts = migrations.MigrateOptions{ CloneAddr: remoteAddr, RepoName: form.RepoName, @@ -135,7 +140,7 @@ func Migrate(ctx *context.APIContext) { Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror, LFS: form.LFS, - LFSEndpoint: form.LFSEndpoint, + LFSEndpoint: lfsEndpoint, AuthUsername: form.AuthUsername, AuthPassword: form.AuthPassword, AuthToken: form.AuthToken, diff --git a/routers/repo/migrate.go b/routers/repo/migrate.go index fa6def10dc67..c5230b9cf5a2 100644 --- a/routers/repo/migrate.go +++ b/routers/repo/migrate.go @@ -47,8 +47,7 @@ func Migrate(ctx *context.Context) { ctx.Data["IsForcedPrivate"] = setting.Repository.ForcePrivate ctx.Data["DisableMirrors"] = setting.Repository.DisableMirrors ctx.Data["mirror"] = ctx.Query("mirror") == "1" - ctx.Data["LFS"] = ctx.Query("lfs") == "1" - ctx.Data["LFSEndpoint"] = ctx.Query("lfs_endpoint") + ctx.Data["lfs"] = ctx.Query("lfs") == "1" ctx.Data["wiki"] = ctx.Query("wiki") == "1" ctx.Data["milestones"] = ctx.Query("milestones") == "1" ctx.Data["labels"] = ctx.Query("labels") == "1" @@ -166,6 +165,11 @@ func MigratePost(ctx *context.Context) { return } + lfsEndpoint := remoteAddr + if len(form.LFSEndpoint) > 0 { + lfsEndpoint = form.LFSEndpoint + } + var opts = migrations.MigrateOptions{ OriginalURL: form.CloneAddr, GitServiceType: structs.GitServiceType(form.Service), @@ -175,7 +179,7 @@ func MigratePost(ctx *context.Context) { Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror && !setting.Repository.DisableMirrors, LFS: form.LFS, - LFSEndpoint: form.LFSEndpoint, + LFSEndpoint: lfsEndpoint, AuthUsername: form.AuthUsername, AuthPassword: form.AuthPassword, AuthToken: form.AuthToken, From 5615d1e113d3eba557256c12d1afc9bceade2bdf Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 19 Feb 2021 14:56:37 +0000 Subject: [PATCH 18/69] Added missing methods. --- modules/lfs/pointer_scanner_nogogit.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/lfs/pointer_scanner_nogogit.go b/modules/lfs/pointer_scanner_nogogit.go index e639ca875f0a..b5dc12ff7723 100644 --- a/modules/lfs/pointer_scanner_nogogit.go +++ b/modules/lfs/pointer_scanner_nogogit.go @@ -7,9 +7,19 @@ package lfs import ( + "io" + "code.gitea.io/gitea/modules/git" ) +func TryReadPointer(reader io.Reader) *Pointer { + return nil +} + +func TryReadPointerFromBuffer(buf []byte) *Pointer { + return nil +} + func SearchPointerFiles(repo *git.Repository) ([]*Pointer, error) { - return []*Pointer, nil + return []*Pointer{}, nil } \ No newline at end of file From a6b0d26cc2b2bc7d9aa0b9ee4bd0b598d58cf510 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 19 Feb 2021 15:46:30 +0000 Subject: [PATCH 19/69] Fixed typo and make linter happy. --- integrations/lfs_getobject_test.go | 7 ++++--- models/lfs.go | 1 + modules/lfs/client.go | 20 +++++++++++++------- modules/lfs/content_store.go | 7 +++++-- modules/lfs/pointer_scanner.go | 2 ++ modules/lfs/pointer_scanner_nogogit.go | 5 ++++- modules/lfs/shared.go | 5 ++++- modules/lfs/transferadapter.go | 8 ++++++-- modules/repofiles/update.go | 2 +- modules/repofiles/upload.go | 2 +- modules/repository/repo.go | 4 ++-- routers/repo/lfs.go | 2 +- routers/repo/view.go | 14 +++++++------- services/lfs/server.go | 10 +++++----- services/pull/lfs.go | 2 +- 15 files changed, 57 insertions(+), 34 deletions(-) diff --git a/integrations/lfs_getobject_test.go b/integrations/lfs_getobject_test.go index e5e7cd15fb5e..47ea64a1be99 100644 --- a/integrations/lfs_getobject_test.go +++ b/integrations/lfs_getobject_test.go @@ -49,11 +49,12 @@ func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string lfsID++ lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject) assert.NoError(t, err) - contentStore := lfs.NewContetStore() - exist, err := contentStore.Exists(lfsMetaObject) + pointer := lfsMetaObject.AsPointer() + contentStore := lfs.NewContentStore() + exist, err := contentStore.Exists(pointer) assert.NoError(t, err) if !exist { - err := contentStore.Put(lfsMetaObject, bytes.NewReader(*content)) + err := contentStore.Put(pointer, bytes.NewReader(*content)) assert.NoError(t, err) } return oid diff --git a/models/lfs.go b/models/lfs.go index b108caa4b341..506e3cd761d1 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -42,6 +42,7 @@ func (m *LFSMetaObject) Pointer() string { return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size) } +// AsPointer creates a Pointer with Oid and Size func (m *LFSMetaObject) AsPointer() *lfs.Pointer { pointer := &lfs.Pointer{Oid: m.Oid, Size: m.Size} return pointer diff --git a/modules/lfs/client.go b/modules/lfs/client.go index 06fbdc5de8c9..d62ad2a3c3ef 100644 --- a/modules/lfs/client.go +++ b/modules/lfs/client.go @@ -14,11 +14,13 @@ import ( "strings" ) +// Client is used to communicate with the LFS server type Client struct { client *http.Client transfers map[string]TransferAdapter } +// NewClient creates a LFS client func NewClient(hc *http.Client) *Client { client := &Client{hc, make(map[string]TransferAdapter)} @@ -41,13 +43,16 @@ func (c *Client) transferNames() []string { return keys } -func (c *Client) batch(repositoryUrl, operation string, objects []*Pointer) (*BatchResponse, error) { - url := fmt.Sprintf("%s.git/info/lfs/objects/batch", strings.TrimSuffix(repositoryUrl, ".git")) +func (c *Client) batch(url, operation string, objects []*Pointer) (*BatchResponse, error) { + url = fmt.Sprintf("%s.git/info/lfs/objects/batch", strings.TrimSuffix(url, ".git")) request := &BatchRequest{operation, c.transferNames(), nil, objects} payload := new(bytes.Buffer) - json.NewEncoder(payload).Encode(request) + err := json.NewEncoder(payload).Encode(request) + if err != nil { + return nil, err + } req, err := http.NewRequest("POST", url, payload) if err != nil { @@ -63,7 +68,7 @@ func (c *Client) batch(repositoryUrl, operation string, objects []*Pointer) (*Ba defer res.Body.Close() if res.StatusCode != http.StatusOK { - return nil, errors.New(fmt.Sprintf("Unexpected servers response: %s", res.Status)) + return nil, fmt.Errorf("Unexpected servers response: %s", res.Status) } var response BatchResponse @@ -79,11 +84,12 @@ func (c *Client) batch(repositoryUrl, operation string, objects []*Pointer) (*Ba return &response, nil } -func (c *Client) Download(repositoryUrl, oid string, size int64) (io.ReadCloser, error) { +// Download reads the specific LFS object from the LFS server +func (c *Client) Download(url, oid string, size int64) (io.ReadCloser, error) { var objects []*Pointer objects = append(objects, &Pointer{oid, size}) - - result, err := c.batch(repositoryUrl, "download", objects) + + result, err := c.batch(url, "download", objects) if err != nil { return nil, err } diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index ffcad3ae951b..d0a86a96b571 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -18,7 +18,9 @@ import ( ) var ( + // ErrHashMismatch occurs if the content has does not match OID ErrHashMismatch = errors.New("Content hash does not match OID") + // ErrSizeMismatch occurs if the content size does not match ErrSizeMismatch = errors.New("Content size does not match") ) @@ -36,7 +38,8 @@ type ContentStore struct { storage.ObjectStorage } -func NewContetStore() *ContentStore { +// NewContentStore creates the default ContentStore +func NewContentStore() *ContentStore { contentStore := &ContentStore{ObjectStorage: storage.LFS} return contentStore } @@ -128,6 +131,6 @@ func (s *ContentStore) Verify(pointer *Pointer) (bool, error) { // ReadMetaObject will read a models.LFSMetaObject and return a reader func ReadMetaObject(pointer *Pointer) (io.ReadCloser, error) { - contentStore := NewContetStore() + contentStore := NewContentStore() return contentStore.Get(pointer, 0) } diff --git a/modules/lfs/pointer_scanner.go b/modules/lfs/pointer_scanner.go index b15b5ccdc72e..a8011b2fd9d3 100644 --- a/modules/lfs/pointer_scanner.go +++ b/modules/lfs/pointer_scanner.go @@ -29,6 +29,7 @@ const ( LFSMetaFileOidPrefix = "oid sha256:" ) +// TryReadPointer tries to read LFS pointer data from the reader func TryReadPointer(reader io.Reader) *Pointer { buf := make([]byte, blobSizeCutoff) n, _ := reader.Read(buf) @@ -58,6 +59,7 @@ func TryReadPointerFromBuffer(buf []byte) *Pointer { return &Pointer{Oid: oid, Size: size} } +// SearchPointerFiles scans the whole repository for LFS pointer files func SearchPointerFiles(repo *git.Repository) ([]*Pointer, error) { gitRepo := repo.GoGitRepo() diff --git a/modules/lfs/pointer_scanner_nogogit.go b/modules/lfs/pointer_scanner_nogogit.go index b5dc12ff7723..7b1e24902a7d 100644 --- a/modules/lfs/pointer_scanner_nogogit.go +++ b/modules/lfs/pointer_scanner_nogogit.go @@ -12,14 +12,17 @@ import ( "code.gitea.io/gitea/modules/git" ) +// TryReadPointer not implemented func TryReadPointer(reader io.Reader) *Pointer { return nil } +// TryReadPointerFromBuffer not implemented func TryReadPointerFromBuffer(buf []byte) *Pointer { return nil } +// SearchPointerFiles not implemented func SearchPointerFiles(repo *git.Repository) ([]*Pointer, error) { return []*Pointer{}, nil -} \ No newline at end of file +} diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go index 0ba1d8669fca..64eac2aab143 100644 --- a/modules/lfs/shared.go +++ b/modules/lfs/shared.go @@ -9,6 +9,7 @@ import ( ) const ( + // MediaType contains the media type for LFS server requests MediaType = "application/vnd.git-lfs+json" ) @@ -21,11 +22,13 @@ type BatchRequest struct { Objects []*Pointer `json:"objects"` } +// Reference contains a git reference. // https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#ref-property type Reference struct { Name string `json:"name"` } +// Pointer contains LFS pointer data type Pointer struct { Oid string `json:"oid"` Size int64 `json:"size"` @@ -39,7 +42,7 @@ type BatchResponse struct { Objects []*ObjectResponse `json:"objects"` } -// Representation is object metadata as seen by clients of the lfs server. +// ObjectResponse is object metadata as seen by clients of the LFS server. type ObjectResponse struct { Oid string `json:"oid"` Size int64 `json:"size"` diff --git a/modules/lfs/transferadapter.go b/modules/lfs/transferadapter.go index 6936efd60839..5145f395fd4b 100644 --- a/modules/lfs/transferadapter.go +++ b/modules/lfs/transferadapter.go @@ -10,20 +10,24 @@ import ( "net/http" ) +// TransferAdapter represents an adapter for downloading/uploading LFS objects type TransferAdapter interface { Name() string Download(r *ObjectResponse) (io.ReadCloser, error) //Upload(reader io.Reader) error - } +} +// BasicTransferAdapter implements the "basic" adapter type BasicTransferAdapter struct { client *http.Client } +// Name returns the name of the adapter func (a *BasicTransferAdapter) Name() string { return "basic" } +// Download reads the download location and downloads the data func (a *BasicTransferAdapter) Download(r *ObjectResponse) (io.ReadCloser, error) { download, ok := r.Actions["download"] if !ok { @@ -37,7 +41,7 @@ func (a *BasicTransferAdapter) Download(r *ObjectResponse) (io.ReadCloser, error for key, value := range download.Header { req.Header.Set(key, value) } - + res, err := a.client.Do(req) if err != nil { return nil, err diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index 4ccdbc066f2f..feff7d948534 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -430,7 +430,7 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up if err != nil { return nil, err } - contentStore := lfs.NewContetStore() + contentStore := lfs.NewContentStore() exist, err := contentStore.Exists(lfsMetaObject.AsPointer()) if err != nil { return nil, err diff --git a/modules/repofiles/upload.go b/modules/repofiles/upload.go index 4e6b4b8a85ea..58e28213922b 100644 --- a/modules/repofiles/upload.go +++ b/modules/repofiles/upload.go @@ -164,7 +164,7 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep // OK now we can insert the data into the store - there's no way to clean up the store // once it's in there, it's in there. - contentStore := lfs.NewContetStore() + contentStore := lfs.NewContentStore() for _, uploadInfo := range infos { if uploadInfo.lfsMetaObject == nil { continue diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 1af0bd18d98a..28dc54db5a03 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -312,7 +312,7 @@ func PushUpdateAddTag(repo *models.Repository, gitRepo *git.Repository, tagName // StoreMissingLfsObjectsInRepository downloads missing LFS objects func StoreMissingLfsObjectsInRepository(repo *models.Repository, gitRepo *git.Repository, lfsAddr string) error { client := lfs.NewClient(&http.Client{}) - contentStore := lfs.NewContetStore() + contentStore := lfs.NewContentStore() pointers, err := lfs.SearchPointerFiles(gitRepo) if err != nil { @@ -360,4 +360,4 @@ func StoreMissingLfsObjectsInRepository(repo *models.Repository, gitRepo *git.Re } return nil -} \ No newline at end of file +} diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index 24cf866719ba..837ec2fcd61b 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -493,7 +493,7 @@ type pointerResult struct { func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) { defer wg.Done() defer catFileBatchReader.Close() - contentStore := lfs.NewContetStore() + contentStore := lfs.NewContentStore() bufferedReader := bufio.NewReader(catFileBatchReader) buf := make([]byte, 1025) diff --git a/routers/repo/view.go b/routers/repo/view.go index be5127d07064..8bb000959375 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -283,7 +283,7 @@ func renderDirectory(ctx *context.Context, treeLink string) { if meta != nil { ctx.Data["IsLFSFile"] = true isLFSFile = true - + // OK read the lfs object var err error dataRc, err = lfs.ReadMetaObject(pointer) @@ -292,7 +292,7 @@ func renderDirectory(ctx *context.Context, treeLink string) { return } defer dataRc.Close() - + buf = make([]byte, 1024) n, err = dataRc.Read(buf) if err != nil { @@ -300,10 +300,10 @@ func renderDirectory(ctx *context.Context, treeLink string) { return } buf = buf[:n] - + isTextFile = base.IsTextFile(buf) ctx.Data["IsTextFile"] = isTextFile - + fileSize = meta.Size ctx.Data["FileSize"] = meta.Size filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name)) @@ -407,7 +407,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st } if meta != nil { isLFSFile = true - + // OK read the lfs object var err error dataRc, err = lfs.ReadMetaObject(pointer) @@ -416,7 +416,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st return } defer dataRc.Close() - + buf = make([]byte, 1024) n, err = dataRc.Read(buf) // Error EOF don't mean there is an error, it just means we read to @@ -426,7 +426,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st return } buf = buf[:n] - + isTextFile = base.IsTextFile(buf) fileSize = meta.Size ctx.Data["RawFileLink"] = fmt.Sprintf("%s/media/%s/%s", ctx.Repo.RepoLink, ctx.Repo.BranchNameSubURL(), ctx.Repo.TreePath) diff --git a/services/lfs/server.go b/services/lfs/server.go index 6cee5ca77a08..7cfda06f50fd 100644 --- a/services/lfs/server.go +++ b/services/lfs/server.go @@ -143,7 +143,7 @@ func getContentHandler(ctx *context.Context) { } } - contentStore := lfs_module.NewContetStore() + contentStore := lfs_module.NewContentStore() content, err := contentStore.Get(meta.AsPointer(), fromByte) if err != nil { var rerr *lfs_module.ErrRangeNotSatisfiable @@ -249,7 +249,7 @@ func PostHandler(ctx *context.Context) { ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType) sentStatus := 202 - contentStore := lfs_module.NewContetStore() + contentStore := lfs_module.NewContentStore() exist, err := contentStore.Exists(p) if err != nil { log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", p.Oid, rc.User, rc.Repo, err) @@ -316,7 +316,7 @@ func BatchHandler(ctx *context.Context) { return } - contentStore := lfs_module.NewContetStore() + contentStore := lfs_module.NewContentStore() meta, err := repository.GetLFSMetaObjectByOid(object.Oid) if err == nil { // Object is found and exists @@ -374,7 +374,7 @@ func PutHandler(ctx *context.Context) { return } - contentStore := lfs_module.NewContetStore() + contentStore := lfs_module.NewContentStore() defer ctx.Req.Body.Close() if err := contentStore.Put(meta.AsPointer(), ctx.Req.Body); err != nil { // Put will log the error itself @@ -415,7 +415,7 @@ func VerifyHandler(ctx *context.Context) { return } - contentStore := lfs_module.NewContetStore() + contentStore := lfs_module.NewContentStore() ok, err := contentStore.Verify(meta.AsPointer()) if err != nil { // Error will be logged in Verify diff --git a/services/pull/lfs.go b/services/pull/lfs.go index 0499de199461..54c1b7f88967 100644 --- a/services/pull/lfs.go +++ b/services/pull/lfs.go @@ -106,7 +106,7 @@ func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg continue } - contentStore := lfs.NewContetStore() + contentStore := lfs.NewContentStore() exist, _ := contentStore.Exists(pointer) if !exist { continue From dd889c651d28471c32f63ff232b07f7b7a69fe37 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 19 Feb 2021 19:08:54 +0000 Subject: [PATCH 20/69] Detached pointer parser from gogit dependency. --- modules/lfs/pointer.go | 54 ++++++++++++++++++++++++++ modules/lfs/pointer_scanner.go | 47 ---------------------- modules/lfs/pointer_scanner_nogogit.go | 12 ------ 3 files changed, 54 insertions(+), 59 deletions(-) create mode 100644 modules/lfs/pointer.go diff --git a/modules/lfs/pointer.go b/modules/lfs/pointer.go new file mode 100644 index 000000000000..25de62b7f066 --- /dev/null +++ b/modules/lfs/pointer.go @@ -0,0 +1,54 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "io" + "strconv" + "strings" +) + +const ( + blobSizeCutoff = 1024 + + // TODO remove duplicate from models + + // LFSMetaFileIdentifier is the string appearing at the first line of LFS pointer files. + // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md + LFSMetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" + + // LFSMetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash. + LFSMetaFileOidPrefix = "oid sha256:" +) + +// TryReadPointer tries to read LFS pointer data from the reader +func TryReadPointer(reader io.Reader) *Pointer { + buf := make([]byte, blobSizeCutoff) + n, _ := reader.Read(buf) + buf = buf[:n] + + return TryReadPointerFromBuffer(buf) +} + +// TryReadPointerFromBuffer will return a pointer if the provided byte slice is a pointer file or nil otherwise. +func TryReadPointerFromBuffer(buf []byte) *Pointer { + headString := string(buf) + if !strings.HasPrefix(headString, LFSMetaFileIdentifier) { + return nil + } + + splitLines := strings.Split(headString, "\n") + if len(splitLines) < 3 { + return nil + } + + oid := strings.TrimPrefix(splitLines[1], LFSMetaFileOidPrefix) + size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) + if len(oid) != 64 || err != nil { + return nil + } + + return &Pointer{Oid: oid, Size: size} +} diff --git a/modules/lfs/pointer_scanner.go b/modules/lfs/pointer_scanner.go index a8011b2fd9d3..f1e06a94ef65 100644 --- a/modules/lfs/pointer_scanner.go +++ b/modules/lfs/pointer_scanner.go @@ -7,58 +7,11 @@ package lfs import ( - "io" - "strconv" - "strings" - "code.gitea.io/gitea/modules/git" "github.com/go-git/go-git/v5/plumbing/object" ) -const ( - blobSizeCutoff = 1024 - - // TODO remove duplicate from models - - // LFSMetaFileIdentifier is the string appearing at the first line of LFS pointer files. - // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md - LFSMetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" - - // LFSMetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash. - LFSMetaFileOidPrefix = "oid sha256:" -) - -// TryReadPointer tries to read LFS pointer data from the reader -func TryReadPointer(reader io.Reader) *Pointer { - buf := make([]byte, blobSizeCutoff) - n, _ := reader.Read(buf) - buf = buf[:n] - - return TryReadPointerFromBuffer(buf) -} - -// TryReadPointerFromBuffer will return a pointer if the provided byte slice is a pointer file or nil otherwise. -func TryReadPointerFromBuffer(buf []byte) *Pointer { - headString := string(buf) - if !strings.HasPrefix(headString, LFSMetaFileIdentifier) { - return nil - } - - splitLines := strings.Split(headString, "\n") - if len(splitLines) < 3 { - return nil - } - - oid := strings.TrimPrefix(splitLines[1], LFSMetaFileOidPrefix) - size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) - if len(oid) != 64 || err != nil { - return nil - } - - return &Pointer{Oid: oid, Size: size} -} - // SearchPointerFiles scans the whole repository for LFS pointer files func SearchPointerFiles(repo *git.Repository) ([]*Pointer, error) { gitRepo := repo.GoGitRepo() diff --git a/modules/lfs/pointer_scanner_nogogit.go b/modules/lfs/pointer_scanner_nogogit.go index 7b1e24902a7d..d0751382f86d 100644 --- a/modules/lfs/pointer_scanner_nogogit.go +++ b/modules/lfs/pointer_scanner_nogogit.go @@ -7,21 +7,9 @@ package lfs import ( - "io" - "code.gitea.io/gitea/modules/git" ) -// TryReadPointer not implemented -func TryReadPointer(reader io.Reader) *Pointer { - return nil -} - -// TryReadPointerFromBuffer not implemented -func TryReadPointerFromBuffer(buf []byte) *Pointer { - return nil -} - // SearchPointerFiles not implemented func SearchPointerFiles(repo *git.Repository) ([]*Pointer, error) { return []*Pointer{}, nil From 86acab3a0a2e7bb39e9919478b33b249175d1249 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 19 Feb 2021 23:22:54 +0000 Subject: [PATCH 21/69] Fixed TestGetLFSRange test. --- modules/lfs/content_store.go | 6 ++++++ services/lfs/server.go | 4 +--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index d0a86a96b571..f013ae0d2b9f 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -29,6 +29,12 @@ type ErrRangeNotSatisfiable struct { FromByte int64 } +// IsErrRangeNotSatisfiable returns true if the error is an ErrRangeNotSatisfiable +func IsErrRangeNotSatisfiable(err error) bool { + _, ok := err.(ErrRangeNotSatisfiable) + return ok +} + func (err ErrRangeNotSatisfiable) Error() string { return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte) } diff --git a/services/lfs/server.go b/services/lfs/server.go index 7cfda06f50fd..bc7bf521ec0b 100644 --- a/services/lfs/server.go +++ b/services/lfs/server.go @@ -7,7 +7,6 @@ package lfs import ( "encoding/base64" "encoding/json" - "errors" "fmt" "io" "net/http" @@ -146,8 +145,7 @@ func getContentHandler(ctx *context.Context) { contentStore := lfs_module.NewContentStore() content, err := contentStore.Get(meta.AsPointer(), fromByte) if err != nil { - var rerr *lfs_module.ErrRangeNotSatisfiable - if errors.As(err, &rerr) { + if lfs_module.IsErrRangeNotSatisfiable(err) { writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable) } else { // Errors are logged in contentStore.Get From a95199265e3028998d8489ce8e03ba87bc699ee7 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 20 Feb 2021 14:28:49 +0000 Subject: [PATCH 22/69] Added context to support cancellation. --- modules/lfs/client.go | 16 +++++++++++----- modules/lfs/transferadapter.go | 14 ++++++++++---- modules/repository/repo.go | 19 ++++++++++++++++--- services/mirror/mirror.go | 10 +++++----- services/mirror/mirror_test.go | 8 +++++--- 5 files changed, 47 insertions(+), 20 deletions(-) diff --git a/modules/lfs/client.go b/modules/lfs/client.go index d62ad2a3c3ef..ffd5e07b1eb0 100644 --- a/modules/lfs/client.go +++ b/modules/lfs/client.go @@ -6,6 +6,7 @@ package lfs import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -43,7 +44,7 @@ func (c *Client) transferNames() []string { return keys } -func (c *Client) batch(url, operation string, objects []*Pointer) (*BatchResponse, error) { +func (c *Client) batch(ctx context.Context, url, operation string, objects []*Pointer) (*BatchResponse, error) { url = fmt.Sprintf("%s.git/info/lfs/objects/batch", strings.TrimSuffix(url, ".git")) request := &BatchRequest{operation, c.transferNames(), nil, objects} @@ -54,7 +55,7 @@ func (c *Client) batch(url, operation string, objects []*Pointer) (*BatchRespons return nil, err } - req, err := http.NewRequest("POST", url, payload) + req, err := http.NewRequestWithContext(ctx, "POST", url, payload) if err != nil { return nil, err } @@ -63,6 +64,11 @@ func (c *Client) batch(url, operation string, objects []*Pointer) (*BatchRespons res, err := c.client.Do(req) if err != nil { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } return nil, err } defer res.Body.Close() @@ -85,11 +91,11 @@ func (c *Client) batch(url, operation string, objects []*Pointer) (*BatchRespons } // Download reads the specific LFS object from the LFS server -func (c *Client) Download(url, oid string, size int64) (io.ReadCloser, error) { +func (c *Client) Download(ctx context.Context, url, oid string, size int64) (io.ReadCloser, error) { var objects []*Pointer objects = append(objects, &Pointer{oid, size}) - result, err := c.batch(url, "download", objects) + result, err := c.batch(ctx, url, "download", objects) if err != nil { return nil, err } @@ -103,7 +109,7 @@ func (c *Client) Download(url, oid string, size int64) (io.ReadCloser, error) { return nil, errors.New("No objects in result") } - content, err := transferAdapter.Download(result.Objects[0]) + content, err := transferAdapter.Download(ctx, result.Objects[0]) if err != nil { return nil, err } diff --git a/modules/lfs/transferadapter.go b/modules/lfs/transferadapter.go index 5145f395fd4b..428710869ff4 100644 --- a/modules/lfs/transferadapter.go +++ b/modules/lfs/transferadapter.go @@ -5,6 +5,7 @@ package lfs import ( + "context" "errors" "io" "net/http" @@ -13,8 +14,8 @@ import ( // TransferAdapter represents an adapter for downloading/uploading LFS objects type TransferAdapter interface { Name() string - Download(r *ObjectResponse) (io.ReadCloser, error) - //Upload(reader io.Reader) error + Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) + //Upload(ctx context.Context, reader io.Reader) error } // BasicTransferAdapter implements the "basic" adapter @@ -28,13 +29,13 @@ func (a *BasicTransferAdapter) Name() string { } // Download reads the download location and downloads the data -func (a *BasicTransferAdapter) Download(r *ObjectResponse) (io.ReadCloser, error) { +func (a *BasicTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) { download, ok := r.Actions["download"] if !ok { return nil, errors.New("Action 'download' not found") } - req, err := http.NewRequest("GET", download.Href, nil) + req, err := http.NewRequestWithContext(ctx, "GET", download.Href, nil) if err != nil { return nil, err } @@ -44,6 +45,11 @@ func (a *BasicTransferAdapter) Download(r *ObjectResponse) (io.ReadCloser, error res, err := a.client.Do(req) if err != nil { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } return nil, err } diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 28dc54db5a03..8eb6a7a7e4ca 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -124,7 +124,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. } if opts.LFS { - if err = StoreMissingLfsObjectsInRepository(repo, gitRepo, opts.LFSEndpoint); err != nil { + if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, opts.LFSEndpoint); err != nil { log.Error("Failed to store missing LFS objects for repository: %v", err) } } @@ -310,7 +310,7 @@ func PushUpdateAddTag(repo *models.Repository, gitRepo *git.Repository, tagName } // StoreMissingLfsObjectsInRepository downloads missing LFS objects -func StoreMissingLfsObjectsInRepository(repo *models.Repository, gitRepo *git.Repository, lfsAddr string) error { +func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, lfsAddr string) error { client := lfs.NewClient(&http.Client{}) contentStore := lfs.NewContentStore() @@ -320,6 +320,13 @@ func StoreMissingLfsObjectsInRepository(repo *models.Repository, gitRepo *git.Re } for _, pointer := range pointers { + select { + case <-ctx.Done(): + log.Info("StoreMissingLfsObjectsInRepository aborted before completion") + return nil + default: + } + meta := &models.LFSMetaObject{Oid: pointer.Oid, Size: pointer.Size, RepositoryID: repo.ID} meta, err = models.NewLFSMetaObject(meta) if err != nil { @@ -341,8 +348,14 @@ func StoreMissingLfsObjectsInRepository(repo *models.Repository, gitRepo *git.Re continue } - stream, err := client.Download(lfsAddr, pointer.Oid, pointer.Size) + stream, err := client.Download(ctx, lfsAddr, pointer.Oid, pointer.Size) if err != nil { + select { + case <-ctx.Done(): + log.Info("StoreMissingLfsObjectsInRepository aborted before completion") + return nil + default: + } log.Error("LFS OID[%s] failed to download: %v", err) continue } diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index d94d0b1e02ca..bb26e4514703 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -206,7 +206,7 @@ func parseRemoteUpdateOutput(output string) []*mirrorSyncResult { } // runSync returns true if sync finished without error. -func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { +func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) { repoPath := m.Repo.RepoPath() wikiPath := m.Repo.WikiPath() timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second @@ -262,7 +262,7 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) lfsAddr, _ := remoteAddress(m.Repo.RepoPath()) - if err = repo_module.StoreMissingLfsObjectsInRepository(m.Repo, gitRepo, lfsAddr); err != nil { // TODO support custom endpoint + if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, lfsAddr); err != nil { // TODO support custom endpoint log.Error("Failed to synchronize LFS objects for repository: %v", err) } @@ -383,12 +383,12 @@ func SyncMirrors(ctx context.Context) { mirrorQueue.Close() return case repoID := <-mirrorQueue.Queue(): - syncMirror(repoID) + syncMirror(ctx, repoID) } } } -func syncMirror(repoID string) { +func syncMirror(ctx context.Context, repoID string) { log.Trace("SyncMirrors [repo_id: %v]", repoID) defer func() { err := recover() @@ -408,7 +408,7 @@ func syncMirror(repoID string) { } log.Trace("SyncMirrors [repo: %-v]: Running Sync", m.Repo) - results, ok := runSync(m) + results, ok := runSync(ctx, m) if !ok { return } diff --git a/services/mirror/mirror_test.go b/services/mirror/mirror_test.go index ddfb6c676b6a..daef957248b0 100644 --- a/services/mirror/mirror_test.go +++ b/services/mirror/mirror_test.go @@ -48,7 +48,9 @@ func TestRelease_MirrorDelete(t *testing.T) { }) assert.NoError(t, err) - mirror, err := repository.MigrateRepositoryGitData(context.Background(), user, mirrorRepo, opts) + ctx := context.Background() + + mirror, err := repository.MigrateRepositoryGitData(ctx, user, mirrorRepo, opts) assert.NoError(t, err) gitRepo, err := git.OpenRepository(repoPath) @@ -74,7 +76,7 @@ func TestRelease_MirrorDelete(t *testing.T) { err = mirror.GetMirror() assert.NoError(t, err) - _, ok := runSync(mirror.Mirror) + _, ok := runSync(ctx, mirror.Mirror) assert.True(t, ok) count, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions) @@ -85,7 +87,7 @@ func TestRelease_MirrorDelete(t *testing.T) { assert.NoError(t, err) assert.NoError(t, release_service.DeleteReleaseByID(release.ID, user, true)) - _, ok = runSync(mirror.Mirror) + _, ok = runSync(ctx, mirror.Mirror) assert.True(t, ok) count, err = models.GetReleaseCountByRepoID(mirror.ID, findOptions) From 3b69a7d2bbf6533c75b35c72f18403fcc5e65cce Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 21 Feb 2021 18:09:21 +0000 Subject: [PATCH 23/69] Use ReadFull to probably read more data. --- modules/lfs/pointer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/lfs/pointer.go b/modules/lfs/pointer.go index 25de62b7f066..ac25b94cad2d 100644 --- a/modules/lfs/pointer.go +++ b/modules/lfs/pointer.go @@ -26,7 +26,7 @@ const ( // TryReadPointer tries to read LFS pointer data from the reader func TryReadPointer(reader io.Reader) *Pointer { buf := make([]byte, blobSizeCutoff) - n, _ := reader.Read(buf) + n, _ := io.ReadFull(reader, buf) buf = buf[:n] return TryReadPointerFromBuffer(buf) From ceefad039df46fd001e11cff1b8b36acb6144632 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 21 Feb 2021 18:15:53 +0000 Subject: [PATCH 24/69] Removed duplicated code from models. --- integrations/git_test.go | 5 +++-- models/lfs.go | 11 +---------- modules/lfs/pointer.go | 14 ++++++-------- services/gitdiff/gitdiff.go | 7 ++++--- 4 files changed, 14 insertions(+), 23 deletions(-) diff --git a/integrations/git_test.go b/integrations/git_test.go index c3c11268296f..508e45bb8a57 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" @@ -215,7 +216,7 @@ func rawTest(t *testing.T, ctx *APITestContext, little, big, littleLFS, bigLFS s req = NewRequest(t, "GET", path.Join("/", username, reponame, "/raw/branch/master/", littleLFS)) resp = session.MakeRequest(t, req, http.StatusOK) assert.NotEqual(t, littleSize, resp.Body.Len()) - assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) + assert.Contains(t, resp.Body.String(), lfs.MetaFileIdentifier) } if !testing.Short() { @@ -227,7 +228,7 @@ func rawTest(t *testing.T, ctx *APITestContext, little, big, littleLFS, bigLFS s req = NewRequest(t, "GET", path.Join("/", username, reponame, "/raw/branch/master/", bigLFS)) resp = session.MakeRequest(t, req, http.StatusOK) assert.NotEqual(t, bigSize, resp.Body.Len()) - assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier) + assert.Contains(t, resp.Body.String(), lfs.MetaFileIdentifier) } } }) diff --git a/models/lfs.go b/models/lfs.go index 506e3cd761d1..5b2661d0f7ad 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -39,7 +39,7 @@ func (m *LFSMetaObject) RelativePath() string { // Pointer returns the string representation of an LFS pointer file func (m *LFSMetaObject) Pointer() string { - return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size) + return fmt.Sprintf("%s\n%s%s\nsize %d\n", lfs.MetaFileIdentifier, lfs.MetaFileOidPrefix, m.Oid, m.Size) } // AsPointer creates a Pointer with Oid and Size @@ -62,15 +62,6 @@ var ( ErrLFSObjectNotExist = errors.New("LFS Meta object does not exist") ) -const ( - // LFSMetaFileIdentifier is the string appearing at the first line of LFS pointer files. - // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md - LFSMetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" - - // LFSMetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash. - LFSMetaFileOidPrefix = "oid sha256:" -) - // NewLFSMetaObject stores a given populated LFSMetaObject structure in the database // if it is not already present. func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) { diff --git a/modules/lfs/pointer.go b/modules/lfs/pointer.go index ac25b94cad2d..7279457225b2 100644 --- a/modules/lfs/pointer.go +++ b/modules/lfs/pointer.go @@ -13,14 +13,12 @@ import ( const ( blobSizeCutoff = 1024 - // TODO remove duplicate from models - - // LFSMetaFileIdentifier is the string appearing at the first line of LFS pointer files. + // MetaFileIdentifier is the string appearing at the first line of LFS pointer files. // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md - LFSMetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" + MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" - // LFSMetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash. - LFSMetaFileOidPrefix = "oid sha256:" + // MetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash. + MetaFileOidPrefix = "oid sha256:" ) // TryReadPointer tries to read LFS pointer data from the reader @@ -35,7 +33,7 @@ func TryReadPointer(reader io.Reader) *Pointer { // TryReadPointerFromBuffer will return a pointer if the provided byte slice is a pointer file or nil otherwise. func TryReadPointerFromBuffer(buf []byte) *Pointer { headString := string(buf) - if !strings.HasPrefix(headString, LFSMetaFileIdentifier) { + if !strings.HasPrefix(headString, MetaFileIdentifier) { return nil } @@ -44,7 +42,7 @@ func TryReadPointerFromBuffer(buf []byte) *Pointer { return nil } - oid := strings.TrimPrefix(splitLines[1], LFSMetaFileOidPrefix) + oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix) size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) if len(oid) != 64 || err != nil { return nil diff --git a/services/gitdiff/gitdiff.go b/services/gitdiff/gitdiff.go index d5c3923516ec..8fdb81e00e76 100644 --- a/services/gitdiff/gitdiff.go +++ b/services/gitdiff/gitdiff.go @@ -25,6 +25,7 @@ import ( "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/highlight" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" @@ -1015,10 +1016,10 @@ func parseHunks(curFile *DiffFile, maxLines, maxLineCharacters int, input *bufio curSection.Lines[len(curSection.Lines)-1].Content = line // handle LFS - if line[1:] == models.LFSMetaFileIdentifier { + if line[1:] == lfs.MetaFileIdentifier { curFileLFSPrefix = true - } else if curFileLFSPrefix && strings.HasPrefix(line[1:], models.LFSMetaFileOidPrefix) { - oid := strings.TrimPrefix(line[1:], models.LFSMetaFileOidPrefix) + } else if curFileLFSPrefix && strings.HasPrefix(line[1:], lfs.MetaFileOidPrefix) { + oid := strings.TrimPrefix(line[1:], lfs.MetaFileOidPrefix) if len(oid) == 64 { m := &models.LFSMetaObject{Oid: oid} count, err := models.Count(m) From ef86fa8f6aa8cdca8f868fdc0957157a435833b6 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 21 Feb 2021 19:42:06 +0000 Subject: [PATCH 25/69] Moved scan implementation into pointer_scanner_nogogit. --- modules/lfs/pointer_scanner.go | 6 +- modules/lfs/pointer_scanner_nogogit.go | 99 ++++++++++++- modules/lfs/shared.go | 6 + modules/repository/repo.go | 20 +-- routers/repo/lfs.go | 197 ++++++++----------------- 5 files changed, 180 insertions(+), 148 deletions(-) diff --git a/modules/lfs/pointer_scanner.go b/modules/lfs/pointer_scanner.go index f1e06a94ef65..bd0b9ef57a2e 100644 --- a/modules/lfs/pointer_scanner.go +++ b/modules/lfs/pointer_scanner.go @@ -13,7 +13,7 @@ import ( ) // SearchPointerFiles scans the whole repository for LFS pointer files -func SearchPointerFiles(repo *git.Repository) ([]*Pointer, error) { +func SearchPointerFiles(repo *git.Repository) ([]PointerBlob, error) { gitRepo := repo.GoGitRepo() blobs, err := gitRepo.BlobObjects() @@ -21,7 +21,7 @@ func SearchPointerFiles(repo *git.Repository) ([]*Pointer, error) { return nil, err } - var pointers []*Pointer + var pointers []PointerBlob err = blobs.ForEach(func(blob *object.Blob) error { if blob.Size > blobSizeCutoff { @@ -36,7 +36,7 @@ func SearchPointerFiles(repo *git.Repository) ([]*Pointer, error) { pointer := TryReadPointer(reader) if pointer != nil { - pointers = append(pointers, pointer) + pointers = append(pointers, PointerBlob{Hash: blob.Hash.String(), Pointer: pointer}) } return nil diff --git a/modules/lfs/pointer_scanner_nogogit.go b/modules/lfs/pointer_scanner_nogogit.go index d0751382f86d..c08fabdbaa4d 100644 --- a/modules/lfs/pointer_scanner_nogogit.go +++ b/modules/lfs/pointer_scanner_nogogit.go @@ -7,10 +7,103 @@ package lfs import ( + "bufio" + "io" + "strconv" + "sync" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/pipeline" ) -// SearchPointerFiles not implemented -func SearchPointerFiles(repo *git.Repository) ([]*Pointer, error) { - return []*Pointer{}, nil +// SearchPointerFiles scans the whole repository for LFS pointer files +func SearchPointerFiles(repo *git.Repository) ([]PointerBlob, error) { + basePath := repo.Path + + pointerChan := make(chan PointerBlob) + + catFileCheckReader, catFileCheckWriter := io.Pipe() + shasToBatchReader, shasToBatchWriter := io.Pipe() + catFileBatchReader, catFileBatchWriter := io.Pipe() + errChan := make(chan error, 1) + wg := sync.WaitGroup{} + wg.Add(5) + + pointers := make([]PointerBlob, 0, 50) + + go func() { + defer wg.Done() + for pointer := range pointerChan { + pointers = append(pointers, pointer) + } + }() + go createPointerResultsFromCatFileBatch(catFileBatchReader, &wg, pointerChan) + go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath) + go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg) + if git.CheckGitVersionAtLeast("2.6.0") != nil { + revListReader, revListWriter := io.Pipe() + shasToCheckReader, shasToCheckWriter := io.Pipe() + wg.Add(2) + go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath) + go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg) + go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan) + } else { + go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan) + } + wg.Wait() + + select { + case err, has := <-errChan: + if has { + return nil, err + } + default: + } + + return pointers, nil } + +func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- PointerBlob) { + defer wg.Done() + defer catFileBatchReader.Close() + + bufferedReader := bufio.NewReader(catFileBatchReader) + buf := make([]byte, 1025) + for { + // File descriptor line: sha + sha, err := bufferedReader.ReadString(' ') + if err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + // Throw away the blob + if _, err := bufferedReader.ReadString(' '); err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + sizeStr, err := bufferedReader.ReadString('\n') + if err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1]) + if err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + pointerBuf := buf[:size+1] + if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil { + _ = catFileBatchReader.CloseWithError(err) + break + } + pointerBuf = pointerBuf[:size] + // Now we need to check if the pointerBuf is an LFS pointer + pointer := TryReadPointerFromBuffer(pointerBuf) + if pointer == nil { + continue + } + + pointerChan <- PointerBlob{Hash: sha, Pointer: pointer} + } + close(pointerChan) +} \ No newline at end of file diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go index 64eac2aab143..70c1a5a54496 100644 --- a/modules/lfs/shared.go +++ b/modules/lfs/shared.go @@ -62,3 +62,9 @@ type ObjectError struct { Code int `json:"code"` Message string `json:"message"` } + +// PointerBlob associates a Git blob with a Pointer. +type PointerBlob struct { + Hash string + Pointer *Pointer +} \ No newline at end of file diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 8eb6a7a7e4ca..6d5cd4e49714 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -314,12 +314,12 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi client := lfs.NewClient(&http.Client{}) contentStore := lfs.NewContentStore() - pointers, err := lfs.SearchPointerFiles(gitRepo) + pointerBlobs, err := lfs.SearchPointerFiles(gitRepo) if err != nil { return err } - for _, pointer := range pointers { + for _, pointerBlob := range pointerBlobs { select { case <-ctx.Done(): log.Info("StoreMissingLfsObjectsInRepository aborted before completion") @@ -327,7 +327,7 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi default: } - meta := &models.LFSMetaObject{Oid: pointer.Oid, Size: pointer.Size, RepositoryID: repo.ID} + meta := &models.LFSMetaObject{Oid: pointerBlob.Pointer.Oid, Size: pointerBlob.Pointer.Size, RepositoryID: repo.ID} meta, err = models.NewLFSMetaObject(meta) if err != nil { return err @@ -336,19 +336,19 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi continue } - log.Trace("LFS OID[%s] not present in repository %s", pointer.Oid, repo.FullName()) + log.Trace("LFS OID[%s] not present in repository %s", pointerBlob.Pointer.Oid, repo.FullName()) - exist, err := contentStore.Exists(pointer) + exist, err := contentStore.Exists(pointerBlob.Pointer) if err != nil { return err } if !exist { - if setting.LFS.MaxFileSize > 0 && pointer.Size > setting.LFS.MaxFileSize { - log.Info("LFS OID[%s] download denied because of LFS_MAX_FILE_SIZE=%d < size %d", pointer.Oid, setting.LFS.MaxFileSize, pointer.Size) + if setting.LFS.MaxFileSize > 0 && pointerBlob.Pointer.Size > setting.LFS.MaxFileSize { + log.Info("LFS OID[%s] download denied because of LFS_MAX_FILE_SIZE=%d < size %d", pointerBlob.Pointer.Oid, setting.LFS.MaxFileSize, pointerBlob.Pointer.Size) continue } - stream, err := client.Download(ctx, lfsAddr, pointer.Oid, pointer.Size) + stream, err := client.Download(ctx, lfsAddr, pointerBlob.Pointer.Oid, pointerBlob.Pointer.Size) if err != nil { select { case <-ctx.Done(): @@ -361,14 +361,14 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi } defer stream.Close() - if err := contentStore.Put(pointer, stream); err != nil { + if err := contentStore.Put(pointerBlob.Pointer, stream); err != nil { if _, err2 := repo.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil { return err2 } return err } } else { - log.Trace("LFS OID[%s] already present in content store", pointer.Oid) + log.Trace("LFS OID[%s] already present in content store", pointerBlob.Pointer.Oid) } } diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index 837ec2fcd61b..cdd034c56664 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -5,7 +5,6 @@ package repo import ( - "bufio" "bytes" "fmt" gotemplate "html/template" @@ -14,7 +13,6 @@ import ( "path" "strconv" "strings" - "sync" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" @@ -420,158 +418,93 @@ func LFSPointerFiles(ctx *context.Context) { } ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs" - basePath := ctx.Repo.Repository.RepoPath() + err = func() error { + pointerBlobs, err := lfs.SearchPointerFiles(ctx.Repo.GitRepo) + if err != nil { + return err + } + + numPointers := len(pointerBlobs) + var numAssociated, numNoExist, numAssociatable int + + type pointerResult struct { + SHA string + Oid string + Size int64 + InRepo bool + Exists bool + Accessible bool + } - pointerChan := make(chan pointerResult) + results := make([]pointerResult, numPointers) - catFileCheckReader, catFileCheckWriter := io.Pipe() - shasToBatchReader, shasToBatchWriter := io.Pipe() - catFileBatchReader, catFileBatchWriter := io.Pipe() - errChan := make(chan error, 1) - wg := sync.WaitGroup{} - wg.Add(5) + contentStore := lfs.NewContentStore() + repo := ctx.Repo.Repository - var numPointers, numAssociated, numNoExist, numAssociatable int + for i, pointerBlob := range pointerBlobs { + result := pointerResult{ + SHA: pointerBlob.Hash, + Oid: pointerBlob.Pointer.Oid, + Size: pointerBlob.Pointer.Size, + } - go func() { - defer wg.Done() - pointers := make([]pointerResult, 0, 50) - for pointer := range pointerChan { - pointers = append(pointers, pointer) - if pointer.InRepo { + if _, err := repo.GetLFSMetaObjectByOid(pointerBlob.Pointer.Oid); err != nil { + if err != models.ErrLFSObjectNotExist { + return err + } + } else { + result.InRepo = true + } + + result.Exists, err = contentStore.Exists(pointerBlob.Pointer) + if err != nil { + return err + } + + if result.Exists { + if !result.InRepo { + // Can we fix? + // OK well that's "simple" + // - we need to check whether current user has access to a repo that has access to the file + result.Accessible, err = models.LFSObjectAccessible(ctx.User, pointerBlob.Pointer.Oid) + if err != nil { + return err + } + } else { + result.Accessible = true + } + } + + if result.InRepo { numAssociated++ } - if !pointer.Exists { + if !result.Exists { numNoExist++ } - if !pointer.InRepo && pointer.Accessible { + if !result.InRepo && result.Accessible { numAssociatable++ } + + results[i] = result } - numPointers = len(pointers) - ctx.Data["Pointers"] = pointers + + ctx.Data["Pointers"] = results ctx.Data["NumPointers"] = numPointers ctx.Data["NumAssociated"] = numAssociated ctx.Data["NumAssociatable"] = numAssociatable ctx.Data["NumNoExist"] = numNoExist ctx.Data["NumNotAssociated"] = numPointers - numAssociated + + return nil }() - go createPointerResultsFromCatFileBatch(catFileBatchReader, &wg, pointerChan, ctx.Repo.Repository, ctx.User) - go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath) - go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg) - if git.CheckGitVersionAtLeast("2.6.0") != nil { - revListReader, revListWriter := io.Pipe() - shasToCheckReader, shasToCheckWriter := io.Pipe() - wg.Add(2) - go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath) - go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg) - go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan) - } else { - go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan) + if err != nil { + ctx.ServerError("LFSPointerFiles", err) + return } - wg.Wait() - select { - case err, has := <-errChan: - if has { - ctx.ServerError("LFSPointerFiles", err) - } - default: - } ctx.HTML(200, tplSettingsLFSPointers) } -type pointerResult struct { - SHA string - Oid string - Size int64 - InRepo bool - Exists bool - Accessible bool -} - -func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) { - defer wg.Done() - defer catFileBatchReader.Close() - contentStore := lfs.NewContentStore() - - bufferedReader := bufio.NewReader(catFileBatchReader) - buf := make([]byte, 1025) - for { - // File descriptor line: sha - sha, err := bufferedReader.ReadString(' ') - if err != nil { - _ = catFileBatchReader.CloseWithError(err) - break - } - // Throw away the blob - if _, err := bufferedReader.ReadString(' '); err != nil { - _ = catFileBatchReader.CloseWithError(err) - break - } - sizeStr, err := bufferedReader.ReadString('\n') - if err != nil { - _ = catFileBatchReader.CloseWithError(err) - break - } - size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1]) - if err != nil { - _ = catFileBatchReader.CloseWithError(err) - break - } - pointerBuf := buf[:size+1] - if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil { - _ = catFileBatchReader.CloseWithError(err) - break - } - pointerBuf = pointerBuf[:size] - // Now we need to check if the pointerBuf is an LFS pointer - pointer := lfs.TryReadPointerFromBuffer(pointerBuf) - if pointer == nil { - continue - } - - result := pointerResult{ - SHA: strings.TrimSpace(sha), - Oid: pointer.Oid, - Size: pointer.Size, - } - - // Then we need to check that this pointer is in the db - if _, err := repo.GetLFSMetaObjectByOid(pointer.Oid); err != nil { - if err != models.ErrLFSObjectNotExist { - _ = catFileBatchReader.CloseWithError(err) - break - } - } else { - result.InRepo = true - } - - result.Exists, err = contentStore.Exists(pointer) - if err != nil { - _ = catFileBatchReader.CloseWithError(err) - break - } - - if result.Exists { - if !result.InRepo { - // Can we fix? - // OK well that's "simple" - // - we need to check whether current user has access to a repo that has access to the file - result.Accessible, err = models.LFSObjectAccessible(user, result.Oid) - if err != nil { - _ = catFileBatchReader.CloseWithError(err) - break - } - } else { - result.Accessible = true - } - } - pointerChan <- result - } - close(pointerChan) -} - // LFSAutoAssociate auto associates accessible lfs files func LFSAutoAssociate(ctx *context.Context) { if !setting.LFS.StartServer { From bffe86c8f0f79e52cc7c502d40fcb7c7736ed106 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 21 Feb 2021 20:00:09 +0000 Subject: [PATCH 26/69] Changed method name. --- modules/lfs/pointer_scanner.go | 4 ++-- modules/lfs/pointer_scanner_nogogit.go | 6 +++--- modules/lfs/shared.go | 2 +- modules/repository/repo.go | 2 +- routers/repo/lfs.go | 10 +++++----- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/lfs/pointer_scanner.go b/modules/lfs/pointer_scanner.go index bd0b9ef57a2e..9e504e27409e 100644 --- a/modules/lfs/pointer_scanner.go +++ b/modules/lfs/pointer_scanner.go @@ -12,8 +12,8 @@ import ( "github.com/go-git/go-git/v5/plumbing/object" ) -// SearchPointerFiles scans the whole repository for LFS pointer files -func SearchPointerFiles(repo *git.Repository) ([]PointerBlob, error) { +// SearchPointerBlobs scans the whole repository for LFS pointer files +func SearchPointerBlobs(repo *git.Repository) ([]PointerBlob, error) { gitRepo := repo.GoGitRepo() blobs, err := gitRepo.BlobObjects() diff --git a/modules/lfs/pointer_scanner_nogogit.go b/modules/lfs/pointer_scanner_nogogit.go index c08fabdbaa4d..abf096dda6e8 100644 --- a/modules/lfs/pointer_scanner_nogogit.go +++ b/modules/lfs/pointer_scanner_nogogit.go @@ -16,8 +16,8 @@ import ( "code.gitea.io/gitea/modules/git/pipeline" ) -// SearchPointerFiles scans the whole repository for LFS pointer files -func SearchPointerFiles(repo *git.Repository) ([]PointerBlob, error) { +// SearchPointerBlobs scans the whole repository for LFS pointer files +func SearchPointerBlobs(repo *git.Repository) ([]PointerBlob, error) { basePath := repo.Path pointerChan := make(chan PointerBlob) @@ -106,4 +106,4 @@ func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg pointerChan <- PointerBlob{Hash: sha, Pointer: pointer} } close(pointerChan) -} \ No newline at end of file +} diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go index 70c1a5a54496..9b6c9a26dd3b 100644 --- a/modules/lfs/shared.go +++ b/modules/lfs/shared.go @@ -67,4 +67,4 @@ type ObjectError struct { type PointerBlob struct { Hash string Pointer *Pointer -} \ No newline at end of file +} diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 6d5cd4e49714..e60d358b6d2d 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -314,7 +314,7 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi client := lfs.NewClient(&http.Client{}) contentStore := lfs.NewContentStore() - pointerBlobs, err := lfs.SearchPointerFiles(gitRepo) + pointerBlobs, err := lfs.SearchPointerBlobs(gitRepo) if err != nil { return err } diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index cdd034c56664..a3865a0c017b 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -419,7 +419,7 @@ func LFSPointerFiles(ctx *context.Context) { ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs" err = func() error { - pointerBlobs, err := lfs.SearchPointerFiles(ctx.Repo.GitRepo) + pointerBlobs, err := lfs.SearchPointerBlobs(ctx.Repo.GitRepo) if err != nil { return err } @@ -443,8 +443,8 @@ func LFSPointerFiles(ctx *context.Context) { for i, pointerBlob := range pointerBlobs { result := pointerResult{ - SHA: pointerBlob.Hash, - Oid: pointerBlob.Pointer.Oid, + SHA: pointerBlob.Hash, + Oid: pointerBlob.Pointer.Oid, Size: pointerBlob.Pointer.Size, } @@ -455,12 +455,12 @@ func LFSPointerFiles(ctx *context.Context) { } else { result.InRepo = true } - + result.Exists, err = contentStore.Exists(pointerBlob.Pointer) if err != nil { return err } - + if result.Exists { if !result.InRepo { // Can we fix? From 50d4814435bf9ba937fad466b08d3edd88a765e3 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 21 Feb 2021 22:01:00 +0000 Subject: [PATCH 27/69] Added comments. --- modules/lfs/pointer_scanner_nogogit.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/modules/lfs/pointer_scanner_nogogit.go b/modules/lfs/pointer_scanner_nogogit.go index abf096dda6e8..1ccc645a7ef2 100644 --- a/modules/lfs/pointer_scanner_nogogit.go +++ b/modules/lfs/pointer_scanner_nogogit.go @@ -29,6 +29,9 @@ func SearchPointerBlobs(repo *git.Repository) ([]PointerBlob, error) { wg := sync.WaitGroup{} wg.Add(5) + // Create the go-routines in reverse order. + + // 5. Copy the results from the channel into the result array pointers := make([]PointerBlob, 0, 50) go func() { @@ -37,9 +40,18 @@ func SearchPointerBlobs(repo *git.Repository) ([]PointerBlob, error) { pointers = append(pointers, pointer) } }() + + // 4. Take the output of cat-file --batch and check if each file in turn + // to see if they're pointers to files in the LFS store go createPointerResultsFromCatFileBatch(catFileBatchReader, &wg, pointerChan) + + // 3. Take the shas of the blobs and batch read them go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath) + + // 2. From the provided objects restrict to blobs <=1k go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg) + + // 1. Run batch-check on all objects in the repository if git.CheckGitVersionAtLeast("2.6.0") != nil { revListReader, revListWriter := io.Pipe() shasToCheckReader, shasToCheckWriter := io.Pipe() From e9024a510ed728216f2ba0686d4ba04a3eb4af1f Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 21 Feb 2021 22:09:00 +0000 Subject: [PATCH 28/69] Added more/specific log/error messages. --- modules/lfs/client.go | 18 +++++++++++------- modules/lfs/pointer_scanner.go | 6 ++++-- modules/lfs/transferadapter.go | 7 ++++--- modules/repository/repo.go | 14 ++++++-------- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/modules/lfs/client.go b/modules/lfs/client.go index ffd5e07b1eb0..123aa07ca1f4 100644 --- a/modules/lfs/client.go +++ b/modules/lfs/client.go @@ -13,6 +13,8 @@ import ( "io" "net/http" "strings" + + "code.gitea.io/gitea/modules/log" ) // Client is used to communicate with the LFS server @@ -52,12 +54,14 @@ func (c *Client) batch(ctx context.Context, url, operation string, objects []*Po payload := new(bytes.Buffer) err := json.NewEncoder(payload).Encode(request) if err != nil { - return nil, err + return nil, fmt.Errorf("lfs.Client.batch json.Encode: %w", err) } + log.Trace("lfs.Client.batch NewRequestWithContext: %s", url) + req, err := http.NewRequestWithContext(ctx, "POST", url, payload) if err != nil { - return nil, err + return nil, fmt.Errorf("lfs.Client.batch http.NewRequestWithContext: %w", err) } req.Header.Set("Content-type", MediaType) req.Header.Set("Accept", MediaType) @@ -69,18 +73,18 @@ func (c *Client) batch(ctx context.Context, url, operation string, objects []*Po return nil, ctx.Err() default: } - return nil, err + return nil, fmt.Errorf("lfs.Client.batch http.Do: %w", err) } defer res.Body.Close() if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("Unexpected servers response: %s", res.Status) + return nil, fmt.Errorf("lfs.Client.batch: Unexpected servers response: %s", res.Status) } var response BatchResponse err = json.NewDecoder(res.Body).Decode(&response) if err != nil { - return nil, err + return nil, fmt.Errorf("lfs.Client.batch json.Decode: %w", err) } if len(response.Transfer) == 0 { @@ -102,11 +106,11 @@ func (c *Client) Download(ctx context.Context, url, oid string, size int64) (io. transferAdapter, ok := c.transfers[result.Transfer] if !ok { - return nil, fmt.Errorf("Transferadapter not found: %s", result.Transfer) + return nil, fmt.Errorf("lfs.Client.Download Transferadapter not found: %s", result.Transfer) } if len(result.Objects) == 0 { - return nil, errors.New("No objects in result") + return nil, errors.New("lfs.Client.Download: No objects in result") } content, err := transferAdapter.Download(ctx, result.Objects[0]) diff --git a/modules/lfs/pointer_scanner.go b/modules/lfs/pointer_scanner.go index 9e504e27409e..a339e9691242 100644 --- a/modules/lfs/pointer_scanner.go +++ b/modules/lfs/pointer_scanner.go @@ -7,6 +7,8 @@ package lfs import ( + "fmt" + "code.gitea.io/gitea/modules/git" "github.com/go-git/go-git/v5/plumbing/object" @@ -18,7 +20,7 @@ func SearchPointerBlobs(repo *git.Repository) ([]PointerBlob, error) { blobs, err := gitRepo.BlobObjects() if err != nil { - return nil, err + return nil, fmt.Errorf("lfs.SearchPointerBlobs BlobObjects: %w", err) } var pointers []PointerBlob @@ -30,7 +32,7 @@ func SearchPointerBlobs(repo *git.Repository) ([]PointerBlob, error) { reader, err := blob.Reader() if err != nil { - return nil + return fmt.Errorf("lfs.SearchPointerBlobs blob.Reader: %w", err) } defer reader.Close() diff --git a/modules/lfs/transferadapter.go b/modules/lfs/transferadapter.go index 428710869ff4..ea3aff0000b9 100644 --- a/modules/lfs/transferadapter.go +++ b/modules/lfs/transferadapter.go @@ -7,6 +7,7 @@ package lfs import ( "context" "errors" + "fmt" "io" "net/http" ) @@ -32,12 +33,12 @@ func (a *BasicTransferAdapter) Name() string { func (a *BasicTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) { download, ok := r.Actions["download"] if !ok { - return nil, errors.New("Action 'download' not found") + return nil, errors.New("lfs.BasicTransferAdapter.Download: Action 'download' not found") } req, err := http.NewRequestWithContext(ctx, "GET", download.Href, nil) if err != nil { - return nil, err + return nil, fmt.Errorf("lfs.BasicTransferAdapter.Download http.NewRequestWithContext: %w", err) } for key, value := range download.Header { req.Header.Set(key, value) @@ -50,7 +51,7 @@ func (a *BasicTransferAdapter) Download(ctx context.Context, r *ObjectResponse) return nil, ctx.Err() default: } - return nil, err + return nil, fmt.Errorf("lfs.BasicTransferAdapter.Download http.Do: %w", err) } return res.Body, nil diff --git a/modules/repository/repo.go b/modules/repository/repo.go index e60d358b6d2d..f6311ca21d89 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -330,17 +330,17 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi meta := &models.LFSMetaObject{Oid: pointerBlob.Pointer.Oid, Size: pointerBlob.Pointer.Size, RepositoryID: repo.ID} meta, err = models.NewLFSMetaObject(meta) if err != nil { - return err + return fmt.Errorf("StoreMissingLfsObjectsInRepository models.NewLFSMetaObject: %w", err) } if meta.Existing { continue } - log.Trace("LFS OID[%s] not present in repository %s", pointerBlob.Pointer.Oid, repo.FullName()) + log.Trace("StoreMissingLfsObjectsInRepository: LFS OID[%s] not present in repository %s", pointerBlob.Pointer.Oid, repo.FullName()) exist, err := contentStore.Exists(pointerBlob.Pointer) if err != nil { - return err + return fmt.Errorf("StoreMissingLfsObjectsInRepository contentStore.Exists: %w", err) } if !exist { if setting.LFS.MaxFileSize > 0 && pointerBlob.Pointer.Size > setting.LFS.MaxFileSize { @@ -356,19 +356,17 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi return nil default: } - log.Error("LFS OID[%s] failed to download: %v", err) + log.Error("StoreMissingLfsObjectsInRepository: LFS OID[%s] failed to download: %v", err) continue } defer stream.Close() if err := contentStore.Put(pointerBlob.Pointer, stream); err != nil { if _, err2 := repo.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil { - return err2 + return fmt.Errorf("StoreMissingLfsObjectsInRepository RemoveLFSMetaObjectByOid: %w", err2) } - return err + return fmt.Errorf("StoreMissingLfsObjectsInRepository contentStore.Put: %w", err) } - } else { - log.Trace("LFS OID[%s] already present in content store", pointerBlob.Pointer.Oid) } } From 59a55b3f8160db01c097da1e7eba682d247e5e8b Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 22 Feb 2021 18:39:00 +0000 Subject: [PATCH 29/69] Embedded lfs.Pointer into models.LFSMetaObject. --- integrations/lfs_getobject_test.go | 4 ++-- models/lfs.go | 31 +++++++++++++++--------------- models/repo.go | 3 ++- modules/lfs/shared.go | 4 ++-- modules/repofiles/update.go | 4 ++-- modules/repofiles/upload.go | 4 ++-- modules/repository/repo.go | 2 +- routers/repo/lfs.go | 4 ++-- services/gitdiff/gitdiff.go | 2 +- services/lfs/server.go | 4 ++-- services/pull/lfs.go | 2 +- 11 files changed, 32 insertions(+), 32 deletions(-) diff --git a/integrations/lfs_getobject_test.go b/integrations/lfs_getobject_test.go index 47ea64a1be99..1557b684c824 100644 --- a/integrations/lfs_getobject_test.go +++ b/integrations/lfs_getobject_test.go @@ -41,9 +41,9 @@ func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string var lfsMetaObject *models.LFSMetaObject if setting.Database.UsePostgreSQL { - lfsMetaObject = &models.LFSMetaObject{ID: lfsID, Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID} + lfsMetaObject = &models.LFSMetaObject{ID: lfsID, Pointer: lfs.Pointer{Oid: oid, Size: int64(len(*content))}, RepositoryID: repositoryID} } else { - lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID} + lfsMetaObject = &models.LFSMetaObject{Pointer: lfs.Pointer{Oid: oid, Size: int64(len(*content))}, RepositoryID: repositoryID} } lfsID++ diff --git a/models/lfs.go b/models/lfs.go index 5b2661d0f7ad..8facdbe6973b 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -20,9 +20,8 @@ import ( // LFSMetaObject stores metadata for LFS tracked files. type LFSMetaObject struct { - ID int64 `xorm:"pk autoincr"` - Oid string `xorm:"UNIQUE(s) INDEX NOT NULL"` - Size int64 `xorm:"NOT NULL"` + ID int64 `xorm:"pk autoincr"` + lfs.Pointer `xorm:"extends"` RepositoryID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` Existing bool `xorm:"-"` CreatedUnix timeutil.TimeStamp `xorm:"created"` @@ -30,16 +29,16 @@ type LFSMetaObject struct { // RelativePath returns the relative path of the lfs object func (m *LFSMetaObject) RelativePath() string { - if len(m.Oid) < 5 { - return m.Oid + if len(m.Pointer.Oid) < 5 { + return m.Pointer.Oid } - return path.Join(m.Oid[0:2], m.Oid[2:4], m.Oid[4:]) + return path.Join(m.Pointer.Oid[0:2], m.Pointer.Oid[2:4], m.Pointer.Oid[4:]) } -// Pointer returns the string representation of an LFS pointer file -func (m *LFSMetaObject) Pointer() string { - return fmt.Sprintf("%s\n%s%s\nsize %d\n", lfs.MetaFileIdentifier, lfs.MetaFileOidPrefix, m.Oid, m.Size) +// PointerS returns the string representation of an LFS pointer file +func (m *LFSMetaObject) PointerS() string { + return fmt.Sprintf("%s\n%s%s\nsize %d\n", lfs.MetaFileIdentifier, lfs.MetaFileOidPrefix, m.Pointer.Oid, m.Pointer.Size) } // AsPointer creates a Pointer with Oid and Size @@ -108,7 +107,7 @@ func (repo *Repository) GetLFSMetaObjectByOid(oid string) (*LFSMetaObject, error return nil, ErrLFSObjectNotExist } - m := &LFSMetaObject{Oid: oid, RepositoryID: repo.ID} + m := &LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}, RepositoryID: repo.ID} has, err := x.Get(m) if err != nil { return nil, err @@ -131,12 +130,12 @@ func (repo *Repository) RemoveLFSMetaObjectByOid(oid string) (int64, error) { return -1, err } - m := &LFSMetaObject{Oid: oid, RepositoryID: repo.ID} + m := &LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}, RepositoryID: repo.ID} if _, err := sess.Delete(m); err != nil { return -1, err } - count, err := sess.Count(&LFSMetaObject{Oid: oid}) + count, err := sess.Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}}) if err != nil { return count, err } @@ -168,11 +167,11 @@ func (repo *Repository) CountLFSMetaObjects() (int64, error) { // LFSObjectAccessible checks if a provided Oid is accessible to the user func LFSObjectAccessible(user *User, oid string) (bool, error) { if user.IsAdmin { - count, err := x.Count(&LFSMetaObject{Oid: oid}) + count, err := x.Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}}) return (count > 0), err } cond := accessibleRepositoryCondition(user) - count, err := x.Where(cond).Join("INNER", "repository", "`lfs_meta_object`.repository_id = `repository`.id").Count(&LFSMetaObject{Oid: oid}) + count, err := x.Where(cond).Join("INNER", "repository", "`lfs_meta_object`.repository_id = `repository`.id").Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}}) return (count > 0), err } @@ -187,7 +186,7 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *User, repoID int64) error { oids := make([]interface{}, len(metas)) oidMap := make(map[string]*LFSMetaObject, len(metas)) for i, meta := range metas { - oids[i] = meta.Oid + oids[i] = meta.Pointer.Oid oidMap[meta.Oid] = meta } @@ -201,7 +200,7 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *User, repoID int64) error { return err } for i := range newMetas { - newMetas[i].Size = oidMap[newMetas[i].Oid].Size + newMetas[i].Size = oidMap[newMetas[i].Pointer.Oid].Pointer.Size newMetas[i].RepositoryID = repoID } if _, err := sess.InsertMulti(newMetas); err != nil { diff --git a/models/repo.go b/models/repo.go index 03259d9e347a..94b54720abe3 100644 --- a/models/repo.go +++ b/models/repo.go @@ -25,6 +25,7 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/options" @@ -1653,7 +1654,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error { } for _, v := range lfsObjects { - count, err := sess.Count(&LFSMetaObject{Oid: v.Oid}) + count, err := sess.Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: v.Oid}}) if err != nil { return err } diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go index 9b6c9a26dd3b..a9d1a3675be3 100644 --- a/modules/lfs/shared.go +++ b/modules/lfs/shared.go @@ -30,8 +30,8 @@ type Reference struct { // Pointer contains LFS pointer data type Pointer struct { - Oid string `json:"oid"` - Size int64 `json:"size"` + Oid string `json:"oid" xorm:"UNIQUE(s) INDEX NOT NULL"` + Size int64 `json:"size" xorm:"NOT NULL"` } // BatchResponse contains multiple object metadata Representation structures diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index feff7d948534..960d4b3211c4 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -386,8 +386,8 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up if err != nil { return nil, err } - lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(opts.Content)), RepositoryID: repo.ID} - content = lfsMetaObject.Pointer() + lfsMetaObject = &models.LFSMetaObject{Pointer: lfs.Pointer{Oid: oid, Size: int64(len(opts.Content))}, RepositoryID: repo.ID} + content = lfsMetaObject.PointerS() } } // Add the object to the database diff --git a/modules/repofiles/upload.go b/modules/repofiles/upload.go index 58e28213922b..730d269a291b 100644 --- a/modules/repofiles/upload.go +++ b/modules/repofiles/upload.go @@ -114,9 +114,9 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep return err } - uploadInfo.lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: t.repo.ID} + uploadInfo.lfsMetaObject = &models.LFSMetaObject{Pointer: lfs.Pointer{Oid: oid, Size: fileInfo.Size()}, RepositoryID: t.repo.ID} - if objectHash, err = t.HashObject(strings.NewReader(uploadInfo.lfsMetaObject.Pointer())); err != nil { + if objectHash, err = t.HashObject(strings.NewReader(uploadInfo.lfsMetaObject.PointerS())); err != nil { return err } infos[i] = uploadInfo diff --git a/modules/repository/repo.go b/modules/repository/repo.go index f6311ca21d89..ab5ce3b715e9 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -327,7 +327,7 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi default: } - meta := &models.LFSMetaObject{Oid: pointerBlob.Pointer.Oid, Size: pointerBlob.Pointer.Size, RepositoryID: repo.ID} + meta := &models.LFSMetaObject{Pointer: *pointerBlob.Pointer, RepositoryID: repo.ID} meta, err = models.NewLFSMetaObject(meta) if err != nil { return fmt.Errorf("StoreMissingLfsObjectsInRepository models.NewLFSMetaObject: %w", err) diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index a3865a0c017b..b48b5417fc6f 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -382,8 +382,8 @@ func LFSFileFind(ctx *context.Context) { ctx.Data["PageIsSettingsLFS"] = true var hash git.SHA1 if len(sha) == 0 { - meta := models.LFSMetaObject{Oid: oid, Size: size} - pointer := meta.Pointer() + meta := models.LFSMetaObject{Pointer: lfs.Pointer{Oid: oid, Size: size}} + pointer := meta.PointerS() hash = git.ComputeBlobHash([]byte(pointer)) sha = hash.String() } else { diff --git a/services/gitdiff/gitdiff.go b/services/gitdiff/gitdiff.go index 8fdb81e00e76..836491b1ecb7 100644 --- a/services/gitdiff/gitdiff.go +++ b/services/gitdiff/gitdiff.go @@ -1021,7 +1021,7 @@ func parseHunks(curFile *DiffFile, maxLines, maxLineCharacters int, input *bufio } else if curFileLFSPrefix && strings.HasPrefix(line[1:], lfs.MetaFileOidPrefix) { oid := strings.TrimPrefix(line[1:], lfs.MetaFileOidPrefix) if len(oid) == 64 { - m := &models.LFSMetaObject{Oid: oid} + m := &models.LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}} count, err := models.Count(m) if err == nil && count > 0 { diff --git a/services/lfs/server.go b/services/lfs/server.go index bc7bf521ec0b..2ce9bbf48568 100644 --- a/services/lfs/server.go +++ b/services/lfs/server.go @@ -237,7 +237,7 @@ func PostHandler(ctx *context.Context) { return } - meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Oid: p.Oid, Size: p.Size, RepositoryID: repository.ID}) + meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: *p, RepositoryID: repository.ID}) if err != nil { log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", p.Oid, p.Size, rc.User, rc.Repo, err) writeStatus(ctx, 404) @@ -337,7 +337,7 @@ func BatchHandler(ctx *context.Context) { } // Object is not found - meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID}) + meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: *object, RepositoryID: repository.ID}) if err == nil { exist, err := contentStore.Exists(meta.AsPointer()) if err != nil { diff --git a/services/pull/lfs.go b/services/pull/lfs.go index 54c1b7f88967..6fb928701235 100644 --- a/services/pull/lfs.go +++ b/services/pull/lfs.go @@ -124,7 +124,7 @@ func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg // OK we have a pointer that is associated with the head repo // and is actually a file in the LFS // Therefore it should be associated with the base repo - meta := &models.LFSMetaObject{Oid: pointer.Oid, Size: pointer.Size} + meta := &models.LFSMetaObject{Pointer: *pointer} meta.RepositoryID = pr.BaseRepoID if _, err := models.NewLFSMetaObject(meta); err != nil { _ = catFileBatchReader.CloseWithError(err) From 649d236092153d538034879ef679ff2fd80dfdc8 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 22 Feb 2021 18:42:00 +0000 Subject: [PATCH 30/69] Moved code from models to module. --- models/lfs.go | 6 ------ modules/lfs/pointer.go | 9 ++++++++- modules/repofiles/update.go | 2 +- modules/repofiles/upload.go | 2 +- routers/repo/lfs.go | 5 ++--- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/models/lfs.go b/models/lfs.go index 8facdbe6973b..fb8aa408ef70 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -8,7 +8,6 @@ import ( "crypto/sha256" "encoding/hex" "errors" - "fmt" "io" "path" @@ -36,11 +35,6 @@ func (m *LFSMetaObject) RelativePath() string { return path.Join(m.Pointer.Oid[0:2], m.Pointer.Oid[2:4], m.Pointer.Oid[4:]) } -// PointerS returns the string representation of an LFS pointer file -func (m *LFSMetaObject) PointerS() string { - return fmt.Sprintf("%s\n%s%s\nsize %d\n", lfs.MetaFileIdentifier, lfs.MetaFileOidPrefix, m.Pointer.Oid, m.Pointer.Size) -} - // AsPointer creates a Pointer with Oid and Size func (m *LFSMetaObject) AsPointer() *lfs.Pointer { pointer := &lfs.Pointer{Oid: m.Oid, Size: m.Size} diff --git a/modules/lfs/pointer.go b/modules/lfs/pointer.go index 7279457225b2..c7f2ec7c4687 100644 --- a/modules/lfs/pointer.go +++ b/modules/lfs/pointer.go @@ -5,6 +5,7 @@ package lfs import ( + "fmt" "io" "strconv" "strings" @@ -48,5 +49,11 @@ func TryReadPointerFromBuffer(buf []byte) *Pointer { return nil } - return &Pointer{Oid: oid, Size: size} + return &Pointer{oid, size} +} + +// StringContent returns the string representation of the pointer +// https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#the-pointer +func (p Pointer) StringContent() string { + return fmt.Sprintf("%s\n%s%s\nsize %d\n", MetaFileIdentifier, MetaFileOidPrefix, p.Oid, p.Size) } diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index 960d4b3211c4..a583d83a0086 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -387,7 +387,7 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up return nil, err } lfsMetaObject = &models.LFSMetaObject{Pointer: lfs.Pointer{Oid: oid, Size: int64(len(opts.Content))}, RepositoryID: repo.ID} - content = lfsMetaObject.PointerS() + content = lfsMetaObject.Pointer.StringContent() } } // Add the object to the database diff --git a/modules/repofiles/upload.go b/modules/repofiles/upload.go index 730d269a291b..9437be0e14d9 100644 --- a/modules/repofiles/upload.go +++ b/modules/repofiles/upload.go @@ -116,7 +116,7 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep uploadInfo.lfsMetaObject = &models.LFSMetaObject{Pointer: lfs.Pointer{Oid: oid, Size: fileInfo.Size()}, RepositoryID: t.repo.ID} - if objectHash, err = t.HashObject(strings.NewReader(uploadInfo.lfsMetaObject.PointerS())); err != nil { + if objectHash, err = t.HashObject(strings.NewReader(uploadInfo.lfsMetaObject.Pointer.StringContent())); err != nil { return err } infos[i] = uploadInfo diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index b48b5417fc6f..7b90dc72931b 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -382,9 +382,8 @@ func LFSFileFind(ctx *context.Context) { ctx.Data["PageIsSettingsLFS"] = true var hash git.SHA1 if len(sha) == 0 { - meta := models.LFSMetaObject{Pointer: lfs.Pointer{Oid: oid, Size: size}} - pointer := meta.PointerS() - hash = git.ComputeBlobHash([]byte(pointer)) + pointer := lfs.Pointer{Oid: oid, Size: size} + hash = git.ComputeBlobHash([]byte(pointer.StringContent())) sha = hash.String() } else { hash = git.MustIDFromString(sha) From 73ce39d6ae26f52f9deebb896790e5a6cba53efe Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 22 Feb 2021 18:44:00 +0000 Subject: [PATCH 31/69] Moved code from models to module. --- models/lfs.go | 10 ---------- models/repo.go | 2 +- modules/lfs/content_store.go | 17 ++++------------- modules/lfs/pointer.go | 10 ++++++++++ 4 files changed, 15 insertions(+), 24 deletions(-) diff --git a/models/lfs.go b/models/lfs.go index fb8aa408ef70..d003abcff69c 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -9,7 +9,6 @@ import ( "encoding/hex" "errors" "io" - "path" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/timeutil" @@ -26,15 +25,6 @@ type LFSMetaObject struct { CreatedUnix timeutil.TimeStamp `xorm:"created"` } -// RelativePath returns the relative path of the lfs object -func (m *LFSMetaObject) RelativePath() string { - if len(m.Pointer.Oid) < 5 { - return m.Pointer.Oid - } - - return path.Join(m.Pointer.Oid[0:2], m.Pointer.Oid[2:4], m.Pointer.Oid[4:]) -} - // AsPointer creates a Pointer with Oid and Size func (m *LFSMetaObject) AsPointer() *lfs.Pointer { pointer := &lfs.Pointer{Oid: m.Oid, Size: m.Size} diff --git a/models/repo.go b/models/repo.go index 94b54720abe3..a644d3883782 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1662,7 +1662,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error { continue } - removeStorageWithNotice(sess, storage.LFS, "Delete orphaned LFS file", v.RelativePath()) + removeStorageWithNotice(sess, storage.LFS, "Delete orphaned LFS file", v.Pointer.RelativePath()) } if _, err := sess.Delete(&LFSMetaObject{RepositoryID: repoID}); err != nil { diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index f013ae0d2b9f..64ac67af1621 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -11,7 +11,6 @@ import ( "fmt" "io" "os" - "path" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" @@ -50,18 +49,10 @@ func NewContentStore() *ContentStore { return contentStore } -func relativePath(p *Pointer) string { - if len(p.Oid) < 5 { - return p.Oid - } - - return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid[4:]) -} - // Get takes a Meta object and retrieves the content from the store, returning // it as an io.Reader. If fromByte > 0, the reader starts from that byte func (s *ContentStore) Get(pointer *Pointer, fromByte int64) (io.ReadCloser, error) { - f, err := s.Open(relativePath(pointer)) + f, err := s.Open(pointer.RelativePath()) if err != nil { log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", pointer.Oid, err) return nil, err @@ -84,7 +75,7 @@ func (s *ContentStore) Get(pointer *Pointer, fromByte int64) (io.ReadCloser, err func (s *ContentStore) Put(pointer *Pointer, r io.Reader) error { hash := sha256.New() rd := io.TeeReader(r, hash) - p := relativePath(pointer) + p := pointer.RelativePath() written, err := s.Save(p, rd) if err != nil { log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", pointer.Oid, p, err) @@ -111,7 +102,7 @@ func (s *ContentStore) Put(pointer *Pointer, r io.Reader) error { // Exists returns true if the object exists in the content store. func (s *ContentStore) Exists(pointer *Pointer) (bool, error) { - _, err := s.ObjectStorage.Stat(relativePath(pointer)) + _, err := s.ObjectStorage.Stat(pointer.RelativePath()) if err != nil { if os.IsNotExist(err) { return false, nil @@ -123,7 +114,7 @@ func (s *ContentStore) Exists(pointer *Pointer) (bool, error) { // Verify returns true if the object exists in the content store and size is correct. func (s *ContentStore) Verify(pointer *Pointer) (bool, error) { - p := relativePath(pointer) + p := pointer.RelativePath() fi, err := s.ObjectStorage.Stat(p) if os.IsNotExist(err) || (err == nil && fi.Size() != pointer.Size) { return false, nil diff --git a/modules/lfs/pointer.go b/modules/lfs/pointer.go index c7f2ec7c4687..ec7ef274e913 100644 --- a/modules/lfs/pointer.go +++ b/modules/lfs/pointer.go @@ -7,6 +7,7 @@ package lfs import ( "fmt" "io" + "path" "strconv" "strings" ) @@ -57,3 +58,12 @@ func TryReadPointerFromBuffer(buf []byte) *Pointer { func (p Pointer) StringContent() string { return fmt.Sprintf("%s\n%s%s\nsize %d\n", MetaFileIdentifier, MetaFileOidPrefix, p.Oid, p.Size) } + +// RelativePath returns the relative storage path of the pointer +func (p Pointer) RelativePath() string { + if len(p.Oid) < 5 { + return p.Oid + } + + return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid[4:]) +} From 80fe20b8df2d873530f80e458010dee8e7508bb0 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 22 Feb 2021 18:47:00 +0000 Subject: [PATCH 32/69] Moved code from models to module. --- integrations/lfs_getobject_test.go | 25 ++++++------------------- models/lfs.go | 13 ------------- modules/lfs/pointer.go | 13 +++++++++++++ modules/repofiles/update.go | 6 +++--- modules/repofiles/upload.go | 10 +++------- 5 files changed, 25 insertions(+), 42 deletions(-) diff --git a/integrations/lfs_getobject_test.go b/integrations/lfs_getobject_test.go index 1557b684c824..7ce70a521a7b 100644 --- a/integrations/lfs_getobject_test.go +++ b/integrations/lfs_getobject_test.go @@ -7,9 +7,6 @@ package integrations import ( "archive/zip" "bytes" - "crypto/sha256" - "encoding/hex" - "io" "io/ioutil" "net/http" "net/http/httptest" @@ -24,40 +21,30 @@ import ( "github.com/stretchr/testify/assert" ) -func GenerateLFSOid(content io.Reader) (string, error) { - h := sha256.New() - if _, err := io.Copy(h, content); err != nil { - return "", err - } - sum := h.Sum(nil) - return hex.EncodeToString(sum), nil -} - var lfsID = int64(20000) func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string { - oid, err := GenerateLFSOid(bytes.NewReader(*content)) + pointer, err := lfs.GeneratePointer(bytes.NewReader(*content)) assert.NoError(t, err) var lfsMetaObject *models.LFSMetaObject if setting.Database.UsePostgreSQL { - lfsMetaObject = &models.LFSMetaObject{ID: lfsID, Pointer: lfs.Pointer{Oid: oid, Size: int64(len(*content))}, RepositoryID: repositoryID} + lfsMetaObject = &models.LFSMetaObject{ID: lfsID, Pointer: pointer, RepositoryID: repositoryID} } else { - lfsMetaObject = &models.LFSMetaObject{Pointer: lfs.Pointer{Oid: oid, Size: int64(len(*content))}, RepositoryID: repositoryID} + lfsMetaObject = &models.LFSMetaObject{Pointer: pointer, RepositoryID: repositoryID} } lfsID++ lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject) assert.NoError(t, err) - pointer := lfsMetaObject.AsPointer() contentStore := lfs.NewContentStore() - exist, err := contentStore.Exists(pointer) + exist, err := contentStore.Exists(&pointer) assert.NoError(t, err) if !exist { - err := contentStore.Put(pointer, bytes.NewReader(*content)) + err := contentStore.Put(&pointer, bytes.NewReader(*content)) assert.NoError(t, err) } - return oid + return pointer.Oid } func storeAndGetLfs(t *testing.T, content *[]byte, extraHeader *http.Header, expectedStatus int) *httptest.ResponseRecorder { diff --git a/models/lfs.go b/models/lfs.go index d003abcff69c..7995ae5c620e 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -5,10 +5,7 @@ package models import ( - "crypto/sha256" - "encoding/hex" "errors" - "io" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/timeutil" @@ -73,16 +70,6 @@ func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) { return m, sess.Commit() } -// GenerateLFSOid generates a Sha256Sum to represent an oid for arbitrary content -func GenerateLFSOid(content io.Reader) (string, error) { - h := sha256.New() - if _, err := io.Copy(h, content); err != nil { - return "", err - } - sum := h.Sum(nil) - return hex.EncodeToString(sum), nil -} - // GetLFSMetaObjectByOid selects a LFSMetaObject entry from database by its OID. // It may return ErrLFSObjectNotExist or a database error. If the error is nil, // the returned pointer is a valid LFSMetaObject. diff --git a/modules/lfs/pointer.go b/modules/lfs/pointer.go index ec7ef274e913..e461540f2fa9 100644 --- a/modules/lfs/pointer.go +++ b/modules/lfs/pointer.go @@ -5,6 +5,8 @@ package lfs import ( + "crypto/sha256" + "encoding/hex" "fmt" "io" "path" @@ -67,3 +69,14 @@ func (p Pointer) RelativePath() string { return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid[4:]) } + +// GeneratePointer generates a pointer for arbitrary content +func GeneratePointer(content io.Reader) (Pointer, error) { + h := sha256.New() + c, err := io.Copy(h, content) + if err != nil { + return Pointer{}, err + } + sum := h.Sum(nil) + return Pointer{Oid: hex.EncodeToString(sum), Size: c}, nil +} diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index a583d83a0086..c58594b24e54 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -382,12 +382,12 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up if filename2attribute2info[treePath] != nil && filename2attribute2info[treePath]["filter"] == "lfs" { // OK so we are supposed to LFS this data! - oid, err := models.GenerateLFSOid(strings.NewReader(opts.Content)) + pointer, err := lfs.GeneratePointer(strings.NewReader(opts.Content)) if err != nil { return nil, err } - lfsMetaObject = &models.LFSMetaObject{Pointer: lfs.Pointer{Oid: oid, Size: int64(len(opts.Content))}, RepositoryID: repo.ID} - content = lfsMetaObject.Pointer.StringContent() + lfsMetaObject = &models.LFSMetaObject{Pointer: pointer, RepositoryID: repo.ID} + content = pointer.StringContent() } } // Add the object to the database diff --git a/modules/repofiles/upload.go b/modules/repofiles/upload.go index 9437be0e14d9..e325f9f5c39e 100644 --- a/modules/repofiles/upload.go +++ b/modules/repofiles/upload.go @@ -105,18 +105,14 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep if setting.LFS.StartServer && filename2attribute2info[uploadInfo.upload.Name] != nil && filename2attribute2info[uploadInfo.upload.Name]["filter"] == "lfs" { // Handle LFS // FIXME: Inefficient! this should probably happen in models.Upload - oid, err := models.GenerateLFSOid(file) - if err != nil { - return err - } - fileInfo, err := file.Stat() + pointer, err := lfs.GeneratePointer(file) if err != nil { return err } - uploadInfo.lfsMetaObject = &models.LFSMetaObject{Pointer: lfs.Pointer{Oid: oid, Size: fileInfo.Size()}, RepositoryID: t.repo.ID} + uploadInfo.lfsMetaObject = &models.LFSMetaObject{Pointer: pointer, RepositoryID: t.repo.ID} - if objectHash, err = t.HashObject(strings.NewReader(uploadInfo.lfsMetaObject.Pointer.StringContent())); err != nil { + if objectHash, err = t.HashObject(strings.NewReader(pointer.StringContent())); err != nil { return err } infos[i] = uploadInfo From 37a1d0215667509236405824217813f6c3d4e715 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 22 Feb 2021 19:21:00 +0000 Subject: [PATCH 33/69] Reduced pointer usage. --- integrations/lfs_getobject_test.go | 4 +-- models/lfs.go | 6 ---- modules/lfs/client.go | 6 ++-- modules/lfs/content_store.go | 10 +++--- modules/lfs/pointer.go | 45 +++++++++++++++++++------- modules/lfs/pointer_scanner.go | 4 +-- modules/lfs/pointer_scanner_nogogit.go | 4 +-- modules/lfs/shared.go | 4 +-- modules/repofiles/update.go | 8 ++--- modules/repofiles/upload.go | 4 +-- modules/repository/repo.go | 2 +- routers/repo/download.go | 5 +-- routers/repo/lfs.go | 2 +- routers/repo/view.go | 8 ++--- services/lfs/server.go | 20 ++++++------ services/pull/lfs.go | 9 +++--- 16 files changed, 80 insertions(+), 61 deletions(-) diff --git a/integrations/lfs_getobject_test.go b/integrations/lfs_getobject_test.go index 7ce70a521a7b..789c7572a77e 100644 --- a/integrations/lfs_getobject_test.go +++ b/integrations/lfs_getobject_test.go @@ -38,10 +38,10 @@ func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject) assert.NoError(t, err) contentStore := lfs.NewContentStore() - exist, err := contentStore.Exists(&pointer) + exist, err := contentStore.Exists(pointer) assert.NoError(t, err) if !exist { - err := contentStore.Put(&pointer, bytes.NewReader(*content)) + err := contentStore.Put(pointer, bytes.NewReader(*content)) assert.NoError(t, err) } return pointer.Oid diff --git a/models/lfs.go b/models/lfs.go index 7995ae5c620e..664568c95c84 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -22,12 +22,6 @@ type LFSMetaObject struct { CreatedUnix timeutil.TimeStamp `xorm:"created"` } -// AsPointer creates a Pointer with Oid and Size -func (m *LFSMetaObject) AsPointer() *lfs.Pointer { - pointer := &lfs.Pointer{Oid: m.Oid, Size: m.Size} - return pointer -} - // LFSTokenResponse defines the JSON structure in which the JWT token is stored. // This structure is fetched via SSH and passed by the Git LFS client to the server // endpoint for authorization. diff --git a/modules/lfs/client.go b/modules/lfs/client.go index 123aa07ca1f4..1825440853ff 100644 --- a/modules/lfs/client.go +++ b/modules/lfs/client.go @@ -46,7 +46,7 @@ func (c *Client) transferNames() []string { return keys } -func (c *Client) batch(ctx context.Context, url, operation string, objects []*Pointer) (*BatchResponse, error) { +func (c *Client) batch(ctx context.Context, url, operation string, objects []Pointer) (*BatchResponse, error) { url = fmt.Sprintf("%s.git/info/lfs/objects/batch", strings.TrimSuffix(url, ".git")) request := &BatchRequest{operation, c.transferNames(), nil, objects} @@ -96,8 +96,8 @@ func (c *Client) batch(ctx context.Context, url, operation string, objects []*Po // Download reads the specific LFS object from the LFS server func (c *Client) Download(ctx context.Context, url, oid string, size int64) (io.ReadCloser, error) { - var objects []*Pointer - objects = append(objects, &Pointer{oid, size}) + var objects []Pointer + objects = append(objects, Pointer{oid, size}) result, err := c.batch(ctx, url, "download", objects) if err != nil { diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index 64ac67af1621..6d62de0854fc 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -51,7 +51,7 @@ func NewContentStore() *ContentStore { // Get takes a Meta object and retrieves the content from the store, returning // it as an io.Reader. If fromByte > 0, the reader starts from that byte -func (s *ContentStore) Get(pointer *Pointer, fromByte int64) (io.ReadCloser, error) { +func (s *ContentStore) Get(pointer Pointer, fromByte int64) (io.ReadCloser, error) { f, err := s.Open(pointer.RelativePath()) if err != nil { log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", pointer.Oid, err) @@ -72,7 +72,7 @@ func (s *ContentStore) Get(pointer *Pointer, fromByte int64) (io.ReadCloser, err } // Put takes a Meta object and an io.Reader and writes the content to the store. -func (s *ContentStore) Put(pointer *Pointer, r io.Reader) error { +func (s *ContentStore) Put(pointer Pointer, r io.Reader) error { hash := sha256.New() rd := io.TeeReader(r, hash) p := pointer.RelativePath() @@ -101,7 +101,7 @@ func (s *ContentStore) Put(pointer *Pointer, r io.Reader) error { } // Exists returns true if the object exists in the content store. -func (s *ContentStore) Exists(pointer *Pointer) (bool, error) { +func (s *ContentStore) Exists(pointer Pointer) (bool, error) { _, err := s.ObjectStorage.Stat(pointer.RelativePath()) if err != nil { if os.IsNotExist(err) { @@ -113,7 +113,7 @@ func (s *ContentStore) Exists(pointer *Pointer) (bool, error) { } // Verify returns true if the object exists in the content store and size is correct. -func (s *ContentStore) Verify(pointer *Pointer) (bool, error) { +func (s *ContentStore) Verify(pointer Pointer) (bool, error) { p := pointer.RelativePath() fi, err := s.ObjectStorage.Stat(p) if os.IsNotExist(err) || (err == nil && fi.Size() != pointer.Size) { @@ -127,7 +127,7 @@ func (s *ContentStore) Verify(pointer *Pointer) (bool, error) { } // ReadMetaObject will read a models.LFSMetaObject and return a reader -func ReadMetaObject(pointer *Pointer) (io.ReadCloser, error) { +func ReadMetaObject(pointer Pointer) (io.ReadCloser, error) { contentStore := NewContentStore() return contentStore.Get(pointer, 0) } diff --git a/modules/lfs/pointer.go b/modules/lfs/pointer.go index e461540f2fa9..e2c66744ff97 100644 --- a/modules/lfs/pointer.go +++ b/modules/lfs/pointer.go @@ -7,6 +7,7 @@ package lfs import ( "crypto/sha256" "encoding/hex" + "errors" "fmt" "io" "path" @@ -25,34 +26,56 @@ const ( MetaFileOidPrefix = "oid sha256:" ) -// TryReadPointer tries to read LFS pointer data from the reader -func TryReadPointer(reader io.Reader) *Pointer { +var ( + // ErrMissingPrefix occurs if the content lacks the LFS prefix + ErrMissingPrefix = errors.New("Content lacks the LFS prefix") + + // ErrInvalidStructure occurs if the content has an invalid structure + ErrInvalidStructure = errors.New("Content has an invalid structure") + + // ErrInvalidOIDLength occurs if the oid has an invalid length + ErrInvalidOIDLength = errors.New("OID has an invalid length") +) + +// ReadPointer tries to read LFS pointer data from the reader +func ReadPointer(reader io.Reader) (Pointer, error) { buf := make([]byte, blobSizeCutoff) - n, _ := io.ReadFull(reader, buf) + n, err := io.ReadFull(reader, buf) + if err != nil { + return Pointer{}, err + } buf = buf[:n] - return TryReadPointerFromBuffer(buf) + return ReadPointerFromBuffer(buf) } -// TryReadPointerFromBuffer will return a pointer if the provided byte slice is a pointer file or nil otherwise. -func TryReadPointerFromBuffer(buf []byte) *Pointer { +// ReadPointerFromBuffer will return a pointer if the provided byte slice is a pointer file or an error otherwise. +func ReadPointerFromBuffer(buf []byte) (Pointer, error) { + var p Pointer + headString := string(buf) if !strings.HasPrefix(headString, MetaFileIdentifier) { - return nil + return p, ErrMissingPrefix } splitLines := strings.Split(headString, "\n") if len(splitLines) < 3 { - return nil + return p, ErrInvalidStructure } oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix) + if len(oid) != 64 { + return p, ErrInvalidOIDLength + } size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) - if len(oid) != 64 || err != nil { - return nil + if err != nil { + return p, err } - return &Pointer{oid, size} + p.Oid = oid + p.Size = size + + return p, nil } // StringContent returns the string representation of the pointer diff --git a/modules/lfs/pointer_scanner.go b/modules/lfs/pointer_scanner.go index a339e9691242..5df509bb4629 100644 --- a/modules/lfs/pointer_scanner.go +++ b/modules/lfs/pointer_scanner.go @@ -36,8 +36,8 @@ func SearchPointerBlobs(repo *git.Repository) ([]PointerBlob, error) { } defer reader.Close() - pointer := TryReadPointer(reader) - if pointer != nil { + pointer, err := ReadPointer(reader) + if err == nil { pointers = append(pointers, PointerBlob{Hash: blob.Hash.String(), Pointer: pointer}) } diff --git a/modules/lfs/pointer_scanner_nogogit.go b/modules/lfs/pointer_scanner_nogogit.go index 1ccc645a7ef2..fdff6b898861 100644 --- a/modules/lfs/pointer_scanner_nogogit.go +++ b/modules/lfs/pointer_scanner_nogogit.go @@ -110,8 +110,8 @@ func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg } pointerBuf = pointerBuf[:size] // Now we need to check if the pointerBuf is an LFS pointer - pointer := TryReadPointerFromBuffer(pointerBuf) - if pointer == nil { + pointer, err := ReadPointerFromBuffer(pointerBuf) + if err != nil { continue } diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go index a9d1a3675be3..f4520140b848 100644 --- a/modules/lfs/shared.go +++ b/modules/lfs/shared.go @@ -19,7 +19,7 @@ type BatchRequest struct { Operation string `json:"operation"` Transfers []string `json:"transfers,omitempty"` Ref *Reference `json:"ref,omitempty"` - Objects []*Pointer `json:"objects"` + Objects []Pointer `json:"objects"` } // Reference contains a git reference. @@ -66,5 +66,5 @@ type ObjectError struct { // PointerBlob associates a Git blob with a Pointer. type PointerBlob struct { Hash string - Pointer *Pointer + Pointer Pointer } diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index c58594b24e54..bfcd0865180e 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -69,8 +69,8 @@ func detectEncodingAndBOM(entry *git.TreeEntry, repo *models.Repository) (string buf = buf[:n] if setting.LFS.StartServer { - pointer := lfs.TryReadPointerFromBuffer(buf) - if pointer != nil { + pointer, err := lfs.ReadPointerFromBuffer(buf) + if err == nil { meta, err := repo.GetLFSMetaObjectByOid(pointer.Oid) if err != nil && err != models.ErrLFSObjectNotExist { // return default @@ -431,12 +431,12 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up return nil, err } contentStore := lfs.NewContentStore() - exist, err := contentStore.Exists(lfsMetaObject.AsPointer()) + exist, err := contentStore.Exists(lfsMetaObject.Pointer) if err != nil { return nil, err } if !exist { - if err := contentStore.Put(lfsMetaObject.AsPointer(), strings.NewReader(opts.Content)); err != nil { + if err := contentStore.Put(lfsMetaObject.Pointer, strings.NewReader(opts.Content)); err != nil { if _, err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil { return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err) } diff --git a/modules/repofiles/upload.go b/modules/repofiles/upload.go index e325f9f5c39e..8b5206e75ed3 100644 --- a/modules/repofiles/upload.go +++ b/modules/repofiles/upload.go @@ -165,7 +165,7 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep if uploadInfo.lfsMetaObject == nil { continue } - exist, err := contentStore.Exists(uploadInfo.lfsMetaObject.AsPointer()) + exist, err := contentStore.Exists(uploadInfo.lfsMetaObject.Pointer) if err != nil { return cleanUpAfterFailure(&infos, t, err) } @@ -177,7 +177,7 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRep defer file.Close() // FIXME: Put regenerates the hash and copies the file over. // I guess this strictly ensures the soundness of the store but this is inefficient. - if err := contentStore.Put(uploadInfo.lfsMetaObject.AsPointer(), file); err != nil { + if err := contentStore.Put(uploadInfo.lfsMetaObject.Pointer, file); err != nil { // OK Now we need to cleanup // Can't clean up the store, once uploaded there they're there. return cleanUpAfterFailure(&infos, t, err) diff --git a/modules/repository/repo.go b/modules/repository/repo.go index ab5ce3b715e9..e995e62bd9e7 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -327,7 +327,7 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi default: } - meta := &models.LFSMetaObject{Pointer: *pointerBlob.Pointer, RepositoryID: repo.ID} + meta := &models.LFSMetaObject{Pointer: pointerBlob.Pointer, RepositoryID: repo.ID} meta, err = models.NewLFSMetaObject(meta) if err != nil { return fmt.Errorf("StoreMissingLfsObjectsInRepository models.NewLFSMetaObject: %w", err) diff --git a/routers/repo/download.go b/routers/repo/download.go index 50a90a8d50ac..e0db6249573b 100644 --- a/routers/repo/download.go +++ b/routers/repo/download.go @@ -96,12 +96,13 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error { } }() - if pointer := lfs.TryReadPointer(dataRc); pointer != nil { + pointer, err := lfs.ReadPointer(dataRc) + if err == nil { meta, _ := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid) if meta == nil { return ServeBlob(ctx, blob) } - lfsDataRc, err := lfs.ReadMetaObject(meta.AsPointer()) + lfsDataRc, err := lfs.ReadMetaObject(meta.Pointer) if err != nil { return err } diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index 7b90dc72931b..4e3f825a3805 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -263,7 +263,7 @@ func LFSFileGet(ctx *context.Context) { return } ctx.Data["LFSFile"] = meta - dataRc, err := lfs.ReadMetaObject(meta.AsPointer()) + dataRc, err := lfs.ReadMetaObject(meta.Pointer) if err != nil { ctx.ServerError("LFSFileGet", err) return diff --git a/routers/repo/view.go b/routers/repo/view.go index 6b2cbf8d014c..512901cc7309 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -273,8 +273,8 @@ func renderDirectory(ctx *context.Context, treeLink string) { // FIXME: what happens when README file is an image? if isTextFile && setting.LFS.StartServer { - pointer := lfs.TryReadPointerFromBuffer(buf) - if pointer != nil { + pointer, err := lfs.ReadPointerFromBuffer(buf) + if err == nil { meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid) if err != nil && err != models.ErrLFSObjectNotExist { ctx.ServerError("GetLFSMetaObject", err) @@ -398,8 +398,8 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st //Check for LFS meta file if isTextFile && setting.LFS.StartServer { - pointer := lfs.TryReadPointerFromBuffer(buf) - if pointer != nil { + pointer, err := lfs.ReadPointerFromBuffer(buf) + if err == nil { meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid) if err != nil && err != models.ErrLFSObjectNotExist { ctx.ServerError("GetLFSMetaObject", err) diff --git a/services/lfs/server.go b/services/lfs/server.go index 2ce9bbf48568..c0a2a38b9471 100644 --- a/services/lfs/server.go +++ b/services/lfs/server.go @@ -80,7 +80,7 @@ func ObjectOidHandler(ctx *context.Context) { writeStatus(ctx, 404) } -func getAuthenticatedRepoAndMeta(ctx *context.Context, rc *requestContext, p *lfs_module.Pointer, requireWrite bool) (*models.LFSMetaObject, *models.Repository) { +func getAuthenticatedRepoAndMeta(ctx *context.Context, rc *requestContext, p lfs_module.Pointer, requireWrite bool) (*models.LFSMetaObject, *models.Repository) { if !isOidValid(p.Oid) { log.Info("Attempt to access invalid LFS OID[%s] in %s/%s", p.Oid, rc.User, rc.Repo) writeStatus(ctx, 404) @@ -143,7 +143,7 @@ func getContentHandler(ctx *context.Context) { } contentStore := lfs_module.NewContentStore() - content, err := contentStore.Get(meta.AsPointer(), fromByte) + content, err := contentStore.Get(meta.Pointer, fromByte) if err != nil { if lfs_module.IsErrRangeNotSatisfiable(err) { writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable) @@ -237,7 +237,7 @@ func PostHandler(ctx *context.Context) { return } - meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: *p, RepositoryID: repository.ID}) + meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: p, RepositoryID: repository.ID}) if err != nil { log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", p.Oid, p.Size, rc.User, rc.Repo, err) writeStatus(ctx, 404) @@ -318,7 +318,7 @@ func BatchHandler(ctx *context.Context) { meta, err := repository.GetLFSMetaObjectByOid(object.Oid) if err == nil { // Object is found and exists - exist, err := contentStore.Exists(meta.AsPointer()) + exist, err := contentStore.Exists(meta.Pointer) if err != nil { log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, reqCtx.User, reqCtx.Repo, err) writeStatus(ctx, 500) @@ -337,9 +337,9 @@ func BatchHandler(ctx *context.Context) { } // Object is not found - meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: *object, RepositoryID: repository.ID}) + meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: object, RepositoryID: repository.ID}) if err == nil { - exist, err := contentStore.Exists(meta.AsPointer()) + exist, err := contentStore.Exists(meta.Pointer) if err != nil { log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, reqCtx.User, reqCtx.Repo, err) writeStatus(ctx, 500) @@ -374,7 +374,7 @@ func PutHandler(ctx *context.Context) { contentStore := lfs_module.NewContentStore() defer ctx.Req.Body.Close() - if err := contentStore.Put(meta.AsPointer(), ctx.Req.Body); err != nil { + if err := contentStore.Put(meta.Pointer, ctx.Req.Body); err != nil { // Put will log the error itself ctx.Resp.WriteHeader(500) if err == lfs_module.ErrSizeMismatch || err == lfs_module.ErrHashMismatch { @@ -414,7 +414,7 @@ func VerifyHandler(ctx *context.Context) { } contentStore := lfs_module.NewContentStore() - ok, err := contentStore.Verify(meta.AsPointer()) + ok, err := contentStore.Verify(meta.Pointer) if err != nil { // Error will be logged in Verify ctx.Resp.WriteHeader(500) @@ -479,14 +479,14 @@ func MetaMatcher(r *http.Request) bool { return mt == lfs_module.MediaType } -func unpack(ctx *context.Context) (*requestContext, *lfs_module.Pointer) { +func unpack(ctx *context.Context) (*requestContext, lfs_module.Pointer) { r := ctx.Req rc := &requestContext{ User: ctx.Params("username"), Repo: strings.TrimSuffix(ctx.Params("reponame"), ".git"), Authorization: r.Header.Get("Authorization"), } - p := &lfs_module.Pointer{Oid: ctx.Params("oid")} + p := lfs_module.Pointer{Oid: ctx.Params("oid")} if r.Method == "POST" { // Maybe also check if +json var p2 lfs_module.Pointer diff --git a/services/pull/lfs.go b/services/pull/lfs.go index 6fb928701235..439f6d69e02f 100644 --- a/services/pull/lfs.go +++ b/services/pull/lfs.go @@ -70,6 +70,8 @@ func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg defer wg.Done() defer catFileBatchReader.Close() + contentStore := lfs.NewContentStore() + bufferedReader := bufio.NewReader(catFileBatchReader) buf := make([]byte, 1025) for { @@ -101,12 +103,11 @@ func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg } pointerBuf = pointerBuf[:size] // Now we need to check if the pointerBuf is an LFS pointer - pointer := lfs.TryReadPointerFromBuffer(pointerBuf) - if pointer == nil { + pointer, err := lfs.ReadPointerFromBuffer(pointerBuf) + if err != nil { continue } - contentStore := lfs.NewContentStore() exist, _ := contentStore.Exists(pointer) if !exist { continue @@ -124,7 +125,7 @@ func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg // OK we have a pointer that is associated with the head repo // and is actually a file in the LFS // Therefore it should be associated with the base repo - meta := &models.LFSMetaObject{Pointer: *pointer} + meta := &models.LFSMetaObject{Pointer: pointer} meta.RepositoryID = pr.BaseRepoID if _, err := models.NewLFSMetaObject(meta); err != nil { _ = catFileBatchReader.CloseWithError(err) From b0e0804ac4cde70061039bff53375487a767608e Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 22 Feb 2021 19:23:00 +0000 Subject: [PATCH 34/69] Embedded type. --- modules/lfs/shared.go | 7 +++---- services/lfs/server.go | 17 ++++++++--------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go index f4520140b848..70b76d7512d5 100644 --- a/modules/lfs/shared.go +++ b/modules/lfs/shared.go @@ -44,8 +44,7 @@ type BatchResponse struct { // ObjectResponse is object metadata as seen by clients of the LFS server. type ObjectResponse struct { - Oid string `json:"oid"` - Size int64 `json:"size"` + Pointer Actions map[string]*Link `json:"actions"` Error *ObjectError `json:"error,omitempty"` } @@ -65,6 +64,6 @@ type ObjectError struct { // PointerBlob associates a Git blob with a Pointer. type PointerBlob struct { - Hash string - Pointer Pointer + Hash string + Pointer } diff --git a/services/lfs/server.go b/services/lfs/server.go index c0a2a38b9471..11f60cc1755b 100644 --- a/services/lfs/server.go +++ b/services/lfs/server.go @@ -189,7 +189,7 @@ func getMetaHandler(ctx *context.Context) { if ctx.Req.Method == "GET" { enc := json.NewEncoder(ctx.Resp) - if err := enc.Encode(represent(rc, meta, true, false)); err != nil { + if err := enc.Encode(represent(rc, meta.Pointer, true, false)); err != nil { log.Error("Failed to encode representation as json. Error: %v", err) } } @@ -260,7 +260,7 @@ func PostHandler(ctx *context.Context) { ctx.Resp.WriteHeader(sentStatus) enc := json.NewEncoder(ctx.Resp) - if err := enc.Encode(represent(rc, meta, meta.Existing, true)); err != nil { + if err := enc.Encode(represent(rc, meta.Pointer, meta.Existing, true)); err != nil { log.Error("Failed to encode representation as json. Error: %v", err) } logRequest(ctx.Req, sentStatus) @@ -325,7 +325,7 @@ func BatchHandler(ctx *context.Context) { return } if exist { - responseObjects = append(responseObjects, represent(reqCtx, meta, true, false)) + responseObjects = append(responseObjects, represent(reqCtx, meta.Pointer, true, false)) continue } } @@ -345,7 +345,7 @@ func BatchHandler(ctx *context.Context) { writeStatus(ctx, 500) return } - responseObjects = append(responseObjects, represent(reqCtx, meta, meta.Existing, !exist)) + responseObjects = append(responseObjects, represent(reqCtx, meta.Pointer, meta.Existing, !exist)) } else { log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, reqCtx.User, reqCtx.Repo, err) } @@ -431,10 +431,9 @@ func VerifyHandler(ctx *context.Context) { // represent takes a requestContext and Meta and turns it into a ObjectResponse suitable // for json encoding -func represent(rc *requestContext, meta *models.LFSMetaObject, download, upload bool) *lfs_module.ObjectResponse { +func represent(rc *requestContext, pointer lfs_module.Pointer, download, upload bool) *lfs_module.ObjectResponse { rep := &lfs_module.ObjectResponse{ - Oid: meta.Oid, - Size: meta.Size, + Pointer: pointer, Actions: make(map[string]*lfs_module.Link), } @@ -448,11 +447,11 @@ func represent(rc *requestContext, meta *models.LFSMetaObject, download, upload } if download { - rep.Actions["download"] = &lfs_module.Link{Href: rc.ObjectLink(meta.Oid), Header: header} + rep.Actions["download"] = &lfs_module.Link{Href: rc.ObjectLink(pointer.Oid), Header: header} } if upload { - rep.Actions["upload"] = &lfs_module.Link{Href: rc.ObjectLink(meta.Oid), Header: header} + rep.Actions["upload"] = &lfs_module.Link{Href: rc.ObjectLink(pointer.Oid), Header: header} } if upload && !download { From 3779dea72dd5006afb79e70145cbdbb4cee122ba Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 22 Feb 2021 19:24:00 +0000 Subject: [PATCH 35/69] Use promoted fields. --- models/lfs.go | 4 ++-- models/repo.go | 2 +- modules/repository/repo.go | 8 ++++---- routers/repo/lfs.go | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/models/lfs.go b/models/lfs.go index 664568c95c84..cf62d2b28dd5 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -151,7 +151,7 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *User, repoID int64) error { oids := make([]interface{}, len(metas)) oidMap := make(map[string]*LFSMetaObject, len(metas)) for i, meta := range metas { - oids[i] = meta.Pointer.Oid + oids[i] = meta.Oid oidMap[meta.Oid] = meta } @@ -165,7 +165,7 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *User, repoID int64) error { return err } for i := range newMetas { - newMetas[i].Size = oidMap[newMetas[i].Pointer.Oid].Pointer.Size + newMetas[i].Size = oidMap[newMetas[i].Oid].Size newMetas[i].RepositoryID = repoID } if _, err := sess.InsertMulti(newMetas); err != nil { diff --git a/models/repo.go b/models/repo.go index a644d3883782..94b54720abe3 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1662,7 +1662,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error { continue } - removeStorageWithNotice(sess, storage.LFS, "Delete orphaned LFS file", v.Pointer.RelativePath()) + removeStorageWithNotice(sess, storage.LFS, "Delete orphaned LFS file", v.RelativePath()) } if _, err := sess.Delete(&LFSMetaObject{RepositoryID: repoID}); err != nil { diff --git a/modules/repository/repo.go b/modules/repository/repo.go index e995e62bd9e7..9e44deb54fd3 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -336,19 +336,19 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi continue } - log.Trace("StoreMissingLfsObjectsInRepository: LFS OID[%s] not present in repository %s", pointerBlob.Pointer.Oid, repo.FullName()) + log.Trace("StoreMissingLfsObjectsInRepository: LFS OID[%s] not present in repository %s", pointerBlob.Oid, repo.FullName()) exist, err := contentStore.Exists(pointerBlob.Pointer) if err != nil { return fmt.Errorf("StoreMissingLfsObjectsInRepository contentStore.Exists: %w", err) } if !exist { - if setting.LFS.MaxFileSize > 0 && pointerBlob.Pointer.Size > setting.LFS.MaxFileSize { - log.Info("LFS OID[%s] download denied because of LFS_MAX_FILE_SIZE=%d < size %d", pointerBlob.Pointer.Oid, setting.LFS.MaxFileSize, pointerBlob.Pointer.Size) + if setting.LFS.MaxFileSize > 0 && pointerBlob.Size > setting.LFS.MaxFileSize { + log.Info("LFS OID[%s] download denied because of LFS_MAX_FILE_SIZE=%d < size %d", pointerBlob.Oid, setting.LFS.MaxFileSize, pointerBlob.Size) continue } - stream, err := client.Download(ctx, lfsAddr, pointerBlob.Pointer.Oid, pointerBlob.Pointer.Size) + stream, err := client.Download(ctx, lfsAddr, pointerBlob.Oid, pointerBlob.Size) if err != nil { select { case <-ctx.Done(): diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index 4e3f825a3805..ffa4b0f0736a 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -443,11 +443,11 @@ func LFSPointerFiles(ctx *context.Context) { for i, pointerBlob := range pointerBlobs { result := pointerResult{ SHA: pointerBlob.Hash, - Oid: pointerBlob.Pointer.Oid, - Size: pointerBlob.Pointer.Size, + Oid: pointerBlob.Oid, + Size: pointerBlob.Size, } - if _, err := repo.GetLFSMetaObjectByOid(pointerBlob.Pointer.Oid); err != nil { + if _, err := repo.GetLFSMetaObjectByOid(pointerBlob.Oid); err != nil { if err != models.ErrLFSObjectNotExist { return err } @@ -465,7 +465,7 @@ func LFSPointerFiles(ctx *context.Context) { // Can we fix? // OK well that's "simple" // - we need to check whether current user has access to a repo that has access to the file - result.Accessible, err = models.LFSObjectAccessible(ctx.User, pointerBlob.Pointer.Oid) + result.Accessible, err = models.LFSObjectAccessible(ctx.User, pointerBlob.Oid) if err != nil { return err } From 932a370354ba9f0574ce169b379fc52535a0caff Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 23 Feb 2021 18:05:46 +0000 Subject: [PATCH 36/69] Fixed unexpected eof. --- modules/lfs/pointer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/lfs/pointer.go b/modules/lfs/pointer.go index e2c66744ff97..80b675223f1f 100644 --- a/modules/lfs/pointer.go +++ b/modules/lfs/pointer.go @@ -41,7 +41,7 @@ var ( func ReadPointer(reader io.Reader) (Pointer, error) { buf := make([]byte, blobSizeCutoff) n, err := io.ReadFull(reader, buf) - if err != nil { + if err != nil && err != io.ErrUnexpectedEOF { return Pointer{}, err } buf = buf[:n] From 6a47890f1ccbfe009812285c28f3ec8d38b0e6ab Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 23 Feb 2021 18:55:10 +0000 Subject: [PATCH 37/69] Added unit tests. --- modules/lfs/pointer_test.go | 72 +++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 modules/lfs/pointer_test.go diff --git a/modules/lfs/pointer_test.go b/modules/lfs/pointer_test.go new file mode 100644 index 000000000000..30195ea19476 --- /dev/null +++ b/modules/lfs/pointer_test.go @@ -0,0 +1,72 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "path" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStringContent(t *testing.T) { + p := Pointer{Oid: "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393", Size: 1234} + expected := "version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n" + assert.Equal(t, p.StringContent(), expected) +} + +func TestRelativePath(t *testing.T) { + p := Pointer{Oid: "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393"} + expected := path.Join("4d", "7a", "214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393") + assert.Equal(t, p.RelativePath(), expected) + + p2 := Pointer{Oid: "4d7a"} + assert.Equal(t, p2.RelativePath(), "4d7a") +} + +func TestGeneratePointer(t *testing.T) { + p, err := GeneratePointer(strings.NewReader("Gitea")) + assert.NoError(t, err) + assert.Equal(t, p.Oid, "94cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc") + assert.Equal(t, p.Size, int64(5)) +} + +func TestReadPointerFromBuffer(t *testing.T) { + _, err := ReadPointerFromBuffer([]byte{}) + assert.ErrorIs(t, err, ErrMissingPrefix) + + _, err = ReadPointerFromBuffer([]byte("test")) + assert.ErrorIs(t, err, ErrMissingPrefix) + + _, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\n")) + assert.ErrorIs(t, err, ErrInvalidStructure) + + _, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a\nsize 1234\n")) + assert.ErrorIs(t, err, ErrInvalidOIDLength) + + _, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\ntest 1234\n")) + assert.Error(t, err) + + _, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize test\n")) + assert.Error(t, err) + + p, err := ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n")) + assert.NoError(t, err) + assert.Equal(t, p.Oid, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393") + assert.Equal(t, p.Size, int64(1234)) + + p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\ntest")) + assert.NoError(t, err) + assert.Equal(t, p.Oid, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393") + assert.Equal(t, p.Size, int64(1234)) +} + +func TestReadPointer(t *testing.T) { + p, err := ReadPointer(strings.NewReader("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n")) + assert.NoError(t, err) + assert.Equal(t, p.Oid, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393") + assert.Equal(t, p.Size, int64(1234)) +} From 3e304217d8c2c5836286f855cb9361feeaea9097 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 8 Mar 2021 21:55:47 +0000 Subject: [PATCH 38/69] Implemented migration of local file paths. --- modules/lfs/client.go | 113 ++-------------------------- modules/lfs/endpoint.go | 103 ++++++++++++++++++++++++++ modules/lfs/filesystem_client.go | 45 +++++++++++ modules/lfs/http_client.go | 123 +++++++++++++++++++++++++++++++ modules/repository/repo.go | 11 +-- services/mirror/mirror.go | 6 +- 6 files changed, 289 insertions(+), 112 deletions(-) create mode 100644 modules/lfs/endpoint.go create mode 100644 modules/lfs/filesystem_client.go create mode 100644 modules/lfs/http_client.go diff --git a/modules/lfs/client.go b/modules/lfs/client.go index 1825440853ff..ec5d8246fdee 100644 --- a/modules/lfs/client.go +++ b/modules/lfs/client.go @@ -5,117 +5,20 @@ package lfs import ( - "bytes" "context" - "encoding/json" - "errors" - "fmt" "io" - "net/http" - "strings" - - "code.gitea.io/gitea/modules/log" + "net/url" ) -// Client is used to communicate with the LFS server -type Client struct { - client *http.Client - transfers map[string]TransferAdapter +// Client is used to communicate with a LFS source +type Client interface { + Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) } // NewClient creates a LFS client -func NewClient(hc *http.Client) *Client { - client := &Client{hc, make(map[string]TransferAdapter)} - - basic := &BasicTransferAdapter{hc} - - client.transfers[basic.Name()] = basic - - return client -} - -func (c *Client) transferNames() []string { - keys := make([]string, len(c.transfers)) - - i := 0 - for k := range c.transfers { - keys[i] = k - i++ - } - - return keys -} - -func (c *Client) batch(ctx context.Context, url, operation string, objects []Pointer) (*BatchResponse, error) { - url = fmt.Sprintf("%s.git/info/lfs/objects/batch", strings.TrimSuffix(url, ".git")) - - request := &BatchRequest{operation, c.transferNames(), nil, objects} - - payload := new(bytes.Buffer) - err := json.NewEncoder(payload).Encode(request) - if err != nil { - return nil, fmt.Errorf("lfs.Client.batch json.Encode: %w", err) - } - - log.Trace("lfs.Client.batch NewRequestWithContext: %s", url) - - req, err := http.NewRequestWithContext(ctx, "POST", url, payload) - if err != nil { - return nil, fmt.Errorf("lfs.Client.batch http.NewRequestWithContext: %w", err) - } - req.Header.Set("Content-type", MediaType) - req.Header.Set("Accept", MediaType) - - res, err := c.client.Do(req) - if err != nil { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - return nil, fmt.Errorf("lfs.Client.batch http.Do: %w", err) - } - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("lfs.Client.batch: Unexpected servers response: %s", res.Status) - } - - var response BatchResponse - err = json.NewDecoder(res.Body).Decode(&response) - if err != nil { - return nil, fmt.Errorf("lfs.Client.batch json.Decode: %w", err) - } - - if len(response.Transfer) == 0 { - response.Transfer = "basic" - } - - return &response, nil -} - -// Download reads the specific LFS object from the LFS server -func (c *Client) Download(ctx context.Context, url, oid string, size int64) (io.ReadCloser, error) { - var objects []Pointer - objects = append(objects, Pointer{oid, size}) - - result, err := c.batch(ctx, url, "download", objects) - if err != nil { - return nil, err - } - - transferAdapter, ok := c.transfers[result.Transfer] - if !ok { - return nil, fmt.Errorf("lfs.Client.Download Transferadapter not found: %s", result.Transfer) - } - - if len(result.Objects) == 0 { - return nil, errors.New("lfs.Client.Download: No objects in result") - } - - content, err := transferAdapter.Download(ctx, result.Objects[0]) - if err != nil { - return nil, err +func NewClient(endpoint *url.URL) Client { + if endpoint.Scheme == "file" { + return newFilesystenClient(endpoint.Path) } - return content, nil + return newHTTPClient(endpoint.String()) } diff --git a/modules/lfs/endpoint.go b/modules/lfs/endpoint.go new file mode 100644 index 000000000000..9f3941802bf7 --- /dev/null +++ b/modules/lfs/endpoint.go @@ -0,0 +1,103 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "fmt" + "net/url" + "os" + "path" + "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/log" +) + +// DetermineEndpoint determines an endpoint from the clone url or uses the specified LFS url. +func DetermineEndpoint(cloneurl, lfsurl string) *url.URL { + if len(lfsurl) > 0 { + return endpointFromURL(lfsurl) + } + return endpointFromCloneURL(cloneurl) +} + +func endpointFromCloneURL(rawurl string) *url.URL { + ep := endpointFromURL(rawurl) + if ep == nil { + return ep + } + + if strings.HasSuffix(ep.Path, "/") { + ep.Path = ep.Path[:len(ep.Path)-1] + } + + if ep.Scheme == "file" { + return ep + } + + if path.Ext(ep.Path) == ".git" { + ep.Path += "/info/lfs" + } else { + ep.Path += ".git/info/lfs" + } + + return ep +} + +func endpointFromURL(rawurl string) *url.URL { + if strings.HasPrefix(rawurl, "/") { + return endpointFromLocalPath(rawurl) + } + + u, err := url.Parse(rawurl) + if err != nil { + log.Error("lfs.endpointFromUrl: %v", err) + return nil + } + + switch u.Scheme { + case "http", "https": + return u + case "file": + return endpointFromLocalPath(u.Path) + default: + if _, err := os.Stat(rawurl); err == nil { + return endpointFromLocalPath(rawurl) + } + + log.Error("lfs.endpointFromUrl: unknown url") + return nil + } +} + +func endpointFromLocalPath(path string) *url.URL { + var slash string + if abs, err := filepath.Abs(path); err == nil { + if !strings.HasPrefix(abs, "/") { + slash = "/" + } + path = abs + } + + var gitpath string + if filepath.Base(path) == ".git" { + gitpath = path + path = filepath.Dir(path) + } else { + gitpath = filepath.Join(path, ".git") + } + + if _, err := os.Stat(gitpath); err == nil { + path = gitpath + } else if _, err := os.Stat(path); err != nil { + return nil + } + + path = fmt.Sprintf("file://%s%s", slash, filepath.ToSlash(path)) + + u, _ := url.Parse(path) + + return u +} diff --git a/modules/lfs/filesystem_client.go b/modules/lfs/filesystem_client.go new file mode 100644 index 000000000000..3d12db3eb819 --- /dev/null +++ b/modules/lfs/filesystem_client.go @@ -0,0 +1,45 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "context" + "io" + "os" + "path/filepath" +) + +// FilesystemClient is used to read LFS data from a filesystem path +type FilesystemClient struct { + lfsdir string +} + +func newFilesystenClient(path string) *FilesystemClient { + lfsdir := filepath.Join(path, "lfs", "objects") + + client := &FilesystemClient{lfsdir} + + return client +} + +func (c *FilesystemClient) objectPath(oid string) string { + return filepath.Join(c.lfsdir, oid[0:2], oid[2:4], oid) +} + +// Download reads the specific LFS object from the target repository +func (c *FilesystemClient) Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) { + objectPath := c.objectPath(oid) + + if _, err := os.Stat(objectPath); os.IsNotExist(err) { + return nil, err + } + + file, err := os.Open(objectPath) + if err != nil { + return nil, err + } + + return file, nil +} diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go new file mode 100644 index 000000000000..39e915db72c7 --- /dev/null +++ b/modules/lfs/http_client.go @@ -0,0 +1,123 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + + "code.gitea.io/gitea/modules/log" +) + +// Client is used to communicate with the LFS server +type HTTPClient struct { + client *http.Client + endpoint string + transfers map[string]TransferAdapter +} + +func newHTTPClient(endpoint string) *HTTPClient { + hc := &http.Client{} + + client := &HTTPClient{hc, endpoint, make(map[string]TransferAdapter)} + + basic := &BasicTransferAdapter{hc} + + client.transfers[basic.Name()] = basic + + return client +} + +func (c *HTTPClient) transferNames() []string { + keys := make([]string, len(c.transfers)) + + i := 0 + for k := range c.transfers { + keys[i] = k + i++ + } + + return keys +} + +func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Pointer) (*BatchResponse, error) { + url := fmt.Sprintf("%s/objects/batch", strings.TrimSuffix(c.endpoint, "/")) + + request := &BatchRequest{operation, c.transferNames(), nil, objects} + + payload := new(bytes.Buffer) + err := json.NewEncoder(payload).Encode(request) + if err != nil { + return nil, fmt.Errorf("lfs.HTTPClient.batch json.Encode: %w", err) + } + + log.Trace("lfs.HTTPClient.batch NewRequestWithContext: %s", url) + + req, err := http.NewRequestWithContext(ctx, "POST", url, payload) + if err != nil { + return nil, fmt.Errorf("lfs.HTTPClient.batch http.NewRequestWithContext: %w", err) + } + req.Header.Set("Content-type", MediaType) + req.Header.Set("Accept", MediaType) + + res, err := c.client.Do(req) + if err != nil { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + return nil, fmt.Errorf("lfs.HTTPClient.batch http.Do: %w", err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("lfs.HTTPClient.batch: Unexpected servers response: %s", res.Status) + } + + var response BatchResponse + err = json.NewDecoder(res.Body).Decode(&response) + if err != nil { + return nil, fmt.Errorf("lfs.HTTPClient.batch json.Decode: %w", err) + } + + if len(response.Transfer) == 0 { + response.Transfer = "basic" + } + + return &response, nil +} + +// Download reads the specific LFS object from the LFS server +func (c *HTTPClient) Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) { + var objects []Pointer + objects = append(objects, Pointer{oid, size}) + + result, err := c.batch(ctx, "download", objects) + if err != nil { + return nil, err + } + + transferAdapter, ok := c.transfers[result.Transfer] + if !ok { + return nil, fmt.Errorf("lfs.HTTPClient.Download Transferadapter not found: %s", result.Transfer) + } + + if len(result.Objects) == 0 { + return nil, errors.New("lfs.HTTPClient.Download: No objects in result") + } + + content, err := transferAdapter.Download(ctx, result.Objects[0]) + if err != nil { + return nil, err + } + return content, nil +} diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 9e44deb54fd3..37fc9b94c414 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -7,7 +7,7 @@ package repository import ( "context" "fmt" - "net/http" + "net/url" "path" "strings" "time" @@ -124,7 +124,8 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. } if opts.LFS { - if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, opts.LFSEndpoint); err != nil { + ep := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint) + if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep); err != nil { log.Error("Failed to store missing LFS objects for repository: %v", err) } } @@ -310,8 +311,8 @@ func PushUpdateAddTag(repo *models.Repository, gitRepo *git.Repository, tagName } // StoreMissingLfsObjectsInRepository downloads missing LFS objects -func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, lfsAddr string) error { - client := lfs.NewClient(&http.Client{}) +func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL) error { + client := lfs.NewClient(endpoint) contentStore := lfs.NewContentStore() pointerBlobs, err := lfs.SearchPointerBlobs(gitRepo) @@ -348,7 +349,7 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi continue } - stream, err := client.Download(ctx, lfsAddr, pointerBlob.Oid, pointerBlob.Size) + stream, err := client.Download(ctx, pointerBlob.Oid, pointerBlob.Size) if err != nil { select { case <-ctx.Done(): diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index bb26e4514703..f7cee5aecfac 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" repo_module "code.gitea.io/gitea/modules/repository" @@ -261,8 +262,9 @@ func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) } log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) - lfsAddr, _ := remoteAddress(m.Repo.RepoPath()) - if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, lfsAddr); err != nil { // TODO support custom endpoint + readAddress(m) + ep := lfs.DetermineEndpoint(m.Address, "") // TODO support custom endpoint + if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep); err != nil { log.Error("Failed to synchronize LFS objects for repository: %v", err) } From 0ef54f7f9d0fe898473732059368323ce7cabbc8 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 8 Mar 2021 22:01:00 +0000 Subject: [PATCH 39/69] Show an error on invalid LFS endpoints. --- options/locale/locale_en-US.ini | 1 + routers/repo/migrate.go | 13 +++++++++---- templates/repo/migrate/options.tmpl | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 5564e73b8184..c2a3ee281df2 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -792,6 +792,7 @@ migrate.clone_address_desc = The HTTP(S) or Git 'clone' URL of an existing repos migrate.clone_local_path = or a local server path migrate.permission_denied = You are not allowed to import local repositories. migrate.invalid_local_path = "The local path is invalid. It does not exist or is not a directory." +migrate.invalid_lfs_endpoint = "The LFS endpoint is not valid." migrate.failed = Migration failed: %v migrate.migrate_items_options = Access Token is required to migrate additional items migrated_from = Migrated from %[2]s diff --git a/routers/repo/migrate.go b/routers/repo/migrate.go index 7bd173a9ba8a..5fd298fa85cf 100644 --- a/routers/repo/migrate.go +++ b/routers/repo/migrate.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" auth "code.gitea.io/gitea/modules/forms" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" @@ -160,9 +161,13 @@ func MigratePost(ctx *context.Context) { return } - lfsEndpoint := remoteAddr - if len(form.LFSEndpoint) > 0 { - lfsEndpoint = form.LFSEndpoint + if form.LFS && len(form.LFSEndpoint) > 0 { + ep := lfs.DetermineEndpoint("", form.LFSEndpoint) + if ep == nil { + ctx.Data["Err_LFSEndpoint"] = true + ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_lfs_endpoint"), tpl, &form) + return + } } var opts = migrations.MigrateOptions{ @@ -174,7 +179,7 @@ func MigratePost(ctx *context.Context) { Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror && !setting.Repository.DisableMirrors, LFS: form.LFS, - LFSEndpoint: lfsEndpoint, + LFSEndpoint: form.LFSEndpoint, AuthUsername: form.AuthUsername, AuthPassword: form.AuthPassword, AuthToken: form.AuthToken, diff --git a/templates/repo/migrate/options.tmpl b/templates/repo/migrate/options.tmpl index 3990ba0b8ce9..720093ba9813 100644 --- a/templates/repo/migrate/options.tmpl +++ b/templates/repo/migrate/options.tmpl @@ -19,7 +19,7 @@
{{.i18n.Tr "repo.migrate_options_lfs_endpoint"}} -
+
From a44781f978e3d18df367b1a2c7b9bcbf3d6da9a6 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 8 Mar 2021 22:04:00 +0000 Subject: [PATCH 40/69] Hide settings if not used. --- templates/repo/migrate/options.tmpl | 10 ++++++---- web_src/js/features/migration.js | 7 ++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/templates/repo/migrate/options.tmpl b/templates/repo/migrate/options.tmpl index 720093ba9813..24a694f6d4ea 100644 --- a/templates/repo/migrate/options.tmpl +++ b/templates/repo/migrate/options.tmpl @@ -18,9 +18,11 @@
-{{.i18n.Tr "repo.migrate_options_lfs_endpoint"}} -
- - +
+ {{.i18n.Tr "repo.migrate_options_lfs_endpoint"}} +
+ + +
{{end}} \ No newline at end of file diff --git a/web_src/js/features/migration.js b/web_src/js/features/migration.js index 597d32728249..5867137f2b62 100644 --- a/web_src/js/features/migration.js +++ b/web_src/js/features/migration.js @@ -9,12 +9,13 @@ const $items = $('#migrate_items').find('input[type=checkbox]'); export default function initMigration() { checkAuth(); + setLFSEndpointVisibility(); $user.on('keyup', () => {checkItems(false)}); $pass.on('keyup', () => {checkItems(false)}); $token.on('keyup', () => {checkItems(true)}); $mirror.on('change', () => {checkItems(true)}); - $lfs.on('change', () => {$lfsEndpoint.attr('disabled', !$lfs.is(':checked'))}); + $lfs.on('change', setLFSEndpointVisibility); const $cloneAddr = $('#clone_addr'); $cloneAddr.on('change', () => { @@ -52,3 +53,7 @@ function checkItems(tokenAuth) { $items.attr('disabled', true); } } + +function setLFSEndpointVisibility() { + $lfsEndpoint.toggle($lfs.is(':checked')); +} From 856416ac65d7d964747b883728a6d64c456f9fdd Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 10 Mar 2021 16:40:36 +0000 Subject: [PATCH 41/69] Added LFS info to mirror struct. --- models/migrations/migrations.go | 2 ++ models/migrations/v175.go | 18 ++++++++++++++++++ models/repo_mirror.go | 3 +++ modules/repository/repo.go | 4 ++++ routers/repo/migrate.go | 2 ++ services/mirror/mirror.go | 12 +++++++----- 6 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 models/migrations/v175.go diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 27c159ee4ade..584c4651875f 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -296,6 +296,8 @@ var migrations = []Migration{ NewMigration("Add time_id column to Comment", addTimeIDCommentColumn), // v174 -> v175 NewMigration("create repo transfer table", addRepoTransfer), + // v175 -> v176 + NewMigration("Add LFS columns to Mirror", addLFSMirrorColumns), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v175.go b/models/migrations/v175.go new file mode 100644 index 000000000000..c2a9af618e43 --- /dev/null +++ b/models/migrations/v175.go @@ -0,0 +1,18 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "xorm.io/xorm" +) + +func addLFSMirrorColumns(x *xorm.Engine) error { + type Mirror struct { + LFS bool `xorm:"lfs_enabled NOT NULL DEFAULT false"` + LFSEndpoint string `xorm:"lfs_endpoint TEXT"` + } + + return x.Sync2(new(Mirror)) +} diff --git a/models/repo_mirror.go b/models/repo_mirror.go index 10b0a7b1396d..2c37b54aa99b 100644 --- a/models/repo_mirror.go +++ b/models/repo_mirror.go @@ -25,6 +25,9 @@ type Mirror struct { UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"` NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"` + LFS bool `xorm:"lfs_enabled NOT NULL DEFAULT false"` + LFSEndpoint string `xorm:"lfs_endpoint TEXT"` + Address string `xorm:"-"` } diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 37fc9b94c414..2edff23331cc 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -141,6 +141,10 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models. Interval: setting.Mirror.DefaultInterval, EnablePrune: true, NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval), + LFS: opts.LFS, + } + if opts.LFS { + mirrorModel.LFSEndpoint = opts.LFSEndpoint } if opts.MirrorInterval != "" { diff --git a/routers/repo/migrate.go b/routers/repo/migrate.go index 5fd298fa85cf..e2c2dd67179b 100644 --- a/routers/repo/migrate.go +++ b/routers/repo/migrate.go @@ -161,6 +161,8 @@ func MigratePost(ctx *context.Context) { return } + form.LFS = form.LFS && setting.LFS.StartServer + if form.LFS && len(form.LFSEndpoint) > 0 { ep := lfs.DetermineEndpoint("", form.LFSEndpoint) if ep == nil { diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index f7cee5aecfac..9e2dde85fca2 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -261,11 +261,13 @@ func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) log.Error("Failed to synchronize tags to releases for repository: %v", err) } - log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) - readAddress(m) - ep := lfs.DetermineEndpoint(m.Address, "") // TODO support custom endpoint - if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep); err != nil { - log.Error("Failed to synchronize LFS objects for repository: %v", err) + if m.LFS && setting.LFS.StartServer { + log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) + readAddress(m) + ep := lfs.DetermineEndpoint(m.Address, m.LFSEndpoint) + if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep); err != nil { + log.Error("Failed to synchronize LFS objects for repository: %v", err) + } } log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo) From 5e9c6066364649a7b67ef1042ec37f970b797904 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 10 Mar 2021 16:40:44 +0000 Subject: [PATCH 42/69] Fixed comment. --- modules/lfs/http_client.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index 39e915db72c7..0b71323c905c 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -17,7 +17,8 @@ import ( "code.gitea.io/gitea/modules/log" ) -// Client is used to communicate with the LFS server +// HTTPClient is used to communicate with the LFS server +// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md type HTTPClient struct { client *http.Client endpoint string From c0b5e5053871d7a86db662050c5f9e46a0a370b3 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 10 Mar 2021 18:05:57 +0000 Subject: [PATCH 43/69] Check LFS endpoint. --- routers/api/v1/repo/migrate.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index 9cc4880b9f86..ba0212926a78 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/convert" auth "code.gitea.io/gitea/modules/forms" "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/migrations/base" @@ -128,9 +129,14 @@ func Migrate(ctx *context.APIContext) { return } - lfsEndpoint := remoteAddr - if len(form.LFSEndpoint) > 0 { - lfsEndpoint = form.LFSEndpoint + form.LFS = form.LFS && setting.LFS.StartServer + + if form.LFS && len(form.LFSEndpoint) > 0 { + ep := lfs.DetermineEndpoint("", form.LFSEndpoint) + if ep == nil { + ctx.Error(http.StatusInternalServerError, "", ctx.Tr("repo.migrate.invalid_lfs_endpoint")) + return + } } var opts = migrations.MigrateOptions{ @@ -140,7 +146,7 @@ func Migrate(ctx *context.APIContext) { Private: form.Private || setting.Repository.ForcePrivate, Mirror: form.Mirror, LFS: form.LFS, - LFSEndpoint: lfsEndpoint, + LFSEndpoint: form.LFSEndpoint, AuthUsername: form.AuthUsername, AuthPassword: form.AuthPassword, AuthToken: form.AuthToken, From 4c127de4cb29d507f0146bfe08cd2afee5b9ca54 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 10 Mar 2021 18:13:20 +0000 Subject: [PATCH 44/69] Manage LFS settings from mirror page. --- modules/forms/repo_form.go | 2 ++ options/locale/locale_en-US.ini | 6 +++++- routers/repo/setting.go | 19 +++++++++++++++++++ templates/repo/settings/options.tmpl | 19 +++++++++++++++++-- 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/modules/forms/repo_form.go b/modules/forms/repo_form.go index e7cd377b6dad..5efa75cfd576 100644 --- a/modules/forms/repo_form.go +++ b/modules/forms/repo_form.go @@ -138,6 +138,8 @@ type RepoSettingForm struct { MirrorAddress string MirrorUsername string MirrorPassword string + LFS bool `form:"mirror_lfs"` + LFSEndpoint string `form:"mirror_lfs_endpoint"` Private bool Template bool EnablePrune bool diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index c2a3ee281df2..33973981450a 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -718,6 +718,10 @@ mirror_address = Clone From URL mirror_address_desc = Put any required credentials in the Clone Authorization section. mirror_address_url_invalid = The provided url is invalid. You must escape all components of the url correctly. mirror_address_protocol_invalid = The provided url is invalid. Only http(s):// or git:// locations can be mirrored from. +mirror_lfs = Large File System (LFS) +mirror_lfs_desc = Activate mirroring of LFS data. +mirror_lfs_endpoint = LFS Endpoint +mirror_lfs_endpoint_desc = Sync will attempt to use the clone url to determine the LFS server. You can also configure a custom LFS server. mirror_last_synced = Last Synchronized watchers = Watchers stargazers = Stargazers @@ -792,7 +796,7 @@ migrate.clone_address_desc = The HTTP(S) or Git 'clone' URL of an existing repos migrate.clone_local_path = or a local server path migrate.permission_denied = You are not allowed to import local repositories. migrate.invalid_local_path = "The local path is invalid. It does not exist or is not a directory." -migrate.invalid_lfs_endpoint = "The LFS endpoint is not valid." +migrate.invalid_lfs_endpoint = The LFS endpoint is not valid. migrate.failed = Migration failed: %v migrate.migrate_items_options = Access Token is required to migrate additional items migrated_from = Migrated from %[2]s diff --git a/routers/repo/setting.go b/routers/repo/setting.go index 692d65b44ce1..2f1625786019 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/context" auth "code.gitea.io/gitea/modules/forms" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" @@ -208,6 +209,24 @@ func SettingsPost(ctx *context.Context) { return } + form.LFS = form.LFS && setting.LFS.StartServer + + if len(form.LFSEndpoint) > 0 { + ep := lfs.DetermineEndpoint("", form.LFSEndpoint) + if ep == nil { + ctx.Data["Err_LFSEndpoint"] = true + ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_lfs_endpoint"), tplSettingsOptions, &form) + return + } + } + + ctx.Repo.Mirror.LFS = form.LFS + ctx.Repo.Mirror.LFSEndpoint = form.LFSEndpoint + if err := models.UpdateMirror(ctx.Repo.Mirror); err != nil { + ctx.ServerError("UpdateMirror", err) + return + } + ctx.Flash.Success(ctx.Tr("repo.settings.update_settings_success")) ctx.Redirect(repo.Link() + "/settings") diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 9d87101671ff..2de7904ba6f6 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -81,8 +81,8 @@
- - + +
@@ -112,6 +112,21 @@
+ {{if .LFSStartServer}} +
+ +
+ + +
+
+
+ + +

{{.i18n.Tr "repo.mirror_lfs_endpoint_desc"}}

+
+ {{end}} +
From 9cf903cbf6bdf03f196f4de86ff0fc61d6377b14 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 10 Mar 2021 18:32:11 +0000 Subject: [PATCH 45/69] Fixed selector. --- web_src/js/features/migration.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/web_src/js/features/migration.js b/web_src/js/features/migration.js index 5867137f2b62..fd1282ebdff1 100644 --- a/web_src/js/features/migration.js +++ b/web_src/js/features/migration.js @@ -4,7 +4,8 @@ const $pass = $('#auth_password'); const $token = $('#auth_token'); const $mirror = $('#mirror'); const $lfs = $('#lfs'); -const $lfsEndpoint = $('#lfs_endpoint'); +const $lfsEndpointBlock = $('#lfs_endpoint'); +const $lfsEndpoint = $lfsEndpointBlock.find('input'); const $items = $('#migrate_items').find('input[type=checkbox]'); export default function initMigration() { @@ -55,5 +56,5 @@ function checkItems(tokenAuth) { } function setLFSEndpointVisibility() { - $lfsEndpoint.toggle($lfs.is(':checked')); + $lfsEndpointBlock.toggle($lfs.is(':checked')); } From 745812b93bc7ec96cb8276ce337be86a893c1b8e Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 10 Mar 2021 18:41:48 +0000 Subject: [PATCH 46/69] Adjusted selector. --- web_src/js/features/migration.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/js/features/migration.js b/web_src/js/features/migration.js index fd1282ebdff1..1341871f1795 100644 --- a/web_src/js/features/migration.js +++ b/web_src/js/features/migration.js @@ -5,7 +5,7 @@ const $token = $('#auth_token'); const $mirror = $('#mirror'); const $lfs = $('#lfs'); const $lfsEndpointBlock = $('#lfs_endpoint'); -const $lfsEndpoint = $lfsEndpointBlock.find('input'); +const $lfsEndpoint = $lfsEndpointBlock.find('input:text'); const $items = $('#migrate_items').find('input[type=checkbox]'); export default function initMigration() { From 8808d778b3308fc5a8d5b96c2a26ee4f324f0801 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 11 Mar 2021 20:08:02 +0000 Subject: [PATCH 47/69] Added more tests. --- integrations/lfs_local_endpoint_test.go | 117 ++++++++++++++++++++++++ modules/lfs/client_test.go | 23 +++++ modules/lfs/endpoint.go | 2 +- modules/lfs/endpoint_test.go | 63 +++++++++++++ 4 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 integrations/lfs_local_endpoint_test.go create mode 100644 modules/lfs/client_test.go create mode 100644 modules/lfs/endpoint_test.go diff --git a/integrations/lfs_local_endpoint_test.go b/integrations/lfs_local_endpoint_test.go new file mode 100644 index 000000000000..eda418c42949 --- /dev/null +++ b/integrations/lfs_local_endpoint_test.go @@ -0,0 +1,117 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integrations + +import ( + "fmt" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "testing" + + "code.gitea.io/gitea/modules/lfs" + + "github.com/stretchr/testify/assert" +) + +func str2url(raw string) *url.URL { + u, _ := url.Parse(raw) + return u +} + +func TestDetermineLocalEndpoint(t *testing.T) { + defer prepareTestEnv(t)() + + root, _ := ioutil.TempDir("", "lfs_test") + defer os.RemoveAll(root) + + rootdotgit, _ := ioutil.TempDir("", "lfs_test") + defer os.RemoveAll(rootdotgit) + os.Mkdir(filepath.Join(rootdotgit, ".git"), 0700) + + lfsroot, _ := ioutil.TempDir("", "lfs_test") + defer os.RemoveAll(lfsroot) + + // Test cases + var cases = []struct { + cloneurl string + lfsurl string + expected *url.URL + }{ + // case 0 + { + cloneurl: root, + lfsurl: "", + expected: str2url(fmt.Sprintf("file://%s", root)), + }, + // case 1 + { + cloneurl: root, + lfsurl: lfsroot, + expected: str2url(fmt.Sprintf("file://%s", lfsroot)), + }, + // case 2 + { + cloneurl: "https://git.com/repo.git", + lfsurl: lfsroot, + expected: str2url(fmt.Sprintf("file://%s", lfsroot)), + }, + // case 3 + { + cloneurl: rootdotgit, + lfsurl: "", + expected: str2url(fmt.Sprintf("file://%s", filepath.Join(rootdotgit, ".git"))), + }, + // case 4 + { + cloneurl: "", + lfsurl: rootdotgit, + expected: str2url(fmt.Sprintf("file://%s", filepath.Join(rootdotgit, ".git"))), + }, + // case 5 + { + cloneurl: rootdotgit, + lfsurl: rootdotgit, + expected: str2url(fmt.Sprintf("file://%s", filepath.Join(rootdotgit, ".git"))), + }, + // case 6 + { + cloneurl: fmt.Sprintf("file://%s", root), + lfsurl: "", + expected: str2url(fmt.Sprintf("file://%s", root)), + }, + // case 7 + { + cloneurl: fmt.Sprintf("file://%s", root), + lfsurl: fmt.Sprintf("file://%s", lfsroot), + expected: str2url(fmt.Sprintf("file://%s", lfsroot)), + }, + // case 8 + { + cloneurl: root, + lfsurl: fmt.Sprintf("file://%s", lfsroot), + expected: str2url(fmt.Sprintf("file://%s", lfsroot)), + }, + // case 9 + { + cloneurl: "", + lfsurl: "/does/not/exist", + expected: nil, + }, + // case 10 + { + cloneurl: "", + lfsurl: "file:///does/not/exist", + expected: str2url("file:///does/not/exist"), + }, + } + + for n, c := range cases { + ep := lfs.DetermineEndpoint(c.cloneurl, c.lfsurl) + + assert.Equal(t, c.expected, ep, "case %d: error should match", n) + } +} diff --git a/modules/lfs/client_test.go b/modules/lfs/client_test.go new file mode 100644 index 000000000000..d4eb00546948 --- /dev/null +++ b/modules/lfs/client_test.go @@ -0,0 +1,23 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "net/url" + + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewClient(t *testing.T) { + u, _ := url.Parse("file:///test") + c := NewClient(u) + assert.IsType(t, &FilesystemClient{}, c) + + u, _ = url.Parse("https://test.com/lfs") + c = NewClient(u) + assert.IsType(t, &HTTPClient{}, c) +} diff --git a/modules/lfs/endpoint.go b/modules/lfs/endpoint.go index 9f3941802bf7..8183b305ee98 100644 --- a/modules/lfs/endpoint.go +++ b/modules/lfs/endpoint.go @@ -61,7 +61,7 @@ func endpointFromURL(rawurl string) *url.URL { case "http", "https": return u case "file": - return endpointFromLocalPath(u.Path) + return u default: if _, err := os.Stat(rawurl); err == nil { return endpointFromLocalPath(rawurl) diff --git a/modules/lfs/endpoint_test.go b/modules/lfs/endpoint_test.go new file mode 100644 index 000000000000..d2dd6147ece7 --- /dev/null +++ b/modules/lfs/endpoint_test.go @@ -0,0 +1,63 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/assert" +) + +func str2url(raw string) *url.URL { + u, _ := url.Parse(raw) + return u +} + +func TestDetermineEndpoint(t *testing.T) { + // Test cases + var cases = []struct { + cloneurl string + lfsurl string + expected *url.URL + }{ + // case 0 + { + cloneurl: "", + lfsurl: "", + expected: nil, + }, + // case 1 + { + cloneurl: "https://git.com/repo", + lfsurl: "", + expected: str2url("https://git.com/repo.git/info/lfs"), + }, + // case 2 + { + cloneurl: "https://git.com/repo.git", + lfsurl: "", + expected: str2url("https://git.com/repo.git/info/lfs"), + }, + // case 3 + { + cloneurl: "", + lfsurl: "https://gitlfs.com/repo", + expected: str2url("https://gitlfs.com/repo"), + }, + // case 4 + { + cloneurl: "https://git.com/repo.git", + lfsurl: "https://gitlfs.com/repo", + expected: str2url("https://gitlfs.com/repo"), + }, + } + + for n, c := range cases { + ep := DetermineEndpoint(c.cloneurl, c.lfsurl) + + assert.Equal(t, c.expected, ep, "case %d: error should match", n) + } +} From 4fcc2f6bb95e6b4d40964461c74027d1a07f5f44 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 11 Mar 2021 22:01:14 +0000 Subject: [PATCH 48/69] Added local filesystem migration test. --- integrations/api_repo_lfs_migrate_test.go | 44 ++++++++++++++++++ .../migration/lfs-test.git/HEAD | 1 + .../migration/lfs-test.git/config | 7 +++ .../migration/lfs-test.git/description | 1 + .../lfs-test.git/hooks/post-checkout | 3 ++ .../migration/lfs-test.git/hooks/post-commit | 3 ++ .../migration/lfs-test.git/hooks/post-merge | 3 ++ .../migration/lfs-test.git/hooks/pre-push | 3 ++ .../migration/lfs-test.git/index | Bin 0 -> 305 bytes ...15326465fa96c3bfd54a4ea06cfd6dbbd8340e0152 | 1 + ...a92395be4d16f2f63116caf36c8ad35c60831ab041 | 1 + .../54/6244003622c64b2fc3c2cd544d7a29882c8383 | Bin 0 -> 128 bytes .../6a/6ccf5d874fec134ee712572cc03a0f2dd7afec | Bin 0 -> 51 bytes .../a6/7134b8484c2abe9fa954e1fd83b39b271383ed | Bin 0 -> 121 bytes .../b7/01ed6ffe410f0c3ac204b929ea47cfec6cef54 | Bin 0 -> 122 bytes .../f2/07b74f55cd7f9e800b7550d587cbc488f6eaf1 | Bin 0 -> 120 bytes .../migration/lfs-test.git/refs/heads/master | 1 + 17 files changed, 68 insertions(+) create mode 100644 integrations/api_repo_lfs_migrate_test.go create mode 100644 integrations/gitea-repositories-meta/migration/lfs-test.git/HEAD create mode 100644 integrations/gitea-repositories-meta/migration/lfs-test.git/config create mode 100644 integrations/gitea-repositories-meta/migration/lfs-test.git/description create mode 100644 integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/post-checkout create mode 100644 integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/post-commit create mode 100644 integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/post-merge create mode 100644 integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/pre-push create mode 100644 integrations/gitea-repositories-meta/migration/lfs-test.git/index create mode 100644 integrations/gitea-repositories-meta/migration/lfs-test.git/lfs/objects/d6/f1/d6f175817f886ec6fbbc1515326465fa96c3bfd54a4ea06cfd6dbbd8340e0152 create mode 100644 integrations/gitea-repositories-meta/migration/lfs-test.git/lfs/objects/fb/8f/fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041 create mode 100644 integrations/gitea-repositories-meta/migration/lfs-test.git/objects/54/6244003622c64b2fc3c2cd544d7a29882c8383 create mode 100644 integrations/gitea-repositories-meta/migration/lfs-test.git/objects/6a/6ccf5d874fec134ee712572cc03a0f2dd7afec create mode 100644 integrations/gitea-repositories-meta/migration/lfs-test.git/objects/a6/7134b8484c2abe9fa954e1fd83b39b271383ed create mode 100644 integrations/gitea-repositories-meta/migration/lfs-test.git/objects/b7/01ed6ffe410f0c3ac204b929ea47cfec6cef54 create mode 100644 integrations/gitea-repositories-meta/migration/lfs-test.git/objects/f2/07b74f55cd7f9e800b7550d587cbc488f6eaf1 create mode 100644 integrations/gitea-repositories-meta/migration/lfs-test.git/refs/heads/master diff --git a/integrations/api_repo_lfs_migrate_test.go b/integrations/api_repo_lfs_migrate_test.go new file mode 100644 index 000000000000..f522da79afa0 --- /dev/null +++ b/integrations/api_repo_lfs_migrate_test.go @@ -0,0 +1,44 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integrations + +import ( + "net/http" + "path" + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + + "github.com/stretchr/testify/assert" +) + +func TestAPIRepoLFSMigrateLocal(t *testing.T) { + defer prepareTestEnv(t)() + + setting.ImportLocalPaths = true + setting.Migrations.AllowLocalNetworks = true + + user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User) + session := loginUser(t, user.Name) + token := getTokenForLoggedInUser(t, session) + + req := NewRequestWithJSON(t, "POST", "/api/v1/repos/migrate?token="+token, &api.MigrateRepoOptions{ + CloneAddr: path.Join(setting.RepoRootPath, "migration/lfs-test.git"), + RepoOwnerID: user.ID, + RepoName: "lfs-test-local", + LFS: true, + }) + resp := MakeRequest(t, req, NoExpectedStatus) + assert.EqualValues(t, http.StatusCreated, resp.Code) + + store := lfs.NewContentStore() + ok, _ := store.Verify(lfs.Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6}) + assert.True(t, ok) + ok, _ = store.Verify(lfs.Pointer{Oid: "d6f175817f886ec6fbbc1515326465fa96c3bfd54a4ea06cfd6dbbd8340e0152", Size: 6}) + assert.True(t, ok) +} diff --git a/integrations/gitea-repositories-meta/migration/lfs-test.git/HEAD b/integrations/gitea-repositories-meta/migration/lfs-test.git/HEAD new file mode 100644 index 000000000000..cb089cd89a7d --- /dev/null +++ b/integrations/gitea-repositories-meta/migration/lfs-test.git/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/integrations/gitea-repositories-meta/migration/lfs-test.git/config b/integrations/gitea-repositories-meta/migration/lfs-test.git/config new file mode 100644 index 000000000000..3f8f41b6b413 --- /dev/null +++ b/integrations/gitea-repositories-meta/migration/lfs-test.git/config @@ -0,0 +1,7 @@ +[core] + bare = false + repositoryformatversion = 0 + filemode = false + symlinks = false + ignorecase = true + logallrefupdates = true diff --git a/integrations/gitea-repositories-meta/migration/lfs-test.git/description b/integrations/gitea-repositories-meta/migration/lfs-test.git/description new file mode 100644 index 000000000000..498b267a8c78 --- /dev/null +++ b/integrations/gitea-repositories-meta/migration/lfs-test.git/description @@ -0,0 +1 @@ +Unnamed repository; edit this file 'description' to name the repository. diff --git a/integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/post-checkout b/integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/post-checkout new file mode 100644 index 000000000000..cab40f264952 --- /dev/null +++ b/integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/post-checkout @@ -0,0 +1,3 @@ +#!/bin/sh +command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-checkout.\n"; exit 2; } +git lfs post-checkout "$@" diff --git a/integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/post-commit b/integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/post-commit new file mode 100644 index 000000000000..9443f4161aca --- /dev/null +++ b/integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/post-commit @@ -0,0 +1,3 @@ +#!/bin/sh +command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-commit.\n"; exit 2; } +git lfs post-commit "$@" diff --git a/integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/post-merge b/integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/post-merge new file mode 100644 index 000000000000..828b70891edd --- /dev/null +++ b/integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/post-merge @@ -0,0 +1,3 @@ +#!/bin/sh +command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-merge.\n"; exit 2; } +git lfs post-merge "$@" diff --git a/integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/pre-push b/integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/pre-push new file mode 100644 index 000000000000..81a9cc6398ab --- /dev/null +++ b/integrations/gitea-repositories-meta/migration/lfs-test.git/hooks/pre-push @@ -0,0 +1,3 @@ +#!/bin/sh +command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/pre-push.\n"; exit 2; } +git lfs pre-push "$@" diff --git a/integrations/gitea-repositories-meta/migration/lfs-test.git/index b/integrations/gitea-repositories-meta/migration/lfs-test.git/index new file mode 100644 index 0000000000000000000000000000000000000000..13f8e269660d5d60e2d3747d4688df40ff6b2a0d GIT binary patch literal 305 zcmZ?q402{*U|<4b<^(UZREaCfHDLPj#0NkzC}>;)V*zRHteo?)?f!3s{hkYj>m0D+ z*S)^}4FjKEdS*#tNl8&=QfWzQF-SeooP4u2OKgC2z3Hv%NM@jz$F`mEZT>$;ejcks zEITz{xu1WN^FD-uE2T6yx6)9rq@o07H_)7Nk;12YfOIS8!EY@vaRvrd^OhBw?C|i> z+Bbh?$iu(Qn`f&FH@}6NXM}8CNRX>5&o;sGKdLPAd**?OOygltrJ_tG{!-|cN49(oF?F0X4_)6S)} iwFG{is$9OZK}&8Rua95=6#tH0{%KX0y{ivD?=_3V$jknsVGrM%giZBEwar?D^^I!OiKeZa#M@aQ^73VlGKV4 JE&!Yf)7|4ASw1Y1c)$GrBiATzFzmTymi)K>2{T4wEouh zJom90^K9pRlQI3TAD&WwnX#8-E>OK?J!-T_YbJpO11u5(l|;gv00EsPPbl$0$+2Yg b(U&+=l1!O72j}N916noOw`fv7i{B|zy#F>6 literal 0 HcmV?d00001 diff --git a/integrations/gitea-repositories-meta/migration/lfs-test.git/objects/b7/01ed6ffe410f0c3ac204b929ea47cfec6cef54 b/integrations/gitea-repositories-meta/migration/lfs-test.git/objects/b7/01ed6ffe410f0c3ac204b929ea47cfec6cef54 new file mode 100644 index 0000000000000000000000000000000000000000..554b7f05b09ee2563788a3153e0ac1d97ad60d7e GIT binary patch literal 122 zcmV-=0EPc}0S(1T62c%506@>Yf)5xnEEC9YhGm4RC@H}le7*Li c05B07OrElF30H4`;%Y&R_AL^{4|sMcnfgXLZ~y=R literal 0 HcmV?d00001 diff --git a/integrations/gitea-repositories-meta/migration/lfs-test.git/objects/f2/07b74f55cd7f9e800b7550d587cbc488f6eaf1 b/integrations/gitea-repositories-meta/migration/lfs-test.git/objects/f2/07b74f55cd7f9e800b7550d587cbc488f6eaf1 new file mode 100644 index 0000000000000000000000000000000000000000..ae6fdce5a291b5c5bfc237ea2eab033ea97e5842 GIT binary patch literal 120 zcmV-;0Ehp00V^p=O;s>7v|unaFfcPQQP4}zEJ-XWDauSLElDkA$jUh%+wT8H*zdVe zxXuA9e%A3`7Ho%ZZVL~Q97Ui literal 0 HcmV?d00001 diff --git a/integrations/gitea-repositories-meta/migration/lfs-test.git/refs/heads/master b/integrations/gitea-repositories-meta/migration/lfs-test.git/refs/heads/master new file mode 100644 index 000000000000..cd602fb935db --- /dev/null +++ b/integrations/gitea-repositories-meta/migration/lfs-test.git/refs/heads/master @@ -0,0 +1 @@ +546244003622c64b2fc3c2cd544d7a29882c8383 From 3283e58d62796ffcc208ccbc29b4053de4898006 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 11 Mar 2021 22:18:36 +0000 Subject: [PATCH 49/69] Fixed typo. --- modules/lfs/client.go | 2 +- modules/lfs/filesystem_client.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/lfs/client.go b/modules/lfs/client.go index ec5d8246fdee..d783a300c05a 100644 --- a/modules/lfs/client.go +++ b/modules/lfs/client.go @@ -18,7 +18,7 @@ type Client interface { // NewClient creates a LFS client func NewClient(endpoint *url.URL) Client { if endpoint.Scheme == "file" { - return newFilesystenClient(endpoint.Path) + return newFilesystemClient(endpoint.Path) } return newHTTPClient(endpoint.String()) } diff --git a/modules/lfs/filesystem_client.go b/modules/lfs/filesystem_client.go index 3d12db3eb819..638ac59cd57c 100644 --- a/modules/lfs/filesystem_client.go +++ b/modules/lfs/filesystem_client.go @@ -16,7 +16,7 @@ type FilesystemClient struct { lfsdir string } -func newFilesystenClient(path string) *FilesystemClient { +func newFilesystemClient(path string) *FilesystemClient { lfsdir := filepath.Join(path, "lfs", "objects") client := &FilesystemClient{lfsdir} From d1904038e7b31bbaddc5709a6bcecdc4025ae861 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 11 Mar 2021 22:34:05 +0000 Subject: [PATCH 50/69] Reset settings. --- integrations/api_repo_lfs_migrate_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/integrations/api_repo_lfs_migrate_test.go b/integrations/api_repo_lfs_migrate_test.go index f522da79afa0..7280658b74a9 100644 --- a/integrations/api_repo_lfs_migrate_test.go +++ b/integrations/api_repo_lfs_migrate_test.go @@ -20,6 +20,8 @@ import ( func TestAPIRepoLFSMigrateLocal(t *testing.T) { defer prepareTestEnv(t)() + oldImportLocalPaths := setting.ImportLocalPaths + oldAllowLocalNetworks := setting.Migrations.AllowLocalNetworks setting.ImportLocalPaths = true setting.Migrations.AllowLocalNetworks = true @@ -41,4 +43,7 @@ func TestAPIRepoLFSMigrateLocal(t *testing.T) { assert.True(t, ok) ok, _ = store.Verify(lfs.Pointer{Oid: "d6f175817f886ec6fbbc1515326465fa96c3bfd54a4ea06cfd6dbbd8340e0152", Size: 6}) assert.True(t, ok) + + setting.ImportLocalPaths = oldImportLocalPaths + setting.Migrations.AllowLocalNetworks = oldAllowLocalNetworks } From 4e864f30853c85777c70652a09ebe40c64bf7052 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 11 Mar 2021 23:01:00 +0000 Subject: [PATCH 51/69] Added special windows path handling. --- modules/lfs/client.go | 4 ++-- modules/lfs/filesystem_client.go | 22 ++++++++++++++++++++-- modules/lfs/http_client.go | 11 ++++++++--- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/modules/lfs/client.go b/modules/lfs/client.go index d783a300c05a..ae35919d770b 100644 --- a/modules/lfs/client.go +++ b/modules/lfs/client.go @@ -18,7 +18,7 @@ type Client interface { // NewClient creates a LFS client func NewClient(endpoint *url.URL) Client { if endpoint.Scheme == "file" { - return newFilesystemClient(endpoint.Path) + return newFilesystemClient(endpoint) } - return newHTTPClient(endpoint.String()) + return newHTTPClient(endpoint) } diff --git a/modules/lfs/filesystem_client.go b/modules/lfs/filesystem_client.go index 638ac59cd57c..2b578e53c27a 100644 --- a/modules/lfs/filesystem_client.go +++ b/modules/lfs/filesystem_client.go @@ -7,8 +7,11 @@ package lfs import ( "context" "io" + "net/url" "os" "path/filepath" + "regexp" + "runtime" ) // FilesystemClient is used to read LFS data from a filesystem path @@ -16,14 +19,29 @@ type FilesystemClient struct { lfsdir string } -func newFilesystemClient(path string) *FilesystemClient { - lfsdir := filepath.Join(path, "lfs", "objects") +func newFilesystemClient(endpoint *url.URL) *FilesystemClient { + lfsdir := filepath.Join(endpointURLToPath(endpoint), "lfs", "objects") client := &FilesystemClient{lfsdir} return client } +func endpointURLToPath(endpoint *url.URL) string { + path := endpoint.Path + + if runtime.GOOS != "windows" { + return path + } + + // If it looks like there's a Windows drive letter at the beginning, strip off the leading slash. + re := regexp.MustCompile("/[A-Za-z]:/") + if re.MatchString(path) { + return path[1:] + } + return path +} + func (c *FilesystemClient) objectPath(oid string) string { return filepath.Join(c.lfsdir, oid[0:2], oid[2:4], oid) } diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index 0b71323c905c..fb45defda1cf 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "net/http" + "net/url" "strings" "code.gitea.io/gitea/modules/log" @@ -25,10 +26,14 @@ type HTTPClient struct { transfers map[string]TransferAdapter } -func newHTTPClient(endpoint string) *HTTPClient { +func newHTTPClient(endpoint *url.URL) *HTTPClient { hc := &http.Client{} - client := &HTTPClient{hc, endpoint, make(map[string]TransferAdapter)} + client := &HTTPClient{ + client: hc, + endpoint: strings.TrimSuffix(endpoint.String(), "/"), + transfers: make(map[string]TransferAdapter), + } basic := &BasicTransferAdapter{hc} @@ -50,7 +55,7 @@ func (c *HTTPClient) transferNames() []string { } func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Pointer) (*BatchResponse, error) { - url := fmt.Sprintf("%s/objects/batch", strings.TrimSuffix(c.endpoint, "/")) + url := fmt.Sprintf("%s/objects/batch", c.endpoint) request := &BatchRequest{operation, c.transferNames(), nil, objects} From 750ad791eec6935486a03be6cfa09321615bcac7 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 12 Mar 2021 18:19:12 +0000 Subject: [PATCH 52/69] Added unit test for HTTPClient. --- modules/lfs/http_client_test.go | 143 ++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 modules/lfs/http_client_test.go diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go new file mode 100644 index 000000000000..4f7d356db1ed --- /dev/null +++ b/modules/lfs/http_client_test.go @@ -0,0 +1,143 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "bytes" + "context" + "encoding/json" + "io" + "io/ioutil" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +type RoundTripFunc func(req *http.Request) *http.Response + +func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +type DummyTransferAdapter struct { +} + +func (a *DummyTransferAdapter) Name() string { + return "dummy" +} + +func (a *DummyTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) { + return ioutil.NopCloser(bytes.NewBufferString("dummy")), nil +} + +func TestHTTPClientDownload(t *testing.T) { + oid := "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041" + size := int64(6) + + roundTripHandler := func(req *http.Request) *http.Response { + url := req.URL.String() + if strings.Contains(url, "status-not-ok") { + return &http.Response{StatusCode: http.StatusBadRequest} + } + if strings.Contains(url, "invalid-json-response") { + return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("invalid json"))} + } + if strings.Contains(url, "correct-batch-request-download") { + assert.Equal(t, MediaType, req.Header.Get("Content-type"), "case %s: error should match", url) + assert.Equal(t, MediaType, req.Header.Get("Accept"), "case %s: error should match", url) + + var batchRequest BatchRequest + err := json.NewDecoder(req.Body).Decode(&batchRequest) + assert.NoError(t, err) + + assert.Equal(t, "download", batchRequest.Operation) + assert.Equal(t, 1, len(batchRequest.Objects)) + assert.Equal(t, oid, batchRequest.Objects[0].Oid) + assert.Equal(t, size, batchRequest.Objects[0].Size) + + batchResponse := &BatchResponse{ + Transfer: "dummy", + Objects: make([]*ObjectResponse, 1), + } + + payload := new(bytes.Buffer) + json.NewEncoder(payload).Encode(batchResponse) + + return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} + } + if strings.Contains(url, "invalid-response-no-objects") { + batchResponse := &BatchResponse{Transfer: "dummy"} + + payload := new(bytes.Buffer) + json.NewEncoder(payload).Encode(batchResponse) + + return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} + } + if strings.Contains(url, "unknown-transfer-adapter") { + batchResponse := &BatchResponse{Transfer: "unknown_adapter"} + + payload := new(bytes.Buffer) + json.NewEncoder(payload).Encode(batchResponse) + + return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} + } + + t.Errorf("Unknown test case: %s", url) + + return nil + } + + hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)} + dummy := &DummyTransferAdapter{} + + var cases = []struct { + endpoint string + expectederror string + }{ + // case 0 + { + endpoint: "https://status-not-ok.io", + expectederror: "Unexpected servers response: ", + }, + // case 1 + { + endpoint: "https://invalid-json-response.io", + expectederror: "json.Decode: ", + }, + // case 2 + { + endpoint: "https://correct-batch-request-download.io", + expectederror: "", + }, + // case 3 + { + endpoint: "https://invalid-response-no-objects.io", + expectederror: "No objects in result", + }, + // case 4 + { + endpoint: "https://unknown-transfer-adapter.io", + expectederror: "Transferadapter not found: ", + }, + } + + for n, c := range cases { + client := &HTTPClient{ + client: hc, + endpoint: c.endpoint, + transfers: make(map[string]TransferAdapter), + } + client.transfers["dummy"] = dummy + + _, err := client.Download(context.Background(), oid, size) + if len(c.expectederror) > 0 { + assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) + } else { + assert.NoError(t, err, "case %d", n) + } + } +} From 8fd1f7d9aa782a43f40906c24202d13389441f7e Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 12 Mar 2021 23:02:55 +0000 Subject: [PATCH 53/69] Added unit test for BasicTransferAdapter. --- modules/lfs/http_client_test.go | 5 +- modules/lfs/transferadapter_test.go | 78 +++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 modules/lfs/transferadapter_test.go diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go index 4f7d356db1ed..043aa0214e86 100644 --- a/modules/lfs/http_client_test.go +++ b/modules/lfs/http_client_test.go @@ -46,7 +46,8 @@ func TestHTTPClientDownload(t *testing.T) { if strings.Contains(url, "invalid-json-response") { return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("invalid json"))} } - if strings.Contains(url, "correct-batch-request-download") { + if strings.Contains(url, "valid-batch-request-download") { + assert.Equal(t, "POST", req.Method) assert.Equal(t, MediaType, req.Header.Get("Content-type"), "case %s: error should match", url) assert.Equal(t, MediaType, req.Header.Get("Accept"), "case %s: error should match", url) @@ -110,7 +111,7 @@ func TestHTTPClientDownload(t *testing.T) { }, // case 2 { - endpoint: "https://correct-batch-request-download.io", + endpoint: "https://valid-batch-request-download.io", expectederror: "", }, // case 3 diff --git a/modules/lfs/transferadapter_test.go b/modules/lfs/transferadapter_test.go new file mode 100644 index 000000000000..0eabd3faeee1 --- /dev/null +++ b/modules/lfs/transferadapter_test.go @@ -0,0 +1,78 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "bytes" + "context" + "io/ioutil" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBasicTransferAdapterName(t *testing.T) { + a := &BasicTransferAdapter{} + + assert.Equal(t, "basic", a.Name()) +} + +func TestBasicTransferAdapterDownload(t *testing.T) { + roundTripHandler := func(req *http.Request) *http.Response { + url := req.URL.String() + if strings.Contains(url, "valid-download-request") { + assert.Equal(t, "GET", req.Method) + assert.Equal(t, "test-value", req.Header.Get("test-header")) + + return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("dummy"))} + } + + t.Errorf("Unknown test case: %s", url) + + return nil + } + + hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)} + a := &BasicTransferAdapter{hc} + + var cases = []struct { + response *ObjectResponse + expectederror string + }{ + // case 0 + { + response: &ObjectResponse{}, + expectederror: "Action 'download' not found", + }, + // case 1 + { + response: &ObjectResponse{ + Actions: map[string]*Link{"upload": nil}, + }, + expectederror: "Action 'download' not found", + }, + // case 2 + { + response: &ObjectResponse{ + Actions: map[string]*Link{"download": { + Href: "https://valid-download-request.io", + Header: map[string]string{"test-header": "test-value"}, + }}, + }, + expectederror: "", + }, + } + + for n, c := range cases { + _, err := a.Download(context.Background(), c.response) + if len(c.expectederror) > 0 { + assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) + } else { + assert.NoError(t, err, "case %d", n) + } + } +} From 028cbd9742c4e2d398205e5c9861ec075b45e8c2 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 13 Mar 2021 11:10:37 +0000 Subject: [PATCH 54/69] Moved into util package. --- modules/lfs/filesystem_client.go | 23 +++---------- modules/util/path.go | 23 +++++++++++++ modules/util/path_test.go | 58 ++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 18 deletions(-) create mode 100644 modules/util/path_test.go diff --git a/modules/lfs/filesystem_client.go b/modules/lfs/filesystem_client.go index 2b578e53c27a..3a51564a821b 100644 --- a/modules/lfs/filesystem_client.go +++ b/modules/lfs/filesystem_client.go @@ -10,8 +10,8 @@ import ( "net/url" "os" "path/filepath" - "regexp" - "runtime" + + "code.gitea.io/gitea/modules/util" ) // FilesystemClient is used to read LFS data from a filesystem path @@ -20,28 +20,15 @@ type FilesystemClient struct { } func newFilesystemClient(endpoint *url.URL) *FilesystemClient { - lfsdir := filepath.Join(endpointURLToPath(endpoint), "lfs", "objects") + path, _ := util.FileURLToPath(endpoint) + + lfsdir := filepath.Join(path, "lfs", "objects") client := &FilesystemClient{lfsdir} return client } -func endpointURLToPath(endpoint *url.URL) string { - path := endpoint.Path - - if runtime.GOOS != "windows" { - return path - } - - // If it looks like there's a Windows drive letter at the beginning, strip off the leading slash. - re := regexp.MustCompile("/[A-Za-z]:/") - if re.MatchString(path) { - return path[1:] - } - return path -} - func (c *FilesystemClient) objectPath(oid string) string { return filepath.Join(c.lfsdir, oid[0:2], oid[2:4], oid) } diff --git a/modules/util/path.go b/modules/util/path.go index aa3d00989927..2ac8f4d80a3e 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -6,9 +6,12 @@ package util import ( "errors" + "net/url" "os" "path" "path/filepath" + "regexp" + "runtime" "strings" ) @@ -150,3 +153,23 @@ func StatDir(rootPath string, includeDir ...bool) ([]string, error) { } return statDir(rootPath, "", isIncludeDir, false, false) } + +// FileURLToPath extracts the path informations from a file://... url. +func FileURLToPath(u *url.URL) (string, error) { + if u.Scheme != "file" { + return "", errors.New("URL scheme is not 'file': " + u.String()) + } + + path := u.Path + + if runtime.GOOS != "windows" { + return path, nil + } + + // If it looks like there's a Windows drive letter at the beginning, strip off the leading slash. + re := regexp.MustCompile("/[A-Za-z]:/") + if re.MatchString(path) { + return path[1:], nil + } + return path, nil +} diff --git a/modules/util/path_test.go b/modules/util/path_test.go new file mode 100644 index 000000000000..41104f79fc83 --- /dev/null +++ b/modules/util/path_test.go @@ -0,0 +1,58 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package util + +import ( + "net/url" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFileURLToPath(t *testing.T) { + var cases = []struct { + url string + expected string + haserror bool + windows bool + }{ + // case 0 + { + url: "", + haserror: true, + }, + // case 1 + { + url: "http://test.io", + haserror: true, + }, + // case 2 + { + url: "file:///path", + expected: "/path", + }, + // case 3 + { + url: "file:///C:/path", + expected: "C:/path", + windows: true, + }, + } + + for n, c := range cases { + if c.windows && runtime.GOOS != "windows" { + continue + } + u, _ := url.Parse(c.url) + p, err := FileURLToPath(u) + if c.haserror { + assert.Error(t, err, "case %d: should return error", n) + } else { + assert.NoError(t, err, "case %d: should not return error", n) + assert.Equal(t, c.expected, p, "case %d: should be equal", n) + } + } +} From 8df4b21c08ba8861173aef72573e1a056044c066 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 13 Mar 2021 11:11:13 +0000 Subject: [PATCH 55/69] Test if LFS endpoint is allowed. --- modules/migrations/migrate.go | 6 ++++++ options/locale/locale_en-US.ini | 1 + templates/repo/migrate/options.tmpl | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 656b78a58489..a575d2175951 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -80,6 +80,12 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, if err != nil { return nil, err } + if opts.LFS && len(opts.LFSEndpoint) > 0 { + err := isMigrateURLAllowed(opts.LFSEndpoint) + if err != nil { + return nil, err + } + } downloader, err := newDownloader(ctx, ownerName, opts) if err != nil { return nil, err diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 33973981450a..0c3d8c359294 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -782,6 +782,7 @@ migrate_options_mirror_helper = This repository will be a - {{.i18n.Tr "repo.migrate_options_lfs_endpoint"}} + {{.i18n.Tr "repo.migrate_options_lfs_endpoint"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate_options_lfs_endpoint.local"}}{{end}}
From 184a94946f15537a83ef60fa524df00453f5fe5d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 17 Mar 2021 07:19:40 +0100 Subject: [PATCH 56/69] Added support for git:// --- modules/lfs/endpoint.go | 3 +++ modules/lfs/endpoint_test.go | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/modules/lfs/endpoint.go b/modules/lfs/endpoint.go index 8183b305ee98..add16ce9f12e 100644 --- a/modules/lfs/endpoint.go +++ b/modules/lfs/endpoint.go @@ -60,6 +60,9 @@ func endpointFromURL(rawurl string) *url.URL { switch u.Scheme { case "http", "https": return u + case "git": + u.Scheme = "https" + return u case "file": return u default: diff --git a/modules/lfs/endpoint_test.go b/modules/lfs/endpoint_test.go index d2dd6147ece7..a7e8b1bfb7db 100644 --- a/modules/lfs/endpoint_test.go +++ b/modules/lfs/endpoint_test.go @@ -53,6 +53,18 @@ func TestDetermineEndpoint(t *testing.T) { lfsurl: "https://gitlfs.com/repo", expected: str2url("https://gitlfs.com/repo"), }, + // case 5 + { + cloneurl: "git://git.com/repo.git", + lfsurl: "", + expected: str2url("https://git.com/repo.git/info/lfs"), + }, + // case 6 + { + cloneurl: "", + lfsurl: "git://gitlfs.com/repo", + expected: str2url("https://gitlfs.com/repo"), + }, } for n, c := range cases { From a727b460c292030cd9f01c8d05bd64fef4719a02 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 21 Mar 2021 17:19:57 +0000 Subject: [PATCH 57/69] Just use a static placeholder as the displayed url may be invalid. --- options/locale/locale_en-US.ini | 6 ++++-- templates/repo/migrate/options.tmpl | 6 +++--- web_src/js/features/migration.js | 2 -- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index b14a7f07027b..b153e5235677 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -781,8 +781,10 @@ migrate_service = Migration Service migrate_options_mirror_helper = This repository will be a mirror migrate_options_mirror_disabled = Your site administrator has disabled new mirrors. migrate_options_lfs = Migrate LFS files -migrate_options_lfs_endpoint = Migration will attempt to use your Git remote to determine the LFS server. You can also configure a custom LFS server. -migrate_options_lfs_endpoint.local = A local server path is supported too. +migrate_options_lfs_endpoint.label = LFS Endpoint +migrate_options_lfs_endpoint.description = Migration will attempt to use your Git remote to determine the LFS server. You can also configure a custom LFS server. +migrate_options_lfs_endpoint.description.local = A local server path is supported too. +migrate_options_lfs_endpoint.placeholder = Leave blank to derive from clone URL migrate_items = Migration Items migrate_items_wiki = Wiki migrate_items_milestones = Milestones diff --git a/templates/repo/migrate/options.tmpl b/templates/repo/migrate/options.tmpl index 9c03358c9bde..b9d23844300e 100644 --- a/templates/repo/migrate/options.tmpl +++ b/templates/repo/migrate/options.tmpl @@ -19,10 +19,10 @@
- {{.i18n.Tr "repo.migrate_options_lfs_endpoint"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate_options_lfs_endpoint.local"}}{{end}} + {{.i18n.Tr "repo.migrate_options_lfs_endpoint.description"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate_options_lfs_endpoint.description.local"}}{{end}}
- - + +
{{end}} \ No newline at end of file diff --git a/web_src/js/features/migration.js b/web_src/js/features/migration.js index 1341871f1795..ab9c092a2ff8 100644 --- a/web_src/js/features/migration.js +++ b/web_src/js/features/migration.js @@ -5,7 +5,6 @@ const $token = $('#auth_token'); const $mirror = $('#mirror'); const $lfs = $('#lfs'); const $lfsEndpointBlock = $('#lfs_endpoint'); -const $lfsEndpoint = $lfsEndpointBlock.find('input:text'); const $items = $('#migrate_items').find('input[type=checkbox]'); export default function initMigration() { @@ -25,7 +24,6 @@ export default function initMigration() { if ($repoName.val().length === 0) { // Only modify if repo_name input is blank $repoName.val($cloneAddr.val().match(/^(.*\/)?((.+?)(\.git)?)$/)[3]); } - $lfsEndpoint.attr('placeholder', $cloneAddr.val()); } }); } From 60a41774d8b15f57a9ba833a003bde63dce12a18 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 21 Mar 2021 17:44:31 +0000 Subject: [PATCH 58/69] Reverted to original code. --- web_src/js/features/migration.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/web_src/js/features/migration.js b/web_src/js/features/migration.js index ab9c092a2ff8..4c9f35881625 100644 --- a/web_src/js/features/migration.js +++ b/web_src/js/features/migration.js @@ -20,10 +20,8 @@ export default function initMigration() { const $cloneAddr = $('#clone_addr'); $cloneAddr.on('change', () => { const $repoName = $('#repo_name'); - if ($cloneAddr.val().length > 0) { - if ($repoName.val().length === 0) { // Only modify if repo_name input is blank - $repoName.val($cloneAddr.val().match(/^(.*\/)?((.+?)(\.git)?)$/)[3]); - } + if ($cloneAddr.val().length > 0 && $repoName.val().length === 0) { // Only modify if repo_name input is blank + $repoName.val($cloneAddr.val().match(/^(.*\/)?((.+?)(\.git)?)$/)[3]); } }); } From 2555639b6e331d45cc0103b3acd64866a1d77a8c Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 23 Mar 2021 15:46:27 +0000 Subject: [PATCH 59/69] Added "Advanced Settings". --- templates/repo/migrate/options.tmpl | 7 ++++--- web_src/js/features/migration.js | 9 +-------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/templates/repo/migrate/options.tmpl b/templates/repo/migrate/options.tmpl index b9d23844300e..b0e4e7e31ee5 100644 --- a/templates/repo/migrate/options.tmpl +++ b/templates/repo/migrate/options.tmpl @@ -14,11 +14,12 @@
- - + +
+ ({{.i18n.Tr "repo.settings.advanced_settings"}})
-
+