Add LFS Migration and Mirror

cmd/serv.go

@@ -18,11 +18,11 @@ import (
 	"time"
 
 	"code.gitea.io/gitea/models"
-	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/pprof"
 	"code.gitea.io/gitea/modules/private"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/services/lfs"
 
 	"github.com/dgrijalva/jwt-go"
 	"github.com/kballard/go-shellquote"

integrations/lfs_getobject_test.go

@@ -18,7 +18,6 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/routers/routes"
 
 	gzipp "github.com/klauspost/compress/gzip"
@@ -50,11 +49,12 @@ func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string
 	lfsID++
 	lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject)
 	assert.NoError(t, err)
-	contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
-	exist, err := contentStore.Exists(lfsMetaObject)
+	pointer := lfsMetaObject.AsPointer()
+	contentStore := lfs.NewContentStore()
+	exist, err := contentStore.Exists(pointer)
 	assert.NoError(t, err)
 	if !exist {
-		err := contentStore.Put(lfsMetaObject, bytes.NewReader(*content))
+		err := contentStore.Put(pointer, bytes.NewReader(*content))
 		assert.NoError(t, err)
 	}
 	return oid

models/lfs.go

@@ -12,6 +12,7 @@ import (
 	"io"
 	"path"
 
+	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/timeutil"
 
 	"xorm.io/builder"
@@ -41,6 +42,12 @@ func (m *LFSMetaObject) Pointer() string {
 	return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size)
 }
 
+// AsPointer creates a Pointer with Oid and Size
+func (m *LFSMetaObject) AsPointer() *lfs.Pointer {
+	pointer := &lfs.Pointer{Oid: m.Oid, Size: m.Size}
+	return pointer
+}
+
 // LFSTokenResponse defines the JSON structure in which the JWT token is stored.
 // This structure is fetched via SSH and passed by the Git LFS client to the server
 // endpoint for authorization.

modules/forms/repo_form.go

@@ -73,6 +73,8 @@ type MigrateRepoForm struct {
 	// required: true
 	RepoName       string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
 	Mirror         bool   `json:"mirror"`
+	LFS            bool   `json:"lfs"`
+	LFSEndpoint    string `json:"lfs_endpoint"`
 	Private        bool   `json:"private"`
 	Description    string `json:"description" binding:"MaxSize(255)"`
 	Wiki           bool   `json:"wiki"`

modules/lfs/client.go

@@ -0,0 +1,111 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+)
+
+// Client is used to communicate with the LFS server
+type Client struct {
+	client    *http.Client
+	transfers map[string]TransferAdapter
+}
+
+// NewClient creates a LFS client
+func NewClient(hc *http.Client) *Client {
+	client := &Client{hc, make(map[string]TransferAdapter)}
+
+	basic := &BasicTransferAdapter{hc}
+
+	client.transfers[basic.Name()] = basic
+
+	return client
+}
+
+func (c *Client) transferNames() []string {
+	keys := make([]string, len(c.transfers))
+
+	i := 0
+	for k := range c.transfers {
+		keys[i] = k
+		i++
+	}
+
+	return keys
+}
+
+func (c *Client) batch(url, operation string, objects []*Pointer) (*BatchResponse, error) {
+	url = fmt.Sprintf("%s.git/info/lfs/objects/batch", strings.TrimSuffix(url, ".git"))
+
+	request := &BatchRequest{operation, c.transferNames(), nil, objects}
+
+	payload := new(bytes.Buffer)
+	err := json.NewEncoder(payload).Encode(request)
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequest("POST", url, payload)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-type", MediaType)
+	req.Header.Set("Accept", MediaType)
+
+	res, err := c.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("Unexpected servers response: %s", res.Status)
+	}
+
+	var response BatchResponse
+	err = json.NewDecoder(res.Body).Decode(&response)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(response.Transfer) == 0 {
+		response.Transfer = "basic"
+	}
+
+	return &response, nil
+}
+
+// Download reads the specific LFS object from the LFS server
+func (c *Client) Download(url, oid string, size int64) (io.ReadCloser, error) {
+	var objects []*Pointer
+	objects = append(objects, &Pointer{oid, size})
+
+	result, err := c.batch(url, "download", objects)
+	if err != nil {
+		return nil, err
+	}
+
+	transferAdapter, ok := c.transfers[result.Transfer]
+	if !ok {
+		return nil, fmt.Errorf("Transferadapter not found: %s", result.Transfer)
+	}
+
+	if len(result.Objects) == 0 {
+		return nil, errors.New("No objects in result")
+	}
+
+	content, err := transferAdapter.Download(result.Objects[0])
+	if err != nil {
+		return nil, err
+	}
+	return content, nil
+}

modules/lfs/content_store.go

@@ -11,91 +11,107 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 
-	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/storage"
 )
 
 var (
-	errHashMismatch = errors.New("Content hash does not match OID")
-	errSizeMismatch = errors.New("Content size does not match")
+	// ErrHashMismatch occurs if the content has does not match OID
+	ErrHashMismatch = errors.New("Content hash does not match OID")
+	// ErrSizeMismatch occurs if the content size does not match
+	ErrSizeMismatch = errors.New("Content size does not match")
 )
 
 // ErrRangeNotSatisfiable represents an error which request range is not satisfiable.
 type ErrRangeNotSatisfiable struct {
 	FromByte int64
 }
 
-func (err ErrRangeNotSatisfiable) Error() string {
-	return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte)
-}
-
 // IsErrRangeNotSatisfiable returns true if the error is an ErrRangeNotSatisfiable
 func IsErrRangeNotSatisfiable(err error) bool {
 	_, ok := err.(ErrRangeNotSatisfiable)
 	return ok
 }
 
+func (err ErrRangeNotSatisfiable) Error() string {
+	return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte)
+}
+
 // ContentStore provides a simple file system based storage.
 type ContentStore struct {
 	storage.ObjectStorage
 }
 
+// NewContentStore creates the default ContentStore
+func NewContentStore() *ContentStore {
+	contentStore := &ContentStore{ObjectStorage: storage.LFS}
+	return contentStore
+}
+
+func relativePath(p *Pointer) string {
+	if len(p.Oid) < 5 {
+		return p.Oid
+	}
+
+	return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid[4:])
+}
+
 // Get takes a Meta object and retrieves the content from the store, returning
 // it as an io.Reader. If fromByte > 0, the reader starts from that byte
-func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) {
-	f, err := s.Open(meta.RelativePath())
+func (s *ContentStore) Get(pointer *Pointer, fromByte int64) (io.ReadCloser, error) {
+	f, err := s.Open(relativePath(pointer))
 	if err != nil {
-		log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", meta.Oid, err)
+		log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", pointer.Oid, err)
 		return nil, err
 	}
 	if fromByte > 0 {
-		if fromByte >= meta.Size {
+		if fromByte >= pointer.Size {
 			return nil, ErrRangeNotSatisfiable{
 				FromByte: fromByte,
 			}
 		}
 		_, err = f.Seek(fromByte, io.SeekStart)
 		if err != nil {
-			log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", meta.Oid, fromByte, err)
+			log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", pointer.Oid, fromByte, err)
 		}
 	}
 	return f, err
 }
 
 // Put takes a Meta object and an io.Reader and writes the content to the store.
-func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
+func (s *ContentStore) Put(pointer *Pointer, r io.Reader) error {
 	hash := sha256.New()
 	rd := io.TeeReader(r, hash)
-	p := meta.RelativePath()
+	p := relativePath(pointer)
 	written, err := s.Save(p, rd)
 	if err != nil {
-		log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, p, err)
+		log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", pointer.Oid, p, err)
 		return err
 	}
 
-	if written != meta.Size {
+	if written != pointer.Size {
 		if err := s.Delete(p); err != nil {
-			log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
+			log.Error("Cleaning the LFS OID[%s] failed: %v", pointer.Oid, err)
 		}
-		return errSizeMismatch
+		return ErrSizeMismatch
 	}
 
 	shaStr := hex.EncodeToString(hash.Sum(nil))
-	if shaStr != meta.Oid {
+	if shaStr != pointer.Oid {
 		if err := s.Delete(p); err != nil {
-			log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
+			log.Error("Cleaning the LFS OID[%s] failed: %v", pointer.Oid, err)
 		}
-		return errHashMismatch
+		return ErrHashMismatch
 	}
 
 	return nil
 }
 
 // Exists returns true if the object exists in the content store.
-func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) {
-	_, err := s.ObjectStorage.Stat(meta.RelativePath())
+func (s *ContentStore) Exists(pointer *Pointer) (bool, error) {
+	_, err := s.ObjectStorage.Stat(relativePath(pointer))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return false, nil
@@ -106,15 +122,21 @@ func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) {
 }
 
 // Verify returns true if the object exists in the content store and size is correct.
-func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) {
-	p := meta.RelativePath()
+func (s *ContentStore) Verify(pointer *Pointer) (bool, error) {
+	p := relativePath(pointer)
 	fi, err := s.ObjectStorage.Stat(p)
-	if os.IsNotExist(err) || (err == nil && fi.Size() != meta.Size) {
+	if os.IsNotExist(err) || (err == nil && fi.Size() != pointer.Size) {
 		return false, nil
 	} else if err != nil {
-		log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, meta.Oid, err)
+		log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, pointer.Oid, err)
 		return false, err
 	}
 
 	return true, nil
 }
+
+// ReadMetaObject will read a models.LFSMetaObject and return a reader
+func ReadMetaObject(pointer *Pointer) (io.ReadCloser, error) {
+	contentStore := NewContentStore()
+	return contentStore.Get(pointer, 0)
+}

modules/lfs/pointer.go

@@ -0,0 +1,54 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"io"
+	"strconv"
+	"strings"
+)
+
+const (
+	blobSizeCutoff = 1024
+
+	// TODO remove duplicate from models
+
+	// LFSMetaFileIdentifier is the string appearing at the first line of LFS pointer files.
+	// https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
+	LFSMetaFileIdentifier = "version https://git-lfs.github.com/spec/v1"
+
+	// LFSMetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash.
+	LFSMetaFileOidPrefix = "oid sha256:"
+)
+
+// TryReadPointer tries to read LFS pointer data from the reader
+func TryReadPointer(reader io.Reader) *Pointer {
+	buf := make([]byte, blobSizeCutoff)
+	n, _ := reader.Read(buf)
+	buf = buf[:n]
+
+	return TryReadPointerFromBuffer(buf)
+}
+
+// TryReadPointerFromBuffer will return a pointer if the provided byte slice is a pointer file or nil otherwise.
+func TryReadPointerFromBuffer(buf []byte) *Pointer {
+	headString := string(buf)
+	if !strings.HasPrefix(headString, LFSMetaFileIdentifier) {
+		return nil
+	}
+
+	splitLines := strings.Split(headString, "\n")
+	if len(splitLines) < 3 {
+		return nil
+	}
+
+	oid := strings.TrimPrefix(splitLines[1], LFSMetaFileOidPrefix)
+	size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64)
+	if len(oid) != 64 || err != nil {
+		return nil
+	}
+
+	return &Pointer{Oid: oid, Size: size}
+}