Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

push optimizations (take 2) #1128

Merged
merged 7 commits into from
Apr 8, 2016
Merged

push optimizations (take 2) #1128

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
19c0e6c
teach git package how to get the local refs
technoweenie Apr 6, 2016
31dc0c2
add ResolveRefs() placeholder for future optimization
technoweenie Apr 6, 2016
abd9b1e
introduce an uploadContext struct to share in push and pre-push
technoweenie Apr 6, 2016
0935817
no need for defer since there's only 1 return after cmd.Start()
technoweenie Apr 7, 2016
231686e
refactor *uploadContext, separate lib responsibilities vs cli respons…
technoweenie Apr 7, 2016
160ad66
these assertions should be more specific
technoweenie Apr 7, 2016
be83f64
rename AddUpload() to SetUploaded() for clarity
technoweenie Apr 8, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 4 additions & 6 deletions commands/command_fetch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,11 @@ func fetchCommand(cmd *cobra.Command, args []string) {
}

if len(args) > 1 {
for _, r := range args[1:] {
ref, err := git.ResolveRef(r)
if err != nil {
Panic(err, "Invalid ref argument")
}
refs = append(refs, ref)
resolvedrefs, err := git.ResolveRefs(args[1:])
if err != nil {
Panic(err, "Invalid ref argument: %v", args[1:])
}
refs = resolvedrefs
} else {
ref, err := git.CurrentRef()
if err != nil {
Expand Down
118 changes: 11 additions & 107 deletions commands/command_pre_push.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,8 @@ var (
Use: "pre-push",
Run: prePushCommand,
}
prePushDryRun = false
prePushDeleteBranch = strings.Repeat("0", 40)
prePushMissingErrMsg = "%s is an LFS pointer to %s, which does not exist in .git/lfs/objects.\n\nRun 'git lfs fsck' to verify Git LFS objects."
prePushDryRun = false
prePushDeleteBranch = strings.Repeat("0", 40)
)

// prePushCommand is run through Git's pre-push hook. The pre-push hook passes
Expand All @@ -43,7 +42,6 @@ var (
// In the case of deleting a branch, no attempts to push Git LFS objects will be
// made.
func prePushCommand(cmd *cobra.Command, args []string) {

if len(args) == 0 {
Print("This should be run through Git's pre-push hook. Run `git lfs update` to install it.")
os.Exit(1)
Expand All @@ -53,7 +51,13 @@ func prePushCommand(cmd *cobra.Command, args []string) {
if err := git.ValidateRemote(args[0]); err != nil {
Exit("Invalid remote name %q", args[0])
}

lfs.Config.CurrentRemote = args[0]
ctx := newUploadContext(prePushDryRun)

scanOpt := lfs.NewScanRefsOptions()
scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
scanOpt.RemoteName = lfs.Config.CurrentRemote

// We can be passed multiple lines of refs
scanner := bufio.NewScanner(os.Stdin)
Expand All @@ -69,113 +73,13 @@ func prePushCommand(cmd *cobra.Command, args []string) {
continue
}

prePushRef(left, right)

}
}

func prePushRef(left, right string) {
// Just use scanner here
scanOpt := lfs.NewScanRefsOptions()
scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
scanOpt.RemoteName = lfs.Config.CurrentRemote

pointers, err := lfs.ScanRefs(left, right, scanOpt)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}

totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}

// Objects to skip because they're missing locally but on server
var skipObjects lfs.StringSet

if !prePushDryRun {
// Do this as a pre-flight check since upload queue starts immediately
skipObjects = prePushCheckForMissingObjects(pointers)
}

uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, prePushDryRun)

for _, pointer := range pointers {
if prePushDryRun {
Print("push %s => %s", pointer.Oid, pointer.Name)
continue
}

if skipObjects.Contains(pointer.Oid) {
// object missing locally but on server, don't bother
continue
}

u, err := lfs.NewUploadable(pointer.Oid, pointer.Name)
pointers, err := lfs.ScanRefs(left, right, scanOpt)
if err != nil {
if lfs.IsCleanPointerError(err) {
Exit(prePushMissingErrMsg, pointer.Name, lfs.ErrorGetContext(err, "pointer").(*lfs.Pointer).Oid)
} else {
ExitWithError(err)
}
Panic(err, "Error scanning for Git LFS files")
}

uploadQueue.Add(u)
upload(ctx, pointers)
}

if !prePushDryRun {
uploadQueue.Wait()
for _, err := range uploadQueue.Errors() {
if Debugging || lfs.IsFatalError(err) {
LoggedError(err, err.Error())
} else {
if inner := lfs.GetInnerError(err); inner != nil {
Error(inner.Error())
}
Error(err.Error())
}
}

if len(uploadQueue.Errors()) > 0 {
os.Exit(2)
}
}

}

func prePushCheckForMissingObjects(pointers []*lfs.WrappedPointer) (objectsOnServer lfs.StringSet) {
var missingLocalObjects []*lfs.WrappedPointer
var missingSize int64
var skipObjects = lfs.NewStringSetWithCapacity(len(pointers))
for _, pointer := range pointers {
if !lfs.ObjectExistsOfSize(pointer.Oid, pointer.Size) {
// We think we need to push this but we don't have it
// Store for server checking later
missingLocalObjects = append(missingLocalObjects, pointer)
missingSize += pointer.Size
}
}
if len(missingLocalObjects) == 0 {
return nil
}

checkQueue := lfs.NewDownloadCheckQueue(len(missingLocalObjects), missingSize, true)
for _, p := range missingLocalObjects {
checkQueue.Add(lfs.NewDownloadCheckable(p))
}
// this channel is filled with oids for which Check() succeeded & Transfer() was called
transferc := checkQueue.Watch()
done := make(chan int)
go func() {
for oid := range transferc {
skipObjects.Add(oid)
}
done <- 1
}()
// Currently this is needed to flush the batch but is not enough to sync transferc completely
checkQueue.Wait()
<-done
return skipObjects
}

// decodeRefs pulls the sha1s out of the line read from the pre-push
Expand Down
148 changes: 49 additions & 99 deletions commands/command_push.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,115 +23,83 @@ var (
// shares some global vars and functions with command_pre_push.go
)

func uploadsBetweenRefs(left string, right string) *lfs.TransferQueue {
func uploadsBetweenRefs(ctx *uploadContext, left string, right string) {
tracerx.Printf("Upload between %v and %v", left, right)

// Just use scanner here
pointers, err := lfs.ScanRefs(left, right, nil)
scanOpt := lfs.NewScanRefsOptions()
scanOpt.ScanMode = lfs.ScanRefsMode
scanOpt.RemoteName = lfs.Config.CurrentRemote

pointers, err := lfs.ScanRefs(left, right, scanOpt)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
return uploadPointers(pointers)

upload(ctx, pointers)
}

func uploadsBetweenRefAndRemote(remote string, refs []string) *lfs.TransferQueue {
tracerx.Printf("Upload refs %v to remote %v", refs, remote)
func uploadsBetweenRefAndRemote(ctx *uploadContext, refnames []string) {
tracerx.Printf("Upload refs %v to remote %v", refnames, lfs.Config.CurrentRemote)

scanOpt := lfs.NewScanRefsOptions()
scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
scanOpt.RemoteName = remote
scanOpt.RemoteName = lfs.Config.CurrentRemote

if pushAll {
if len(refs) == 0 {
pointers := scanAll()
Print("Pushing objects...")
return uploadPointers(pointers)
} else {
scanOpt.ScanMode = lfs.ScanRefsMode
}
scanOpt.ScanMode = lfs.ScanRefsMode
}

// keep a unique set of pointers
oidPointerMap := make(map[string]*lfs.WrappedPointer)
refs, err := refsByNames(refnames)
if err != nil {
Error(err.Error())
Exit("Error getting local refs.")
}

for _, ref := range refs {
pointers, err := lfs.ScanRefs(ref, "", scanOpt)
pointers, err := lfs.ScanRefs(ref.Name, "", scanOpt)
if err != nil {
Panic(err, "Error scanning for Git LFS files in the %q ref", ref)
}

for _, p := range pointers {
oidPointerMap[p.Oid] = p
Panic(err, "Error scanning for Git LFS files in the %q ref", ref.Name)
}
}

i := 0
pointers := make([]*lfs.WrappedPointer, len(oidPointerMap))
for _, pointer := range oidPointerMap {
pointers[i] = pointer
i += 1
upload(ctx, pointers)
}

return uploadPointers(pointers)
}

func uploadPointers(pointers []*lfs.WrappedPointer) *lfs.TransferQueue {
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}

skipObjects := prePushCheckForMissingObjects(pointers)

uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, pushDryRun)
for i, pointer := range pointers {
if pushDryRun {
Print("push %s => %s", pointer.Oid, pointer.Name)
continue
}

if _, skip := skipObjects[pointer.Oid]; skip {
// object missing locally but on server, don't bother
continue
}

tracerx.Printf("prepare upload: %s %s %d/%d", pointer.Oid, pointer.Name, i+1, len(pointers))
func uploadsWithObjectIDs(ctx *uploadContext, oids []string) {
pointers := make([]*lfs.WrappedPointer, len(oids))

u, err := lfs.NewUploadable(pointer.Oid, pointer.Name)
if err != nil {
ExitWithError(err)
}
uploadQueue.Add(u)
for idx, oid := range oids {
pointers[idx] = &lfs.WrappedPointer{Pointer: &lfs.Pointer{Oid: oid}}
}

return uploadQueue
upload(ctx, pointers)
}

func uploadsWithObjectIDs(oids []string) *lfs.TransferQueue {
uploads := []*lfs.Uploadable{}
totalSize := int64(0)

for i, oid := range oids {
if pushDryRun {
Print("push object ID %s", oid)
continue
}
tracerx.Printf("prepare upload: %s %d/%d", oid, i+1, len(oids))
func refsByNames(refnames []string) ([]*git.Ref, error) {
localrefs, err := git.LocalRefs()
if err != nil {
return nil, err
}

u, err := lfs.NewUploadable(oid, "")
if err != nil {
ExitWithError(err)
}
uploads = append(uploads, u)
if pushAll && len(refnames) == 0 {
return localrefs, nil
}

uploadQueue := lfs.NewUploadQueue(len(oids), totalSize, pushDryRun)
reflookup := make(map[string]*git.Ref, len(localrefs))
for _, ref := range localrefs {
reflookup[ref.Name] = ref
}

for _, u := range uploads {
uploadQueue.Add(u)
refs := make([]*git.Ref, len(refnames))
for i, name := range refnames {
if ref, ok := reflookup[name]; ok {
refs[i] = ref
} else {
refs[i] = &git.Ref{name, git.RefTypeOther, name}
}
}

return uploadQueue
return refs, nil
}

// pushCommand pushes local objects to a Git LFS server. It takes two
Expand All @@ -144,8 +112,6 @@ func uploadsWithObjectIDs(oids []string) *lfs.TransferQueue {
// pushCommand calculates the git objects to send by looking comparing the range
// of commits between the local and remote git servers.
func pushCommand(cmd *cobra.Command, args []string) {
var uploadQueue *lfs.TransferQueue

if len(args) == 0 {
Print("Specify a remote and a remote branch name (`git lfs push origin master`)")
os.Exit(1)
Expand All @@ -155,7 +121,9 @@ func pushCommand(cmd *cobra.Command, args []string) {
if err := git.ValidateRemote(args[0]); err != nil {
Exit("Invalid remote name %q", args[0])
}

lfs.Config.CurrentRemote = args[0]
ctx := newUploadContext(pushDryRun)

if useStdin {
requireStdin("Run this command from the Git pre-push hook, or leave the --stdin flag off.")
Expand All @@ -178,39 +146,21 @@ func pushCommand(cmd *cobra.Command, args []string) {
return
}

uploadQueue = uploadsBetweenRefs(left, right)
uploadsBetweenRefs(ctx, left, right)
} else if pushObjectIDs {
if len(args) < 2 {
Print("Usage: git lfs push --object-id <remote> <lfs-object-id> [lfs-object-id] ...")
return
}

uploadQueue = uploadsWithObjectIDs(args[1:])
uploadsWithObjectIDs(ctx, args[1:])
} else {
if len(args) < 1 {
Print("Usage: git lfs push --dry-run <remote> [ref]")
return
}

uploadQueue = uploadsBetweenRefAndRemote(args[0], args[1:])
}

if !pushDryRun {
uploadQueue.Wait()
for _, err := range uploadQueue.Errors() {
if Debugging || lfs.IsFatalError(err) {
LoggedError(err, err.Error())
} else {
if inner := lfs.GetInnerError(err); inner != nil {
Error(inner.Error())
}
Error(err.Error())
}
}

if len(uploadQueue.Errors()) > 0 {
os.Exit(2)
}
uploadsBetweenRefAndRemote(ctx, args[1:])
}
}

Expand Down