Skip to content

Commit

Permalink
Merge pull request #1128 from github/skip-pre-push-dupes-2
Browse files Browse the repository at this point in the history 
push optimizations (take 2)
  • Loading branch information
@technoweenie
technoweenie committed Apr 8, 2016
2 parents 276892b + be83f64 commit 3598328
Show file tree
Hide file tree
Showing 9 changed files with 301 additions and 225 deletions.
10 changes: 4 additions & 6 deletions commands/command_fetch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,11 @@ func fetchCommand(cmd *cobra.Command, args []string) {
}

if len(args) > 1 {
for _, r := range args[1:] {
ref, err := git.ResolveRef(r)
if err != nil {
Panic(err, "Invalid ref argument")
}
refs = append(refs, ref)
resolvedrefs, err := git.ResolveRefs(args[1:])
if err != nil {
Panic(err, "Invalid ref argument: %v", args[1:])
}
refs = resolvedrefs
} else {
ref, err := git.CurrentRef()
if err != nil {
Expand Down
118 changes: 11 additions & 107 deletions commands/command_pre_push.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,8 @@ var (
Use: "pre-push",
Run: prePushCommand,
}
prePushDryRun = false
prePushDeleteBranch = strings.Repeat("0", 40)
prePushMissingErrMsg = "%s is an LFS pointer to %s, which does not exist in .git/lfs/objects.\n\nRun 'git lfs fsck' to verify Git LFS objects."
prePushDryRun = false
prePushDeleteBranch = strings.Repeat("0", 40)
)

// prePushCommand is run through Git's pre-push hook. The pre-push hook passes
Expand All @@ -43,7 +42,6 @@ var (
// In the case of deleting a branch, no attempts to push Git LFS objects will be
// made.
func prePushCommand(cmd *cobra.Command, args []string) {

if len(args) == 0 {
Print("This should be run through Git's pre-push hook. Run `git lfs update` to install it.")
os.Exit(1)
Expand All @@ -53,7 +51,13 @@ func prePushCommand(cmd *cobra.Command, args []string) {
if err := git.ValidateRemote(args[0]); err != nil {
Exit("Invalid remote name %q", args[0])
}

lfs.Config.CurrentRemote = args[0]
ctx := newUploadContext(prePushDryRun)

scanOpt := lfs.NewScanRefsOptions()
scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
scanOpt.RemoteName = lfs.Config.CurrentRemote

// We can be passed multiple lines of refs
scanner := bufio.NewScanner(os.Stdin)
Expand All @@ -69,113 +73,13 @@ func prePushCommand(cmd *cobra.Command, args []string) {
continue
}

prePushRef(left, right)

}
}

func prePushRef(left, right string) {
// Just use scanner here
scanOpt := lfs.NewScanRefsOptions()
scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
scanOpt.RemoteName = lfs.Config.CurrentRemote

pointers, err := lfs.ScanRefs(left, right, scanOpt)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}

totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}

// Objects to skip because they're missing locally but on server
var skipObjects lfs.StringSet

if !prePushDryRun {
// Do this as a pre-flight check since upload queue starts immediately
skipObjects = prePushCheckForMissingObjects(pointers)
}

uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, prePushDryRun)

for _, pointer := range pointers {
if prePushDryRun {
Print("push %s => %s", pointer.Oid, pointer.Name)
continue
}

if skipObjects.Contains(pointer.Oid) {
// object missing locally but on server, don't bother
continue
}

u, err := lfs.NewUploadable(pointer.Oid, pointer.Name)
pointers, err := lfs.ScanRefs(left, right, scanOpt)
if err != nil {
if lfs.IsCleanPointerError(err) {
Exit(prePushMissingErrMsg, pointer.Name, lfs.ErrorGetContext(err, "pointer").(*lfs.Pointer).Oid)
} else {
ExitWithError(err)
}
Panic(err, "Error scanning for Git LFS files")
}

uploadQueue.Add(u)
upload(ctx, pointers)
}

if !prePushDryRun {
uploadQueue.Wait()
for _, err := range uploadQueue.Errors() {
if Debugging || lfs.IsFatalError(err) {
LoggedError(err, err.Error())
} else {
if inner := lfs.GetInnerError(err); inner != nil {
Error(inner.Error())
}
Error(err.Error())
}
}

if len(uploadQueue.Errors()) > 0 {
os.Exit(2)
}
}

}

func prePushCheckForMissingObjects(pointers []*lfs.WrappedPointer) (objectsOnServer lfs.StringSet) {
var missingLocalObjects []*lfs.WrappedPointer
var missingSize int64
var skipObjects = lfs.NewStringSetWithCapacity(len(pointers))
for _, pointer := range pointers {
if !lfs.ObjectExistsOfSize(pointer.Oid, pointer.Size) {
// We think we need to push this but we don't have it
// Store for server checking later
missingLocalObjects = append(missingLocalObjects, pointer)
missingSize += pointer.Size
}
}
if len(missingLocalObjects) == 0 {
return nil
}

checkQueue := lfs.NewDownloadCheckQueue(len(missingLocalObjects), missingSize, true)
for _, p := range missingLocalObjects {
checkQueue.Add(lfs.NewDownloadCheckable(p))
}
// this channel is filled with oids for which Check() succeeded & Transfer() was called
transferc := checkQueue.Watch()
done := make(chan int)
go func() {
for oid := range transferc {
skipObjects.Add(oid)
}
done <- 1
}()
// Currently this is needed to flush the batch but is not enough to sync transferc completely
checkQueue.Wait()
<-done
return skipObjects
}

// decodeRefs pulls the sha1s out of the line read from the pre-push
Expand Down
148 changes: 49 additions & 99 deletions commands/command_push.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,115 +23,83 @@ var (
// shares some global vars and functions with command_pre_push.go
)

func uploadsBetweenRefs(left string, right string) *lfs.TransferQueue {
func uploadsBetweenRefs(ctx *uploadContext, left string, right string) {
tracerx.Printf("Upload between %v and %v", left, right)

// Just use scanner here
pointers, err := lfs.ScanRefs(left, right, nil)
scanOpt := lfs.NewScanRefsOptions()
scanOpt.ScanMode = lfs.ScanRefsMode
scanOpt.RemoteName = lfs.Config.CurrentRemote

pointers, err := lfs.ScanRefs(left, right, scanOpt)
if err != nil {
Panic(err, "Error scanning for Git LFS files")
}
return uploadPointers(pointers)

upload(ctx, pointers)
}

func uploadsBetweenRefAndRemote(remote string, refs []string) *lfs.TransferQueue {
tracerx.Printf("Upload refs %v to remote %v", refs, remote)
func uploadsBetweenRefAndRemote(ctx *uploadContext, refnames []string) {
tracerx.Printf("Upload refs %v to remote %v", refnames, lfs.Config.CurrentRemote)

scanOpt := lfs.NewScanRefsOptions()
scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
scanOpt.RemoteName = remote
scanOpt.RemoteName = lfs.Config.CurrentRemote

if pushAll {
if len(refs) == 0 {
pointers := scanAll()
Print("Pushing objects...")
return uploadPointers(pointers)
} else {
scanOpt.ScanMode = lfs.ScanRefsMode
}
scanOpt.ScanMode = lfs.ScanRefsMode
}

// keep a unique set of pointers
oidPointerMap := make(map[string]*lfs.WrappedPointer)
refs, err := refsByNames(refnames)
if err != nil {
Error(err.Error())
Exit("Error getting local refs.")
}

for _, ref := range refs {
pointers, err := lfs.ScanRefs(ref, "", scanOpt)
pointers, err := lfs.ScanRefs(ref.Name, "", scanOpt)
if err != nil {
Panic(err, "Error scanning for Git LFS files in the %q ref", ref)
}

for _, p := range pointers {
oidPointerMap[p.Oid] = p
Panic(err, "Error scanning for Git LFS files in the %q ref", ref.Name)
}
}

i := 0
pointers := make([]*lfs.WrappedPointer, len(oidPointerMap))
for _, pointer := range oidPointerMap {
pointers[i] = pointer
i += 1
upload(ctx, pointers)
}

return uploadPointers(pointers)
}

func uploadPointers(pointers []*lfs.WrappedPointer) *lfs.TransferQueue {
totalSize := int64(0)
for _, p := range pointers {
totalSize += p.Size
}

skipObjects := prePushCheckForMissingObjects(pointers)

uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, pushDryRun)
for i, pointer := range pointers {
if pushDryRun {
Print("push %s => %s", pointer.Oid, pointer.Name)
continue
}

if _, skip := skipObjects[pointer.Oid]; skip {
// object missing locally but on server, don't bother
continue
}

tracerx.Printf("prepare upload: %s %s %d/%d", pointer.Oid, pointer.Name, i+1, len(pointers))
func uploadsWithObjectIDs(ctx *uploadContext, oids []string) {
pointers := make([]*lfs.WrappedPointer, len(oids))

u, err := lfs.NewUploadable(pointer.Oid, pointer.Name)
if err != nil {
ExitWithError(err)
}
uploadQueue.Add(u)
for idx, oid := range oids {
pointers[idx] = &lfs.WrappedPointer{Pointer: &lfs.Pointer{Oid: oid}}
}

return uploadQueue
upload(ctx, pointers)
}

func uploadsWithObjectIDs(oids []string) *lfs.TransferQueue {
uploads := []*lfs.Uploadable{}
totalSize := int64(0)

for i, oid := range oids {
if pushDryRun {
Print("push object ID %s", oid)
continue
}
tracerx.Printf("prepare upload: %s %d/%d", oid, i+1, len(oids))
func refsByNames(refnames []string) ([]*git.Ref, error) {
localrefs, err := git.LocalRefs()
if err != nil {
return nil, err
}

u, err := lfs.NewUploadable(oid, "")
if err != nil {
ExitWithError(err)
}
uploads = append(uploads, u)
if pushAll && len(refnames) == 0 {
return localrefs, nil
}

uploadQueue := lfs.NewUploadQueue(len(oids), totalSize, pushDryRun)
reflookup := make(map[string]*git.Ref, len(localrefs))
for _, ref := range localrefs {
reflookup[ref.Name] = ref
}

for _, u := range uploads {
uploadQueue.Add(u)
refs := make([]*git.Ref, len(refnames))
for i, name := range refnames {
if ref, ok := reflookup[name]; ok {
refs[i] = ref
} else {
refs[i] = &git.Ref{name, git.RefTypeOther, name}
}
}

return uploadQueue
return refs, nil
}

// pushCommand pushes local objects to a Git LFS server. It takes two
Expand All @@ -144,8 +112,6 @@ func uploadsWithObjectIDs(oids []string) *lfs.TransferQueue {
// pushCommand calculates the git objects to send by looking comparing the range
// of commits between the local and remote git servers.
func pushCommand(cmd *cobra.Command, args []string) {
var uploadQueue *lfs.TransferQueue

if len(args) == 0 {
Print("Specify a remote and a remote branch name (`git lfs push origin master`)")
os.Exit(1)
Expand All @@ -155,7 +121,9 @@ func pushCommand(cmd *cobra.Command, args []string) {
if err := git.ValidateRemote(args[0]); err != nil {
Exit("Invalid remote name %q", args[0])
}

lfs.Config.CurrentRemote = args[0]
ctx := newUploadContext(pushDryRun)

if useStdin {
requireStdin("Run this command from the Git pre-push hook, or leave the --stdin flag off.")
Expand All @@ -178,39 +146,21 @@ func pushCommand(cmd *cobra.Command, args []string) {
return
}

uploadQueue = uploadsBetweenRefs(left, right)
uploadsBetweenRefs(ctx, left, right)
} else if pushObjectIDs {
if len(args) < 2 {
Print("Usage: git lfs push --object-id <remote> <lfs-object-id> [lfs-object-id] ...")
return
}

uploadQueue = uploadsWithObjectIDs(args[1:])
uploadsWithObjectIDs(ctx, args[1:])
} else {
if len(args) < 1 {
Print("Usage: git lfs push --dry-run <remote> [ref]")
return
}

uploadQueue = uploadsBetweenRefAndRemote(args[0], args[1:])
}

if !pushDryRun {
uploadQueue.Wait()
for _, err := range uploadQueue.Errors() {
if Debugging || lfs.IsFatalError(err) {
LoggedError(err, err.Error())
} else {
if inner := lfs.GetInnerError(err); inner != nil {
Error(inner.Error())
}
Error(err.Error())
}
}

if len(uploadQueue.Errors()) > 0 {
os.Exit(2)
}
uploadsBetweenRefAndRemote(ctx, args[1:])
}
}

Expand Down

0 comments on commit 3598328

Please sign in to comment.