extract common pre-push and push uploading code

commands/command_fetch.go

@@ -43,13 +43,11 @@ func fetchCommand(cmd *cobra.Command, args []string) {
 	}
 
 	if len(args) > 1 {
-		for _, r := range args[1:] {
-			ref, err := git.ResolveRef(r)
-			if err != nil {
-				Panic(err, "Invalid ref argument")
-			}
-			refs = append(refs, ref)
+		resolvedrefs, err := git.ResolveRefs(args[1:])
+		if err != nil {
+			Panic(err, "Invalid ref argument")
 		}
+		refs = resolvedrefs
 	} else {
 		ref, err := git.CurrentRef()
 		if err != nil {
@@ -77,7 +75,7 @@ func fetchCommand(cmd *cobra.Command, args []string) {
 		// Fetch refs sequentially per arg order; duplicates in later refs will be ignored
 		for _, ref := range refs {
 			Print("Fetching %v", ref.Name)
-			s := fetchRef(ref.Sha, includePaths, excludePaths)
+			s := fetchRef(ref, includePaths, excludePaths)
 			success = success && s
 		}
 
@@ -115,35 +113,38 @@ func pointersToFetchForRef(ref string) ([]*lfs.WrappedPointer, error) {
 	return lfs.ScanRefs(ref, "", opts)
 }
 
-func fetchRefToChan(ref string, include, exclude []string) chan *lfs.WrappedPointer {
+func fetchRefToChan(ref *git.Ref, include, exclude []string) chan *lfs.WrappedPointer {
 	c := make(chan *lfs.WrappedPointer)
-	pointers, err := pointersToFetchForRef(ref)
+	pointers, err := pointersToFetchForRef(ref.Sha)
 	if err != nil {
 		Panic(err, "Could not scan for Git LFS files")
 	}
 
-	go fetchAndReportToChan(pointers, include, exclude, c)
+	metadata := lfs.NewTransferMetadata(ref.NameOnRemote(lfs.Config.CurrentRemote))
+	go fetchAndReportToChan(pointers, include, exclude, metadata, c)
 
 	return c
 }
 
 // Fetch all binaries for a given ref (that we don't have already)
-func fetchRef(ref string, include, exclude []string) bool {
-	pointers, err := pointersToFetchForRef(ref)
+func fetchRef(ref *git.Ref, include, exclude []string) bool {
+	pointers, err := pointersToFetchForRef(ref.Sha)
 	if err != nil {
 		Panic(err, "Could not scan for Git LFS files")
 	}
-	return fetchPointers(pointers, include, exclude)
+	metadata := lfs.NewTransferMetadata(ref.NameOnRemote(lfs.Config.CurrentRemote))
+	return fetchPointers(pointers, include, exclude, metadata)
 }
 
 // Fetch all previous versions of objects from since to ref (not including final state at ref)
 // So this will fetch all the '-' sides of the diff from since to ref
-func fetchPreviousVersions(ref string, since time.Time, include, exclude []string) bool {
-	pointers, err := lfs.ScanPreviousVersions(ref, since)
+func fetchPreviousVersions(ref *git.Ref, since time.Time, include, exclude []string) bool {
+	pointers, err := lfs.ScanPreviousVersions(ref.Sha, since)
 	if err != nil {
 		Panic(err, "Could not scan for Git LFS previous versions")
 	}
-	return fetchPointers(pointers, include, exclude)
+	metadata := lfs.NewTransferMetadata(ref.NameOnRemote(lfs.Config.CurrentRemote))
+	return fetchPointers(pointers, include, exclude, metadata)
 }
 
 // Fetch recent objects based on config
@@ -177,7 +178,7 @@ func fetchRecent(alreadyFetchedRefs []*git.Ref, include, exclude []string) bool
 			} else {
 				uniqueRefShas[ref.Sha] = ref.Name
 				Print("Fetching %v", ref.Name)
-				k := fetchRef(ref.Sha, include, exclude)
+				k := fetchRef(ref, include, exclude)
 				ok = ok && k
 			}
 		}
@@ -193,7 +194,8 @@ func fetchRecent(alreadyFetchedRefs []*git.Ref, include, exclude []string) bool
 			}
 			Print("Fetching changes within %v days of %v", fetchconf.FetchRecentCommitsDays, refName)
 			commitsSince := summ.CommitDate.AddDate(0, 0, -fetchconf.FetchRecentCommitsDays)
-			k := fetchPreviousVersions(commit, commitsSince, include, exclude)
+			commitRef := &git.Ref{Name: refName, Sha: commit}
+			k := fetchPreviousVersions(commitRef, commitsSince, include, exclude)
 			ok = ok && k
 		}
 
@@ -204,7 +206,7 @@ func fetchRecent(alreadyFetchedRefs []*git.Ref, include, exclude []string) bool
 func fetchAll() bool {
 	pointers := scanAll()
 	Print("Fetching objects...")
-	return fetchPointers(pointers, nil, nil)
+	return fetchPointers(pointers, nil, nil, nil)
 }
 
 func scanAll() []*lfs.WrappedPointer {
@@ -235,18 +237,19 @@ func scanAll() []*lfs.WrappedPointer {
 	return pointers
 }
 
-func fetchPointers(pointers []*lfs.WrappedPointer, include, exclude []string) bool {
-	return fetchAndReportToChan(pointers, include, exclude, nil)
+func fetchPointers(pointers []*lfs.WrappedPointer, include, exclude []string, metadata *lfs.TransferMetadata) bool {
+	return fetchAndReportToChan(pointers, include, exclude, metadata, nil)
 }
 
 // Fetch and report completion of each OID to a channel (optional, pass nil to skip)
 // Returns true if all completed with no errors, false if errors were written to stderr/log
-func fetchAndReportToChan(pointers []*lfs.WrappedPointer, include, exclude []string, out chan<- *lfs.WrappedPointer) bool {
+func fetchAndReportToChan(pointers []*lfs.WrappedPointer, include, exclude []string, metadata *lfs.TransferMetadata, out chan<- *lfs.WrappedPointer) bool {
+
 	totalSize := int64(0)
 	for _, p := range pointers {
 		totalSize += p.Size
 	}
-	q := lfs.NewDownloadQueue(len(pointers), totalSize, false)
+	q := lfs.NewDownloadQueue(len(pointers), totalSize, false, metadata)
 
 	if out != nil {
 		dlwatch := q.Watch()

commands/command_pre_push.go

@@ -43,7 +43,6 @@ var (
 // In the case of deleting a branch, no attempts to push Git LFS objects will be
 // made.
 func prePushCommand(cmd *cobra.Command, args []string) {
-
 	if len(args) == 0 {
 		Print("This should be run through Git's pre-push hook.  Run `git lfs update` to install it.")
 		os.Exit(1)
@@ -55,7 +54,13 @@ func prePushCommand(cmd *cobra.Command, args []string) {
 	}
 	lfs.Config.CurrentRemote = args[0]
 
-	uploadedOids := lfs.NewStringSet()
+	cli := newClient()
+	cli.RemoteName = lfs.Config.CurrentRemote
+	cli.DryRun = prePushDryRun
+
+	scanOpt := lfs.NewScanRefsOptions()
+	scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
+	scanOpt.RemoteName = cli.RemoteName
 
 	// We can be passed multiple lines of refs
 	scanner := bufio.NewScanner(os.Stdin)
@@ -66,141 +71,41 @@ func prePushCommand(cmd *cobra.Command, args []string) {
 			continue
 		}
 
-		left, right := decodeRefs(line)
+		left, right, destRef := decodeRefs(line)
 		if left == prePushDeleteBranch {
 			continue
 		}
 
-		prePushRef(left, right, uploadedOids)
-	}
-}
-
-func prePushRef(left, right string, uploadedOids lfs.StringSet) {
-	// Just use scanner here
-	scanOpt := lfs.NewScanRefsOptions()
-	scanOpt.ScanMode = lfs.ScanLeftToRemoteMode
-	scanOpt.RemoteName = lfs.Config.CurrentRemote
-
-	pointers, err := lfs.ScanRefs(left, right, scanOpt)
-	if err != nil {
-		Panic(err, "Error scanning for Git LFS files")
-	}
+		pointers, err := lfs.ScanRefs(left, right, scanOpt)
 
-	pointers = filteredPointers(pointers, uploadedOids)
-	pointers = filteredPointers(pointers, prePushCheckForMissingObjects(pointers))
-
-	totalSize := int64(0)
-	for _, p := range pointers {
-		totalSize += p.Size
-	}
-
-	if totalSize < 1 {
-		return
-	}
-
-	uploadQueue := lfs.NewUploadQueue(len(pointers), totalSize, prePushDryRun)
-
-	for _, pointer := range pointers {
-		if prePushDryRun {
-			Print("push %s => %s", pointer.Oid, pointer.Name)
-			continue
-		}
-
-		u, err := lfs.NewUploadable(pointer.Oid, pointer.Name)
 		if err != nil {
-			if lfs.IsCleanPointerError(err) {
-				Exit(prePushMissingErrMsg, pointer.Name, lfs.ErrorGetContext(err, "pointer").(*lfs.Pointer).Oid)
-			} else {
-				ExitWithError(err)
-			}
-		}
-
-		uploadedOids.Add(pointer.Oid)
-		uploadQueue.Add(u)
-	}
-
-	if !prePushDryRun {
-		uploadQueue.Wait()
-		for _, err := range uploadQueue.Errors() {
-			if Debugging || lfs.IsFatalError(err) {
-				LoggedError(err, err.Error())
-			} else {
-				if inner := lfs.GetInnerError(err); inner != nil {
-					Error(inner.Error())
-				}
-				Error(err.Error())
-			}
+			Panic(err, "Error scanning for Git LFS files")
 		}
 
-		if len(uploadQueue.Errors()) > 0 {
-			os.Exit(2)
-		}
+		metadata := lfs.NewTransferMetadata(destRef)
+		cli.Upload(metadata, pointers)
 	}
 }
 
-func filteredPointers(pointers []*lfs.WrappedPointer, filter lfs.StringSet) []*lfs.WrappedPointer {
-	filtered := make([]*lfs.WrappedPointer, 0, len(pointers))
-	for _, pointer := range pointers {
-		if filter.Contains(pointer.Oid) {
-			continue
-		}
-
-		filtered = append(filtered, pointer)
-	}
-
-	return filtered
-}
-
-func prePushCheckForMissingObjects(pointers []*lfs.WrappedPointer) (objectsOnServer lfs.StringSet) {
-	var missingLocalObjects []*lfs.WrappedPointer
-	var missingSize int64
-	var skipObjects = lfs.NewStringSetWithCapacity(len(pointers))
-	for _, pointer := range pointers {
-		if !lfs.ObjectExistsOfSize(pointer.Oid, pointer.Size) {
-			// We think we need to push this but we don't have it
-			// Store for server checking later
-			missingLocalObjects = append(missingLocalObjects, pointer)
-			missingSize += pointer.Size
-		}
-	}
-	if len(missingLocalObjects) == 0 {
-		return nil
-	}
-
-	checkQueue := lfs.NewDownloadCheckQueue(len(missingLocalObjects), missingSize, true)
-	for _, p := range missingLocalObjects {
-		checkQueue.Add(lfs.NewDownloadCheckable(p))
-	}
-	// this channel is filled with oids for which Check() succeeded & Transfer() was called
-	transferc := checkQueue.Watch()
-	done := make(chan int)
-	go func() {
-		for oid := range transferc {
-			skipObjects.Add(oid)
-		}
-		done <- 1
-	}()
-	// Currently this is needed to flush the batch but is not enough to sync transferc completely
-	checkQueue.Wait()
-	<-done
-	return skipObjects
-}
-
-// decodeRefs pulls the sha1s out of the line read from the pre-push
+// decodeRefs pulls the sha1s and destination ref out of the line read from the pre-push
 // hook's stdin.
-func decodeRefs(input string) (string, string) {
+func decodeRefs(input string) (string, string, string) {
 	refs := strings.Split(strings.TrimSpace(input), " ")
-	var left, right string
+	var left, right, destRef string
 
 	if len(refs) > 1 {
 		left = refs[1]
 	}
 
+	if len(refs) > 2 {
+		destRef = refs[2]
+	}
+
 	if len(refs) > 3 {
 		right = "^" + refs[3]
 	}
 
-	return left, right
+	return left, right, destRef
 }
 
 func init() {

commands/command_pull.go

@@ -40,15 +40,13 @@ func pullCommand(cmd *cobra.Command, args []string) {
 }
 
 func pull(includePaths, excludePaths []string) {
-
 	ref, err := git.CurrentRef()
 	if err != nil {
 		Panic(err, "Could not pull")
 	}
 
-	c := fetchRefToChan(ref.Sha, includePaths, excludePaths)
+	c := fetchRefToChan(ref, includePaths, excludePaths)
 	checkoutFromFetchChan(includePaths, excludePaths, c)
-
 }
 
 func init() {