add: add support for git source

Signed-off-by: danishprakash <danish.prakash@suse.com>

add.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/containers/buildah/copier"
 	"github.com/containers/buildah/define"
+	"github.com/containers/buildah/internal/tmpdir"
 	"github.com/containers/buildah/pkg/chrootuser"
 	"github.com/containers/storage/pkg/fileutils"
 	"github.com/containers/storage/pkg/idtools"
@@ -74,9 +75,15 @@ type AddAndCopyOptions struct {
 	StripStickyBit bool
 }
 
+// sourceIsGit returns true if "source" is a git location.
+func sourceIsGit(source string) bool {
+	// TODO: ssh protocol
+	return strings.HasPrefix(source, "git://") || strings.Contains(source, ".git")
+}
+
 // sourceIsRemote returns true if "source" is a remote location.
 func sourceIsRemote(source string) bool {
-	return strings.HasPrefix(source, "http://") || strings.HasPrefix(source, "https://")
+	return (strings.HasPrefix(source, "http://") || strings.HasPrefix(source, "https://")) && !strings.Contains(source, ".git")
 }
 
 // getURL writes a tar archive containing the named content
@@ -233,18 +240,28 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 	}
 
 	// Figure out what sorts of sources we have.
-	var localSources, remoteSources []string
+	var localSources, remoteSources, gitSources []string
 	for i, src := range sources {
 		if sourceIsRemote(src) {
 			remoteSources = append(remoteSources, src)
 			continue
 		}
+		if sourceIsGit(src) {
+			gitSources = append(gitSources, src)
+			continue
+		}
 		if !filepath.IsAbs(src) && options.ContextDir == "" {
 			sources[i] = filepath.Join(currentDir, src)
 		}
 		localSources = append(localSources, sources[i])
 	}
 
+	// Treat git sources as a subset of remote sources
+	// differentiating only in how we fetch the two later on.
+	if len(gitSources) > 0 {
+		remoteSources = append(remoteSources, gitSources...)
+	}
+
 	// Check how many items our local source specs matched.  Each spec
 	// should have matched at least one item, otherwise we consider it an
 	// error.
@@ -276,7 +293,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 		}
 		numLocalSourceItems += len(localSourceStat.Globbed)
 	}
-	if numLocalSourceItems+len(remoteSources) == 0 {
+	if numLocalSourceItems+len(remoteSources)+len(gitSources) == 0 {
 		return fmt.Errorf("no sources %v found: %w", sources, syscall.ENOENT)
 	}
 
@@ -333,6 +350,9 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 				destCanBeFile = true
 			}
 		}
+		if len(gitSources) > 0 {
+			destMustBeDirectory = true
+		}
 	}
 
 	// We care if the destination either doesn't exist, or exists and is a
@@ -404,7 +424,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 		var multiErr *multierror.Error
 		var getErr, closeErr, renameErr, putErr error
 		var wg sync.WaitGroup
-		if sourceIsRemote(src) {
+		if sourceIsRemote(src) || sourceIsGit(src) {
 			pipeReader, pipeWriter := io.Pipe()
 			var srcDigest digest.Digest
 			if options.Checksum != "" {
@@ -413,12 +433,46 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 					return fmt.Errorf("invalid checksum flag: %w", err)
 				}
 			}
+
 			wg.Add(1)
-			go func() {
-				getErr = getURL(src, chownFiles, mountPoint, renameTarget, pipeWriter, chmodDirsFiles, srcDigest)
-				pipeWriter.Close()
-				wg.Done()
-			}()
+			if sourceIsGit(src) {
+				go func() {
+					var cloneDir string
+					cloneDir, _, getErr = define.TempDirForURL(tmpdir.GetTempDir(), "", src)
+
+					renamedItems := 0
+					writer := io.WriteCloser(pipeWriter)
+					writer = newTarFilterer(writer, func(hdr *tar.Header) (bool, bool, io.Reader) {
+						return false, false, nil
+					})
+					getOptions := copier.GetOptions{
+						UIDMap:         srcUIDMap,
+						GIDMap:         srcGIDMap,
+						Excludes:       options.Excludes,
+						ExpandArchives: extract,
+						ChownDirs:      chownDirs,
+						ChmodDirs:      chmodDirsFiles,
+						ChownFiles:     chownFiles,
+						ChmodFiles:     chmodDirsFiles,
+						StripSetuidBit: options.StripSetuidBit,
+						StripSetgidBit: options.StripSetgidBit,
+						StripStickyBit: options.StripStickyBit,
+					}
+					getErr = copier.Get(cloneDir, cloneDir, getOptions, []string{"."}, writer)
+					closeErr = writer.Close()
+					if renameTarget != "" && renamedItems > 1 {
+						renameErr = fmt.Errorf("internal error: renamed %d items when we expected to only rename 1", renamedItems)
+					}
+					wg.Done()
+				}()
+			} else {
+				go func() {
+					getErr = getURL(src, chownFiles, mountPoint, renameTarget, pipeWriter, chmodDirsFiles, srcDigest)
+					pipeWriter.Close()
+					wg.Done()
+				}()
+			}
+
 			wg.Add(1)
 			go func() {
 				b.ContentDigester.Start("")

define/types.go

@@ -12,6 +12,7 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/containers/image/v5/manifest"
@@ -89,6 +90,9 @@ var (
 	Xz           = archive.Xz
 	Zstd         = archive.Zstd
 	Uncompressed = archive.Uncompressed
+
+	validCommitSHA = regexp.MustCompile(`^[a-f0-9]{40}$`)
+	defaultBranch  = regexp.MustCompile(`refs/heads/(\S+)`)
 )
 
 // IDMappingOptions controls how we set up UID/GID mapping when we set up a
@@ -254,9 +258,35 @@ func parseGitBuildContext(url string) (string, string, string) {
 	return gitBranchPart[0], gitSubdir, gitBranch
 }
 
+func isCommitSHA(str string) bool {
+	return validCommitSHA.MatchString(str)
+}
+
+// getDefaultBranch gets the default branch of a repository using ls-remote
+func getDefaultBranch(remoteURL string) (string, error) {
+	cmd := exec.Command("git", "ls-remote", "--symref", remoteURL, "HEAD")
+	combinedOutput, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("error fetching default branch for repository %s", remoteURL)
+	}
+
+	ss := defaultBranch.FindAllStringSubmatch(string(combinedOutput), -1)
+	if len(ss) == 0 || len(ss[0]) != 2 {
+		return "", fmt.Errorf("could not find default branch for repository: %s", remoteURL)
+	}
+	return ss[0][1], nil
+}
+
+func isGitTag(remote, ref, dir string) (bool, error) {
+	if _, err := exec.Command("git", "ls-remote", "--exit-code", remote, ref).Output(); err != nil {
+		return true, nil
+	}
+	return false, nil
+}
+
 func cloneToDirectory(url, dir string) ([]byte, string, error) {
 	var cmd *exec.Cmd
-	gitRepo, gitSubdir, gitBranch := parseGitBuildContext(url)
+	gitRepo, gitSubdir, gitRef := parseGitBuildContext(url)
 	// init repo
 	cmd = exec.Command("git", "init", dir)
 	combinedOutput, err := cmd.CombinedOutput()
@@ -270,27 +300,23 @@ func cloneToDirectory(url, dir string) ([]byte, string, error) {
 	if err != nil {
 		return combinedOutput, gitSubdir, fmt.Errorf("failed while performing `git remote add`: %w", err)
 	}
-	// fetch required branch or commit and perform checkout
-	// Always default to `HEAD` if nothing specified
-	fetch := "HEAD"
-	if gitBranch != "" {
-		fetch = gitBranch
+
+	if gitRef != "" {
+		if ok, _ := isGitTag(url, gitRef, dir); ok {
+			gitRef += ":refs/tags/" + gitRef
+		}
 	}
-	logrus.Debugf("fetching repo %q and branch (or commit ID) %q to %q", gitRepo, fetch, dir)
-	cmd = exec.Command("git", "fetch", "--depth=1", "origin", "--", fetch)
+
+	logrus.Debugf("fetching repo %q and branch (or commit ID) %q to %q", gitRepo, gitRef, dir)
+	args := []string{"fetch", "-u", "--depth=1", "origin", "--", gitRef}
+	cmd = exec.Command("git", args...)
 	cmd.Dir = dir
 	combinedOutput, err = cmd.CombinedOutput()
 	if err != nil {
 		return combinedOutput, gitSubdir, fmt.Errorf("failed while performing `git fetch`: %w", err)
 	}
-	if fetch == "HEAD" {
-		// We fetched default branch therefore
-		// we don't have any valid `branch` or
-		// `commit` name hence checkout detached
-		// `FETCH_HEAD`
-		fetch = "FETCH_HEAD"
-	}
-	cmd = exec.Command("git", "checkout", fetch)
+
+	cmd = exec.Command("git", "checkout", "FETCH_HEAD")
 	cmd.Dir = dir
 	combinedOutput, err = cmd.CombinedOutput()
 	if err != nil {