Remove git2go

since libgit2 does not expose cloning with --depth, I can't use it for larger repos. see libgit2/libgit2#3058 I would love to switch to git2go once this is solved.
nicolai86 · Aug 12, 2016 · d91b20f · d91b20f
1 parent f9ebefe
commit d91b20f
Show file tree

Hide file tree

Showing 7 changed files with 56 additions and 225 deletions.
diff --git a/cmd/dependency-js-worker/main.go b/cmd/dependency-js-worker/main.go
@@ -14,9 +14,9 @@ import (
 	"github.com/docker/engine-api/types"
 	"github.com/docker/engine-api/types/container"
 	"github.com/google/go-github/github"
-	"github.com/libgit2/git2go"
 	"github.com/nats-io/nats"
 	"github.com/nicolai86/sisyphus/github/pr"
+	"github.com/nicolai86/sisyphus/github/repo"
 	"github.com/nicolai86/sisyphus/storage"
 	"golang.org/x/net/context"
 )
@@ -84,86 +84,22 @@ type repoConfig struct {
 
 var filesToExtract = []string{"package.json"}
 
-func repoFrom(r storage.Repository) *git.Repository {
-	cloneOptions := &git.CloneOptions{
-		Bare:           false,
-		CheckoutBranch: "master",
-	}
-	cachePath := fmt.Sprintf("/tmp/%s", r.ID)
-	if _, err := os.Stat(cachePath); err != nil {
-		repo, err := git.Clone(r.GitURL, cachePath, cloneOptions)
-		if err != nil {
-			log.Panic(err)
-		}
-		return repo
-	}
-
-	repo, err := git.OpenRepository(cachePath)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	remote, err := repo.Remotes.Lookup("origin")
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if err := remote.Fetch([]string{}, nil, ""); err != nil {
-		log.Fatal(err)
-	}
-
-	return repo
-}
-
-func fileContent(g *git.Repository, path string) ([]byte, error) {
-	head, err := g.References.Lookup("refs/remotes/origin/master")
-	if err != nil {
-		return nil, err
-	}
-
-	commit, err := g.LookupCommit(head.Target())
-	if err != nil {
-		return nil, err
-	}
-
-	tree, err := commit.Tree()
-	if err != nil {
-		return nil, err
-	}
-
-	t, err := tree.EntryByPath(path)
-	if err != nil {
-		return nil, err
-	}
-
-	if t.Filemode != git.FilemodeBlob {
-		return nil, fmt.Errorf("Not a blob")
-	}
-	blob, err := g.LookupBlob(t.Id)
-	if err != nil {
-		return nil, err
-	}
-
-	return blob.Contents(), nil
-}
-
 func checkDependencies(r storage.Repository, c config) {
 	log.Printf("looking for %q (%q): %q", c.Path, c.Language, filesToExtract)
 
-	repo := repoFrom(r)
+	owner := strings.Split(r.FullName, "/")[0]
+	repoName := strings.Split(r.FullName, "/")[1]
+	tmpDir, _ := repo.Clone(r.AccessToken, owner, repoName)
+
 	data := []byte(fmt.Sprintf("%s-%s", c.Path, c.Language))
 	cachePath := fmt.Sprintf("/tmp/build/%s/%x", r.ID, md5.Sum(data))
 	os.MkdirAll(cachePath, 0700)
-	for _, file := range filesToExtract {
-		content, err := fileContent(repo, fmt.Sprintf("%s/%s", c.Path, file))
-		if err != nil {
-			panic(err)
-		}
-
-		log.Printf("build %q\n", cachePath)
-		f, err := os.OpenFile(fmt.Sprintf("%s/%s", cachePath, file), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600)
-		f.Write(content)
 
+	for _, file := range filesToExtract {
+		os.Rename(
+			fmt.Sprintf("%s/%s/%s", tmpDir, c.Path, file),
+			fmt.Sprintf("%s/%s", cachePath, file),
+		)
 	}
 
 	runDependencyCheck(r, c, cachePath)

diff --git a/cmd/dependency-rb-worker/main.go b/cmd/dependency-rb-worker/main.go
@@ -15,9 +15,9 @@ import (
 	"github.com/docker/engine-api/types/container"
 	"github.com/docker/engine-api/types/strslice"
 	"github.com/google/go-github/github"
-	"github.com/libgit2/git2go"
 	"github.com/nats-io/nats"
 	"github.com/nicolai86/sisyphus/github/pr"
+	"github.com/nicolai86/sisyphus/github/repo"
 	"github.com/nicolai86/sisyphus/storage"
 	"golang.org/x/net/context"
 )
@@ -47,88 +47,23 @@ func init() {
 	fileStorage = storage.NewFileStorage(dataPath)
 }
 
-func repoFrom(r storage.Repository) *git.Repository {
-	cloneOptions := &git.CloneOptions{
-		Bare:           false,
-		CheckoutBranch: "master",
-	}
-	cachePath := fmt.Sprintf("/tmp/%s", r.ID)
-	if _, err := os.Stat(cachePath); err != nil {
-		repo, err := git.Clone(r.GitURL, cachePath, cloneOptions)
-		if err != nil {
-			log.Panic(err)
-		}
-		return repo
-	}
-
-	repo, err := git.OpenRepository(cachePath)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	remote, err := repo.Remotes.Lookup("origin")
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if err := remote.Fetch([]string{}, nil, ""); err != nil {
-		log.Fatal(err)
-	}
-
-	return repo
-}
-
-func fileContent(g *git.Repository, path string) ([]byte, error) {
-	head, err := g.References.Lookup("refs/remotes/origin/master")
-	if err != nil {
-		return nil, err
-	}
-
-	commit, err := g.LookupCommit(head.Target())
-	if err != nil {
-		return nil, err
-	}
-
-	tree, err := commit.Tree()
-	if err != nil {
-		return nil, err
-	}
-
-	t, err := tree.EntryByPath(path)
-	if err != nil {
-		return nil, err
-	}
-
-	if t.Filemode != git.FilemodeBlob {
-		return nil, fmt.Errorf("Not a blob")
-	}
-	blob, err := g.LookupBlob(t.Id)
-	if err != nil {
-		return nil, err
-	}
-
-	return blob.Contents(), nil
-}
-
 var filesToExtract = []string{"Gemfile", "Gemfile.lock"}
 
 func checkDependencies(r storage.Repository, c config) {
 	log.Printf("looking for %q (%q): %q", c.Path, c.Language, filesToExtract)
 
-	repo := repoFrom(r)
+	owner := strings.Split(r.FullName, "/")[0]
+	repoName := strings.Split(r.FullName, "/")[1]
+	tmpDir, _ := repo.Clone(r.AccessToken, owner, repoName)
+
 	data := []byte(fmt.Sprintf("%s-%s", c.Path, c.Language))
 	cachePath := fmt.Sprintf("/tmp/build/%s/%x", r.ID, md5.Sum(data))
 	os.MkdirAll(cachePath, 0700)
 	for _, file := range filesToExtract {
-		content, err := fileContent(repo, fmt.Sprintf("%s/%s", c.Path, file))
-		if err != nil {
-			panic(err)
-		}
-
-		log.Printf("build %q\n", cachePath)
-		f, err := os.OpenFile(fmt.Sprintf("%s/%s", cachePath, file), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600)
-		f.Write(content)
-
+		os.Rename(
+			fmt.Sprintf("%s/%s/%s", tmpDir, c.Path, file),
+			fmt.Sprintf("%s/%s", cachePath, file),
+		)
 	}
 
 	runDependencyCheck(r, c, cachePath)

diff --git a/cmd/repos-worker/main.go b/cmd/repos-worker/main.go
@@ -8,8 +8,8 @@ import (
 	"os"
 	"strings"
 
-	git "github.com/libgit2/git2go"
 	"github.com/nats-io/nats"
+	"github.com/nicolai86/sisyphus/github/repo"
 	"github.com/nicolai86/sisyphus/storage"
 )
 
@@ -27,88 +27,21 @@ func init() {
 	fileStorage = storage.NewFileStorage(dataPath)
 }
 
-func repoFrom(r storage.Repository) *git.Repository {
-	cloneOptions := &git.CloneOptions{
-		Bare:           false,
-		CheckoutBranch: "master",
-	}
-	cachePath := fmt.Sprintf("/tmp/%s", r.ID)
-	if _, err := os.Stat(cachePath); err != nil {
-		repo, err := git.Clone(r.GitURL, cachePath, cloneOptions)
-		if err != nil {
-			log.Panic(err)
-		}
-		return repo
-	}
-
-	repo, err := git.OpenRepository(cachePath)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	remote, err := repo.Remotes.Lookup("origin")
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if err := remote.Fetch([]string{}, nil, ""); err != nil {
-		log.Fatal(err)
-	}
-
-	return repo
-}
-
-func fileContent(g *git.Repository, path string) ([]byte, error) {
-	head, err := g.References.Lookup("refs/remotes/origin/master")
-	if err != nil {
-		return nil, err
-	}
-
-	commit, err := g.LookupCommit(head.Target())
-	if err != nil {
-		return nil, err
-	}
-
-	tree, err := commit.Tree()
-	if err != nil {
-		return nil, err
-	}
-
-	t, err := tree.EntryByPath(path)
-	if err != nil {
-		return nil, err
-	}
-
-	if t.Filemode != git.FilemodeBlob {
-		return nil, fmt.Errorf("Not a blob")
-	}
-	blob, err := g.LookupBlob(t.Id)
-	if err != nil {
-		return nil, err
-	}
-
-	return blob.Contents(), nil
-}
-
 func cachePluginSettings(r storage.Repository, plugin string) {
 	log.Printf("caching %q for %s\n", plugin, r.ID)
 
-	var repo = repoFrom(r)
-	defer repo.Free()
+	owner := strings.Split(r.FullName, "/")[0]
+	repoName := strings.Split(r.FullName, "/")[1]
+	tmpDir, _ := repo.Clone(r.AccessToken, owner, repoName)
 
 	pluginCacheDir := fmt.Sprintf("%s/%s", dataPath, plugin)
 	os.MkdirAll(pluginCacheDir, 0700)
 	pluginCache := fmt.Sprintf("%s/%s.json", pluginCacheDir, r.ID)
 
-	bs, err := fileContent(repo, ".sisyphus")
-	if err != nil {
-		log.Printf("Error reading .sisyphus file: %q\n", err)
-		// TODO remove previous cached file
-		return
-	}
+	f, _ := os.Open(fmt.Sprintf("%s/.sisyphus", tmpDir))
 
 	var configs map[string]interface{}
-	json.NewDecoder(strings.NewReader(string(bs))).Decode(&configs)
+	json.NewDecoder(strings.NewReader(string(f))).Decode(&configs)
 
 	f, _ := os.OpenFile(pluginCache, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
 	json.NewEncoder(f).Encode(configs[plugin])

diff --git a/github/repo/repo.go b/github/repo/repo.go
@@ -0,0 +1,31 @@
+package repo
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+)
+
+func Clone(accessToken, owner, repo string) (string, error) {
+	dir, err := ioutil.TempDir("", fmt.Sprintf("gh-%s-%s", owner, repo))
+	if err != nil {
+		return "", err
+	}
+
+	cmds := [][]string{
+		[]string{"git", "init"},
+		[]string{"git", "pull", fmt.Sprintf("https://%s@github.com/%s/%s.git", accessToken, owner, repo), "--depth", "1"},
+	}
+
+	for _, args := range cmds {
+		cmd := exec.Command(args[0], args[1:]...)
+		cmd.Dir = dir
+		cmd.Env = os.Environ()
+		if err := cmd.Run(); err != nil {
+			return "", err
+		}
+	}
+
+	return dir, nil
+}
diff --git a/glide.lock b/glide.lock
diff --git a/glide.yaml b/glide.yaml
@@ -1,7 +1,6 @@
 package: github.com/nicolai86/sisyphus
 import:
 - package: golang.org/x/oauth2
-- package: github.com/libgit2/git2go
 - package: github.com/google/go-github
   subpackages:
   - github

diff --git a/vendor/github.com/libgit2/git2go b/vendor/github.com/libgit2/git2go