Fixed bug where shhgit would only clone the master branch ... == more…

… secrets!

config.yaml

@@ -1,5 +1,7 @@
 github_access_tokens:
-  - ''
+  - '4388b2658182341d61c1506bdbf249d49d5f2acc'
+  - 'dcefdee459ea41ffd80b0372f056ca1a7aec49a1'
+  - '26e3f7340239e28acf3a2a1b79736f6f5d81ee9a'
 webhook: '' # URL to which the payload is POSTed
 
 # This default payload will work for Slack and MatterMost.

core/git.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"gopkg.in/src-d/go-git.v4"
+	"gopkg.in/src-d/go-git.v4/plumbing"
 )
 
 type GitResourceType int
@@ -23,21 +24,28 @@ type GitResource struct {
 	Id   int64
 	Type GitResourceType
 	Url  string
+	Ref  string
 }
 
-func CloneRepository(session *Session, url string, dir string) (*git.Repository, error) {
+func CloneRepository(session *Session, url string, ref string, dir string) (*git.Repository, error) {
 	timeout := time.Duration(*session.Options.CloneRepositoryTimeout) * time.Second
 	localCtx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
-	session.Log.Debug("[%s] Cloning in to %s", url, strings.Replace(dir, *session.Options.TempDirectory, "", -1))
-	repository, err := git.PlainCloneContext(localCtx, dir, false, &git.CloneOptions{
+	session.Log.Debug("[%s] Cloning %s in to %s", url, ref, strings.Replace(dir, *session.Options.TempDirectory, "", -1))
+	opts := &git.CloneOptions{
 		Depth:             1,
 		RecurseSubmodules: git.NoRecurseSubmodules,
 		URL:               url,
 		SingleBranch:      true,
 		Tags:              git.NoTags,
-	})
+	}
+
+	if ref != "" {
+		opts.ReferenceName = plumbing.ReferenceName(ref)
+	}
+
+	repository, err := git.PlainCloneContext(localCtx, dir, false, opts)
 
 	if err != nil {
 		session.Log.Debug("[%s] Cloning failed: %s", url, err.Error())

core/github.go

@@ -80,7 +80,12 @@ func GetRepositories(session *Session) {
 
 					dst := &github.PushEvent{}
 					json.Unmarshal(e.GetRawPayload(), dst)
-					session.Repositories <- *e.Repo.ID
+					session.Repositories <- GitResource{
+						Id:   e.GetRepo().GetID(),
+						Type: GITHUB_SOURCE,
+						Url:  e.GetRepo().GetURL(),
+						Ref:  dst.GetRef(),
+					}
 				} else if *e.Type == "IssueCommentEvent" {
 					observedKeys[*e.ID] = true
 

core/session.go

@@ -22,7 +22,7 @@ type Session struct {
 	Options          *Options
 	Config           *Config
 	Signatures       []Signature
-	Repositories     chan int64
+	Repositories     chan GitResource
 	Gists            chan string
 	Comments         chan string
 	Context          context.Context
@@ -163,7 +163,7 @@ func GetSession() *Session {
 	sessionSync.Do(func() {
 		session = &Session{
 			Context:      context.Background(),
-			Repositories: make(chan int64, 1000),
+			Repositories: make(chan GitResource, 1000),
 			Gists:        make(chan string, 100),
 			Comments:     make(chan string, 1000),
 		}

main.go

@@ -3,13 +3,15 @@ package main
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"encoding/json"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/eth0izzle/shhgit/core"
 	"github.com/fatih/color"
@@ -31,21 +33,25 @@ func ProcessRepositories() {
 
 	for i := 0; i < threadNum; i++ {
 		go func(tid int) {
-
 			for {
-				repositoryId := <-session.Repositories
-				repo, err := core.GetRepository(session, repositoryId)
+				timeout := time.Duration(*session.Options.CloneRepositoryTimeout) * time.Second
+				_, cancel := context.WithTimeout(context.Background(), timeout)
+				defer cancel()
+
+				repository := <-session.Repositories
+
+				repo, err := core.GetRepository(session, repository.Id)
 
 				if err != nil {
-					session.Log.Warn("Failed to retrieve repository %d: %s", repositoryId, err)
+					session.Log.Warn("Failed to retrieve repository %d: %s", repository.Id, err)
 					continue
 				}
 
 				if repo.GetPermissions()["pull"] &&
 					uint(repo.GetStargazersCount()) >= *session.Options.MinimumStars &&
 					uint(repo.GetSize()) < *session.Options.MaximumRepositorySize {
 
-					processRepositoryOrGist(repo.GetCloneURL(), repo.GetStargazersCount(), core.GITHUB_SOURCE)
+					processRepositoryOrGist(repo.GetCloneURL(), repository.Ref, repo.GetStargazersCount(), core.GITHUB_SOURCE)
 				}
 			}
 		}(i)
@@ -59,7 +65,7 @@ func ProcessGists() {
 		go func(tid int) {
 			for {
 				gistUrl := <-session.Gists
-				processRepositoryOrGist(gistUrl, -1, core.GIST_SOURCE)
+				processRepositoryOrGist(gistUrl, "", -1, core.GIST_SOURCE)
 			}
 		}(i)
 	}
@@ -83,21 +89,21 @@ func ProcessComments() {
 	}
 }
 
-func processRepositoryOrGist(url string, stars int, source core.GitResourceType) {
+func processRepositoryOrGist(url string, ref string, stars int, source core.GitResourceType) {
 	var (
 		matchedAny bool = false
 	)
 
 	dir := core.GetTempDir(core.GetHash(url))
-	_, err := core.CloneRepository(session, url, dir)
+	_, err := core.CloneRepository(session, url, ref, dir)
 
 	if err != nil {
 		session.Log.Debug("[%s] Cloning failed: %s", url, err.Error())
 		os.RemoveAll(dir)
 		return
 	}
 
-	session.Log.Debug("[%s] Cloning in to %s", url, strings.Replace(dir, *session.Options.TempDirectory, "", -1))
+	session.Log.Debug("[%s] Cloning %s in to %s", url, ref, strings.Replace(dir, *session.Options.TempDirectory, "", -1))
 	matchedAny = checkSignatures(dir, url, stars, source)
 	if !matchedAny {
 		os.RemoveAll(dir)