This should solve #1, findMailinText should be way less memory hungry

git.go

@@ -6,34 +6,46 @@ import (
 	"gopkg.in/src-d/go-git.v4"
 	"gopkg.in/src-d/go-git.v4/plumbing/object"
 	"gopkg.in/src-d/go-git.v4/storage/memory"
+	"os"
+	"strconv"
 	"strings"
 )
 
 func gitSearch(target string, WebsiteAPI string, mailSet mapset.Set) mapset.Set {
 	domain := ""
-	rawParse := false
 	targetSplit := strings.Split(target, "/")
+	commits := ""
 
-	//TODO: work on pagination but i'm too lazy :(
 	fmt.Println("==== GIT SEARCH FOR " + target + " ==== ")
+
+	//If using GitHub API
 	if strings.Contains(target, "https://github.com") || WebsiteAPI == "github" {
 		fmt.Println("[+] Using github API")
-		domain = targetSplit[0] + "//api." + targetSplit[2] + "/repos/" + targetSplit[3] + "/" + targetSplit[4] + "/commits?per_page=5000"
+		domain = targetSplit[0] + "//api." + targetSplit[2] + "/repos/" + targetSplit[3] + "/" + targetSplit[4] + "/commits?per_page=100"
+		//GitHub Pagination
+		lastPage := retriveLastPage(domain)
+		fmt.Println("[+] Looping through pages.This MAY take a while...")
+		for i := 1; i < lastPage+1; i++ {
+			commits = retriveRequestBody(domain + "&page=" + strconv.Itoa(i))
+			findMailInText(commits, mailSet)
+		}
 	} else if strings.Contains(target, "https://bitbucket.org") || WebsiteAPI == "bitbucket" {
+		// If using BitBucket API
 		fmt.Println("[+] Using bitbucket API")
 		domain = targetSplit[0] + "//api." + targetSplit[2] + "/2.0/repositories/" + targetSplit[3] + "/" + targetSplit[4] + "/commits?per_page=5000"
+		//TODO: add BitBucket Pagination: https://developer.atlassian.com/bitbucket/api/2/reference/meta/pagination
+		commits = retriveRequestBody(domain)
+		findMailInText(commits, mailSet)
 	} else {
-		rawParse = true
-	}
-
-	commits := ""
-	if rawParse {
 		commits = cloneAndSearchCommit(target)
-	} else {
-		commits = retriveRequestBody(domain)
+		findMailInText(commits, mailSet)
 	}
 
-	mailSet = findMailInText(commits, mailSet)
+	//Check if the mailset has been populated (this avoids problems with mispelled repositories too)
+	if mailSet == nil {
+		fmt.Println("[-] Nothing Found")
+		os.Exit(1)
+	}
 	fmt.Println("[+] Mails Found")
 	readFromSet(mailSet)
 	return mailSet

pgp.go

@@ -14,7 +14,7 @@ func pgpSearch(mailSet mapset.Set) mapset.Set {
 		fmt.Println("[+] pgp search for " + mail.(string))
 		domain := "http://pgp.mit.edu/pks/lookup?search=" + mail.(string)
 		body := retriveRequestBody(domain)
-		pgpSet = findMailInText(body, pgpSet)
+		findMailInText(body, pgpSet)
 		if pgpSet != nil {
 			pgpIterator := pgpSet.Iterator()
 			for email := range pgpIterator.C {

utils.go

@@ -5,10 +5,27 @@ import (
 	"github.com/deckarep/golang-set"
 	"io/ioutil"
 	"net/http"
+	//"os"
 	"regexp"
+	"strconv"
 	"strings"
 )
 
+func retriveLastPage(domain string) int {
+	req, err := http.Get(domain)
+	if err != nil {
+		panic(err)
+	}
+	pagInfo := req.Header.Get("Link")
+	if pagInfo != "" {
+		re := regexp.MustCompile(`page=(\d+)>;\srel="last"`)
+		match := re.FindStringSubmatch(pagInfo)
+		lastPage, _ := strconv.Atoi(match[1])
+		return lastPage
+	}
+	return 1
+}
+
 func retriveRequestBody(domain string) string {
 	req, err := http.Get(domain)
 	if err != nil {
@@ -19,22 +36,20 @@ func retriveRequestBody(domain string) string {
 	return string(body)
 }
 
-func findMailInText(body string, mailSet mapset.Set) mapset.Set {
+func findMailInText(body string, mailSet mapset.Set) {
 
 	//re := regexp.MustCompile(`[\w\-\.]+\@[\w \.\-]+\.[\w]+`)
 	re := regexp.MustCompile(`(?:![\n|\s])*(?:[\w\d\.\w\d]|(?:[\w\d]+[\-]+[\w\d]+))+[\@]+[\w]+[\.]+[\w]+`)
 	mails := re.FindAllString(body, -1)
 	if len(mails) == 0 {
-		return nil
+		return
 	}
-
 	for _, mail := range mails {
 		if !strings.Contains(mail, "noreply") {
 			mailSet.Add(mail)
 		}
 	}
 
-	return (mailSet)
 }
 
 func readFromSet(mailSet mapset.Set) {