Allow using a glob for include list.

main.go

@@ -64,6 +64,8 @@ var (
 	githubScanToken      = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
 	githubIncludeForks   = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
 	githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
+	githubIncludeRepos   = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
+	githubExcludeRepos   = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
 
 	gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
 	// TODO: Add more GitLab options
@@ -220,15 +222,24 @@ func run(state overseer.State) {
 		if len(*githubScanOrgs) == 0 && len(*githubScanRepos) == 0 {
 			logrus.Fatal("You must specify at least one organization or repository.")
 		}
+		var repos []string
+		// If an org is provided, the repos would be the IncludeRepos.
+		// If an org is not provided, the repos would be the repos passed in via the --repo flag.
+		if len(*githubScanOrgs) == 0 {
+			repos = *githubScanRepos
+		} else {
+			repos = *githubIncludeRepos
+		}
 
 		github := func(c *sources.Config) {
 			c.Endpoint = *githubScanEndpoint
-			c.Repos = *githubScanRepos
+			c.Repos = repos
 			c.Orgs = *githubScanOrgs
 			c.Token = *githubScanToken
 			c.IncludeForks = *githubIncludeForks
 			c.IncludeMembers = *githubIncludeMembers
 			c.Concurrency = *concurrency
+			c.ExcludeRepos = *githubExcludeRepos
 		}
 
 		if err = e.ScanGitHub(ctx, sources.NewConfig(github)); err != nil {

pkg/engine/github.go

@@ -15,11 +15,13 @@ import (
 // ScanGitHub scans Github with the provided options.
 func (e *Engine) ScanGitHub(ctx context.Context, c sources.Config) error {
 	source := github.Source{}
+
 	connection := sourcespb.GitHub{
 		Endpoint:      c.Endpoint,
 		Organizations: c.Orgs,
 		Repositories:  c.Repos,
 		ScanUsers:     c.IncludeMembers,
+		IgnoreRepos:   c.ExcludeRepos,
 	}
 	if len(c.Token) > 0 {
 		connection.Credential = &sourcespb.GitHub_Token{

pkg/sources/github/github.go

@@ -52,6 +52,7 @@ type Source struct {
 	repos,
 	orgs,
 	members,
+	includeRepos,
 	ignoreRepos []string
 	git             *git.Git
 	httpClient      *http.Client
@@ -147,6 +148,12 @@ func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64,
 
 	s.repos = s.conn.Repositories
 	s.orgs = s.conn.Organizations
+	// If an org is specified along with repos, all the repos should be
+	// added to the includeRepos list. This way they are treated independently of the repo scans.
+	if len(s.orgs) > 0 {
+		s.includeRepos = append(s.includeRepos, s.repos...)
+		s.repos = nil
+	}
 	s.ignoreRepos = s.conn.IgnoreRepos
 
 	// Head or base should only be used with incoming webhooks
@@ -671,6 +678,9 @@ func (s *Source) getReposByOrg(ctx context.Context, org string) ([]string, error
 			if s.ignoreRepo(r.GetFullName()) {
 				continue
 			}
+			if !s.includeRepo(r.GetFullName()) {
+				continue
+			}
 
 			numRepos++
 			if r.GetFork() {
@@ -730,6 +740,9 @@ func (s *Source) getReposByUser(ctx context.Context, user string) ([]string, err
 			if s.ignoreRepo(r.GetFullName()) {
 				continue
 			}
+			if !s.includeRepo(r.GetFullName()) {
+				continue
+			}
 
 			if r.GetFork() && !s.conn.IncludeForks {
 				continue
@@ -744,6 +757,25 @@ func (s *Source) getReposByUser(ctx context.Context, user string) ([]string, err
 	return repos, nil
 }
 
+func (s *Source) includeRepo(r string) bool {
+	if len(s.includeRepos) == 0 {
+		return true
+	}
+
+	for _, include := range s.includeRepos {
+		g, err := glob.Compile(include)
+		if err != nil {
+			s.log.WithError(err).Errorf("invalid glob %s", include)
+			continue
+		}
+		if g.Match(r) {
+			s.log.Debugf("including repo %s", r)
+			return true
+		}
+	}
+	return false
+}
+
 func (s *Source) ignoreRepo(r string) bool {
 	for _, ignore := range s.ignoreRepos {
 		g, err := glob.Compile(ignore)

pkg/sources/github/github_test.go

@@ -80,6 +80,31 @@ func TestAddReposByOrg(t *testing.T) {
 	assert.True(t, gock.IsDone())
 }
 
+func TestAddReposByOrg_IncludeRepos(t *testing.T) {
+	defer gock.Off()
+
+	gock.New("https://api.github.com").
+		Get("/orgs/super-secret-org/repos").
+		Reply(200).
+		JSON([]map[string]string{
+			{"clone_url": "super-secret-repo", "full_name": "secret/super-secret-repo"},
+			{"clone_url": "super-secret-repo2", "full_name": "secret/super-secret-repo2"},
+			{"clone_url": "super-secret-repo2", "full_name": "secret/not-super-secret-repo"},
+		})
+
+	src := &sourcespb.GitHub{
+		Organizations: []string{"super-secret-org"},
+	}
+	s := initTestSource(src)
+	s.includeRepos = []string{"secret/super*"}
+	// gock works here because github.NewClient is using the default HTTP Transport
+	err := s.addRepos(context.TODO(), "super-secret-org", s.getReposByOrg)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(s.repos))
+	assert.Equal(t, []string{"super-secret-repo", "super-secret-repo2"}, s.repos)
+	assert.True(t, gock.IsDone())
+}
+
 func TestAddReposByUser(t *testing.T) {
 	defer gock.Off()
 

pkg/sources/sources.go

@@ -89,6 +89,8 @@ type Config struct {
 	Orgs,
 	// Buckets is the list of buckets to scan.
 	Buckets,
+	// ExcludeRepos is a list of repositories to exclude from the scan.
+	ExcludeRepos,
 	// Directories is the list of directories to scan.
 	Directories []string
 	// Filter is the filter to use to scan the source.