Skip to content

Commit

Permalink
fix(github): scan user repos (#2814)
Browse files Browse the repository at this point in the history
  • Loading branch information
@rgmz
rgmz authored May 23, 2024
1 parent f7214cf commit 1441289
Show file tree
Hide file tree
Showing 2 changed files with 107 additions and 47 deletions.
74 changes: 41 additions & 33 deletions pkg/sources/github/github.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,12 @@ import (
"time"

"golang.org/x/exp/rand"
"golang.org/x/oauth2"

"github.com/bradleyfalzon/ghinstallation/v2"
"github.com/go-logr/logr"
"github.com/gobwas/glob"
"github.com/google/go-github/v62/github"
"golang.org/x/oauth2"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
Expand Down Expand Up @@ -480,8 +480,17 @@ func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, bas
s.apiClient = ghClient

for _, org := range s.orgsCache.Keys() {
if err := s.getReposByOrg(ctx, org); err != nil {
s.log.Error(err, "error fetching repos for org or user")
orgCtx := context.WithValue(ctx, "account", org)
userType, err := s.getReposByOrgOrUser(ctx, org)
if err != nil {
orgCtx.Logger().Error(err, "error fetching repos for org or user")
continue
}

if userType == organization && s.conn.ScanUsers {
if err := s.addMembersByOrg(ctx, org); err != nil {
orgCtx.Logger().Error(err, "Unable to add members by org")
}
}
}

Expand All @@ -499,17 +508,15 @@ func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint strin
}

for _, org := range s.orgsCache.Keys() {
if err := s.getReposByOrg(ctx, org); err != nil {
s.log.Error(err, "error fetching repos for org")
}

// We probably don't need to do this, since getting repos by org makes more sense?
if err := s.getReposByUser(ctx, org); err != nil {
s.log.Error(err, "error fetching repos for user")
orgCtx := context.WithValue(ctx, "account", org)
userType, err := s.getReposByOrgOrUser(ctx, org)
if err != nil {
orgCtx.Logger().Error(err, "error fetching repos for org or user")
continue
}

if s.conn.ScanUsers {
s.log.Info("Enumerating unauthenticated does not support scanning organization members")
if userType == organization && s.conn.ScanUsers {
orgCtx.Logger().Info("WARNING: Enumerating unauthenticated does not support scanning organization members (--include-members)")
}
}
}
Expand Down Expand Up @@ -562,16 +569,16 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
if s.orgsCache.Count() > 0 {
specificScope = true
for _, org := range s.orgsCache.Keys() {
logger := s.log.WithValues("org", org)
if err := s.getReposByOrg(ctx, org); err != nil {
logger.Error(err, "error fetching repos for org")
orgCtx := context.WithValue(ctx, "account", org)
userType, err := s.getReposByOrgOrUser(ctx, org)
if err != nil {
orgCtx.Logger().Error(err, "error fetching repos for org or user")
continue
}

if s.conn.ScanUsers {
err := s.addMembersByOrg(ctx, org)
if err != nil {
logger.Error(err, "Unable to add members by org")
continue
if userType == organization && s.conn.ScanUsers {
if err := s.addMembersByOrg(ctx, org); err != nil {
orgCtx.Logger().Error(err, "Unable to add members by org")
}
}
}
Expand All @@ -593,27 +600,28 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
}

for _, org := range s.orgsCache.Keys() {
logger := s.log.WithValues("org", org)
if err := s.getReposByOrg(ctx, org); err != nil {
logger.Error(err, "error fetching repos by org")
}

if err := s.getReposByUser(ctx, ghUser.GetLogin()); err != nil {
logger.Error(err, "error fetching repos by user")
orgCtx := context.WithValue(ctx, "account", org)
userType, err := s.getReposByOrgOrUser(ctx, org)
if err != nil {
orgCtx.Logger().Error(err, "error fetching repos for org or user")
continue
}

if s.conn.ScanUsers {
err := s.addMembersByOrg(ctx, org)
if err != nil {
logger.Error(err, "Unable to add members by org for org")
if userType == organization && s.conn.ScanUsers {
if err := s.addMembersByOrg(ctx, org); err != nil {
orgCtx.Logger().Error(err, "Unable to add members by org for org")
}
}
}

if err := s.getReposByUser(ctx, ghUser.GetLogin()); err != nil {
s.log.Error(err, "error fetching repos for the current user", "user", ghUser.GetLogin())
}

// If we enabled ScanUsers above, we've already added the gists for the current user and users from the orgs.
// So if we don't have ScanUsers enabled, add the user gists as normal.
if err := s.addUserGistsToCache(ctx, ghUser.GetLogin()); err != nil {
s.log.Error(err, "error fetching gists", "user", ghUser.GetLogin())
s.log.Error(err, "error fetching gists for the current user", "user", ghUser.GetLogin())
}

return nil
Expand Down Expand Up @@ -693,7 +701,7 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *
s.log.Info("Scanning repos", "org_members", len(s.memberCache))
for member := range s.memberCache {
logger := s.log.WithValues("member", member)
if err := s.getReposByUser(ctx, member); err != nil {
if err := s.addUserGistsToCache(ctx, member); err != nil {
logger.Error(err, "error fetching gists by user")
}
if err := s.getReposByUser(ctx, member); err != nil {
Expand Down
80 changes: 66 additions & 14 deletions pkg/sources/github/repo.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package github

import (
"errors"
"fmt"
"io"
"net/http"
Expand Down Expand Up @@ -169,14 +170,6 @@ func (a *appListOptions) getListOptions() *github.ListOptions {
return &a.ListOptions
}

func (s *Source) getReposByApp(ctx context.Context) error {
return s.processRepos(ctx, "", s.appListReposWrapper, &appListOptions{
ListOptions: github.ListOptions{
PerPage: defaultPagination,
},
})
}

func (s *Source) appListReposWrapper(ctx context.Context, _ string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
someRepos, res, err := s.apiClient.Apps.ListRepos(ctx, opts.getListOptions())
if someRepos != nil {
Expand All @@ -185,6 +178,14 @@ func (s *Source) appListReposWrapper(ctx context.Context, _ string, opts repoLis
return nil, res, err
}

func (s *Source) getReposByApp(ctx context.Context) error {
return s.processRepos(ctx, "", s.appListReposWrapper, &appListOptions{
ListOptions: github.ListOptions{
PerPage: defaultPagination,
},
})
}

type userListOptions struct {
github.RepositoryListByUserOptions
}
Expand All @@ -193,6 +194,10 @@ func (u *userListOptions) getListOptions() *github.ListOptions {
return &u.ListOptions
}

func (s *Source) userListReposWrapper(ctx context.Context, user string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
return s.apiClient.Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions)
}

func (s *Source) getReposByUser(ctx context.Context, user string) error {
return s.processRepos(ctx, user, s.userListReposWrapper, &userListOptions{
RepositoryListByUserOptions: github.RepositoryListByUserOptions{
Expand All @@ -203,10 +208,6 @@ func (s *Source) getReposByUser(ctx context.Context, user string) error {
})
}

func (s *Source) userListReposWrapper(ctx context.Context, user string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
return s.apiClient.Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions)
}

type orgListOptions struct {
github.RepositoryListByOrgOptions
}
Expand All @@ -215,6 +216,10 @@ func (o *orgListOptions) getListOptions() *github.ListOptions {
return &o.ListOptions
}

func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
return s.apiClient.Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions)
}

func (s *Source) getReposByOrg(ctx context.Context, org string) error {
return s.processRepos(ctx, org, s.orgListReposWrapper, &orgListOptions{
RepositoryListByOrgOptions: github.RepositoryListByOrgOptions{
Expand All @@ -225,8 +230,55 @@ func (s *Source) getReposByOrg(ctx context.Context, org string) error {
})
}

func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
return s.apiClient.Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions)
// userType indicates whether an account belongs to a person or organization.
//
// See:
// - https://docs.github.com/en/get-started/learning-about-github/types-of-github-accounts
// - https://docs.github.com/en/rest/users/users?apiVersion=2022-11-28#get-a-user
type userType int

const (
// Default invalid state.
unknown userType = iota
// The account is a person (https://docs.github.com/en/rest/users/users).
user
// The account is an organization (https://docs.github.com/en/rest/orgs/orgs).
organization
)

func (s *Source) getReposByOrgOrUser(ctx context.Context, name string) (userType, error) {
var err error

// List repositories for the organization |name|.
err = s.getReposByOrg(ctx, name)
if err == nil {
return organization, nil
} else if !isGitHub404Error(err) {
return unknown, err
}

// List repositories for the user |name|.
err = s.getReposByUser(ctx, name)
if err == nil {
if err := s.addUserGistsToCache(ctx, name); err != nil {
ctx.Logger().Error(err, "Unable to add user to cache")
}
return user, nil
} else if !isGitHub404Error(err) {
return unknown, err
}

return unknown, fmt.Errorf("account '%s' not found", name)
}

// isGitHub404Error returns true if |err| is a `github.ErrorResponse` and has the status code `404`.
func isGitHub404Error(err error) bool {
var ghErr *github.ErrorResponse
if !errors.As(err, &ghErr) {
return false
}

return ghErr.Response.StatusCode == http.StatusNotFound
}

func (s *Source) processRepos(ctx context.Context, target string, listRepos repoLister, listOpts repoListOptions) error {
Expand Down

0 comments on commit 1441289

Please sign in to comment.