Skip to content

Commit

Permalink
feat: use repo search to select repositories (#397)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jamestelfer
jamestelfer committed Oct 27, 2023
1 parent d454e5b commit 0f8c2dc
Show file tree
Hide file tree
Showing 3 changed files with 226 additions and 15 deletions.
17 changes: 10 additions & 7 deletions cmd/platform.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ func configurePlatform(cmd *cobra.Command) {
flags.StringSliceP("group", "G", nil, "The name of a GitLab organization. All repositories in that group will be used.")
flags.StringSliceP("user", "U", nil, "The name of a user. All repositories owned by that user will be used.")
flags.StringSliceP("repo", "R", nil, "The name, including owner of a GitHub repository in the format \"ownerName/repoName\".")
flags.StringP("repo-search", "", "", "Use a repository search to find repositories to target (GitHub only). Forks are NOT included by default, use `fork:true` to include them. See the GitHub documentation for full syntax: https://docs.github.com/en/search-github/searching-on-github/searching-for-repositories")
flags.StringSliceP("topic", "", nil, "The topic of a GitHub/GitLab/Gitea repository. All repositories having at least one matching topic are targeted.")
flags.StringSliceP("project", "P", nil, "The name, including owner of a GitLab project in the format \"ownerName/repoName\".")
flags.BoolP("include-subgroups", "", false, "Include GitLab subgroups when using the --group flag.")
Expand Down Expand Up @@ -122,14 +123,15 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu
orgs, _ := flag.GetStringSlice("org")
users, _ := flag.GetStringSlice("user")
repos, _ := flag.GetStringSlice("repo")
repoSearch, _ := flag.GetString("repo-search")
topics, _ := flag.GetStringSlice("topic")
forkMode, _ := flag.GetBool("fork")
forkOwner, _ := flag.GetString("fork-owner")
sshAuth, _ := flag.GetBool("ssh-auth")
skipForks, _ := flag.GetBool("skip-forks")

if verifyFlags && len(orgs) == 0 && len(users) == 0 && len(repos) == 0 {
return nil, errors.New("no organization, user or repo set")
if verifyFlags && len(orgs) == 0 && len(users) == 0 && len(repos) == 0 && repoSearch == "" {
return nil, errors.New("no organization, user, repo or repo-search set")
}

token, err := getToken(flag)
Expand Down Expand Up @@ -164,11 +166,12 @@ func createGithubClient(flag *flag.FlagSet, verifyFlags bool, readOnly bool) (mu
BaseURL: gitBaseURL,
TransportMiddleware: http.NewLoggingRoundTripper,
RepoListing: github.RepositoryListing{
Organizations: orgs,
Users: users,
Repositories: repoRefs,
Topics: topics,
SkipForks: skipForks,
Organizations: orgs,
Users: users,
Repositories: repoRefs,
RepositorySearch: repoSearch,
Topics: topics,
SkipForks: skipForks,
},
MergeTypes: mergeTypes,
ForkMode: forkMode,
Expand Down
61 changes: 56 additions & 5 deletions internal/scm/github/github.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,11 +105,12 @@ type Github struct {

// RepositoryListing contains information about which repositories that should be fetched
type RepositoryListing struct {
Organizations []string
Users []string
Repositories []RepositoryReference
Topics []string
SkipForks bool
Organizations []string
Users []string
Repositories []RepositoryReference
RepositorySearch string
Topics []string
SkipForks bool
}

// RepositoryReference contains information to be able to reference a repository
Expand Down Expand Up @@ -213,6 +214,14 @@ func (g *Github) getRepositories(ctx context.Context) ([]*github.Repository, err
allRepos = append(allRepos, repo)
}

if g.RepositorySearch != "" {
repos, err := g.getSearchRepositories(ctx, g.RepositorySearch)
if err != nil {
return nil, errors.Wrapf(err, "could not get repository search results for '%s'", g.RepositorySearch)
}
allRepos = append(allRepos, repos...)
}

// Remove duplicate repos
repoMap := map[string]*github.Repository{}
for _, repo := range allRepos {
Expand Down Expand Up @@ -282,6 +291,48 @@ func (g *Github) getUserRepositories(ctx context.Context, user string) ([]*githu
return repos, nil
}

func (g *Github) getSearchRepositories(ctx context.Context, search string) ([]*github.Repository, error) {
var repos []*github.Repository
i := 1
for {
rr, _, err := retry(ctx, func() ([]*github.Repository, *github.Response, error) {
rr, resp, err := g.ghClient.Search.Repositories(ctx, search, &github.SearchOptions{
ListOptions: github.ListOptions{
Page: i,
PerPage: 100,
},
})

if err != nil {
return nil, nil, err
}

if rr.GetIncompleteResults() {
// can occur when search times out on the server: for now, fail instead
// of handling the issue
return nil, nil, fmt.Errorf("search timed out on GitHub and was marked incomplete: try refining the search to return fewer results or be less complex")
}

if rr.GetTotal() > 1000 {
return nil, nil, fmt.Errorf("%d results for this search, but only the first 1000 results will be returned: try refining your search terms", rr.GetTotal())
}

return rr.Repositories, resp, nil
})

if err != nil {
return nil, err
}
repos = append(repos, rr...)
if len(rr) != 100 {
break
}
i++
}

return repos, nil
}

func (g *Github) getRepository(ctx context.Context, repoRef RepositoryReference) (*github.Repository, error) {
repo, _, err := retry(ctx, func() (*github.Repository, *github.Response, error) {
return g.ghClient.Repositories.Get(ctx, repoRef.OwnerName, repoRef.Name)
Expand Down
163 changes: 160 additions & 3 deletions internal/scm/github/github_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ func Test_GetRepositories(t *testing.T) {
"push": true,
"pull": true
},
"created_at": "2020-01-02T16:49:16Z"
"created_at": "2020-01-02T16:49:17Z"
}`,
"/users/test-user/repos": `[
{
Expand All @@ -117,9 +117,41 @@ func Test_GetRepositories(t *testing.T) {
"push": true,
"pull": true
},
"created_at": "2020-01-03T16:49:16Z"
"created_at": "2020-01-03T16:49:18Z"
}
]`,
"/search/repositories": `{
"total_count": 1,
"incomplete_results": false,
"items": [
{
"id": 3,
"name": "search-repo1",
"full_name": "lindell/search-repo1",
"private": false,
"topics": [
"backend",
"go"
],
"owner": {
"login": "lindell",
"type": "User",
"site_admin": false
},
"html_url": "https://github.com/lindell/search-repo1",
"fork": true,
"archived": false,
"disabled": false,
"default_branch": "main",
"permissions": {
"admin": true,
"push": true,
"pull": true
},
"created_at": "2020-01-03T16:49:19Z"
}
]
}`,
},
}

Expand Down Expand Up @@ -205,6 +237,25 @@ func Test_GetRepositories(t *testing.T) {
}
}

// Repo search
{
gh, err := github.New(github.Config{
TransportMiddleware: transport.Wrapper,
RepoListing: github.RepositoryListing{
RepositorySearch: "search-string",
},
MergeTypes: []scm.MergeType{scm.MergeTypeMerge},
})
require.NoError(t, err)

repos, err := gh.GetRepositories(context.Background())
assert.NoError(t, err)
if assert.Len(t, repos, 1) {
assert.Equal(t, "main", repos[0].DefaultBranch())
assert.Equal(t, "lindell/search-repo1", repos[0].FullName())
}
}

// Multiple
{
gh, err := github.New(github.Config{
Expand All @@ -218,18 +269,20 @@ func Test_GetRepositories(t *testing.T) {
Name: "test1",
},
},
RepositorySearch: "test-search",
},
MergeTypes: []scm.MergeType{scm.MergeTypeMerge},
})
require.NoError(t, err)

repos, err := gh.GetRepositories(context.Background())
assert.NoError(t, err)
if assert.Len(t, repos, 2) {
if assert.Len(t, repos, 3) {
assert.Equal(t, "master", repos[0].DefaultBranch())
assert.Equal(t, "test-org/test1", repos[0].FullName())
assert.Equal(t, "main", repos[1].DefaultBranch())
assert.Equal(t, "lindell/test2", repos[1].FullName())
assert.Equal(t, "lindell/search-repo1", repos[2].FullName())
}
}

Expand Down Expand Up @@ -259,3 +312,107 @@ func Test_GetRepositories(t *testing.T) {
}
}
}

func Test_GetSearchRepository_Incomplete(t *testing.T) {
transport := testTransport{
pathBodies: map[string]string{
"/search/repositories": `{
"total_count": 1,
"incomplete_results": true,
"items": [
{
"id": 3,
"name": "search-repo1",
"full_name": "lindell/search-repo1",
"private": false,
"topics": [
"backend",
"go"
],
"owner": {
"login": "lindell",
"type": "User",
"site_admin": false
},
"html_url": "https://github.com/lindell/search-repo1",
"fork": true,
"archived": false,
"disabled": false,
"default_branch": "main",
"permissions": {
"admin": true,
"push": true,
"pull": true
},
"created_at": "2020-01-03T16:49:16Z"
}
]
}`,
},
}

gh, err := github.New(github.Config{
TransportMiddleware: transport.Wrapper,
RepoListing: github.RepositoryListing{
RepositorySearch: "search-string",
},
MergeTypes: []scm.MergeType{scm.MergeTypeMerge},
})
require.NoError(t, err)

repos, err := gh.GetRepositories(context.Background())
assert.ErrorContains(t, err, "search timed out on GitHub and was marked incomplete")
assert.Len(t, repos, 0)
}

func Test_GetSearchRepository_TooManyResults(t *testing.T) {
transport := testTransport{
pathBodies: map[string]string{
"/search/repositories": `{
"total_count": 2054,
"incomplete_results": false,
"items": [
{
"id": 3,
"name": "search-repo1",
"full_name": "lindell/search-repo1",
"private": false,
"topics": [
"backend",
"go"
],
"owner": {
"login": "lindell",
"type": "User",
"site_admin": false
},
"html_url": "https://github.com/lindell/search-repo1",
"fork": true,
"archived": false,
"disabled": false,
"default_branch": "main",
"permissions": {
"admin": true,
"push": true,
"pull": true
},
"created_at": "2020-01-03T16:49:16Z"
}
]
}`,
},
}

gh, err := github.New(github.Config{
TransportMiddleware: transport.Wrapper,
RepoListing: github.RepositoryListing{
RepositorySearch: "search-string",
},
MergeTypes: []scm.MergeType{scm.MergeTypeMerge},
})
require.NoError(t, err)

repos, err := gh.GetRepositories(context.Background())
assert.ErrorContains(t, err, "only the first 1000 results will be returned")
assert.Len(t, repos, 0)
}

0 comments on commit 0f8c2dc

Please sign in to comment.