diff --git a/README.md b/README.md index c9a0669..bff05d3 100644 --- a/README.md +++ b/README.md @@ -15,3 +15,6 @@ If you feel there's a project that would benefit from our help, please open an issue above. The issue itself will have some questions that you can answer to help us discover more about the project. From there we can begin to evaluate the project, and our ability to properly support it. + +We've also started writing a script that tries to find projects which are highly +used, but also lacking recent updates. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..8f665a1 --- /dev/null +++ b/go.mod @@ -0,0 +1,11 @@ +module github.com/gofrs/help-requests + +require ( + github.com/golang/protobuf v1.2.0 // indirect + github.com/google/go-github v17.0.0+incompatible + github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect + golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 + golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be + golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect + google.golang.org/appengine v1.1.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..aa4cc8f --- /dev/null +++ b/go.sum @@ -0,0 +1,14 @@ +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 h1:zLTLjkaOFEFIOxY5BWLFLwh+cL8vOBW4XJ2aqLE/Tf0= +github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..35e72d4 --- /dev/null +++ b/main.go @@ -0,0 +1,115 @@ +package main + +import ( + "context" + "errors" + "fmt" + "math" + "net/http" + "os" + "strconv" + "strings" + "text/tabwriter" + "time" + + "github.com/google/go-github/github" + "golang.org/x/net/html" + "golang.org/x/oauth2" +) + +func main() { + ghClient, err := createGithubClient(context.Background()) + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: problem creating github client: %v", err) + } + + query := "stars:>100 pushed:<2018-01-01 language:Go" + repoRes, res, err := ghClient.Search.Repositories(context.Background(), query, &github.SearchOptions{ + Sort: "stars", + Order: "desc", + ListOptions: github.ListOptions{ + PerPage: 25, + Page: 1, + }, + }) + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: problem reading github repositories: %v", err) + } + res.Close = true + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + fmt.Fprintf(w, "name\tstars\tlast commit (days)\timporters\n") + defer w.Flush() + for i := range repoRes.Repositories { + repo := repoRes.Repositories[i] + + cleanName := strings.Replace(*repo.HTMLURL, `https://`, "", 1) + + // TODO(adam): goroutines + sync.WaitGroup + importers, err := scrapeGodocImports(cleanName) + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: problem grabbing %s godoc importers: %v\n", cleanName, err) + } + + days := int(math.Abs(float64(repo.PushedAt.Sub(time.Now()).Hours()) / 24.0)) + fmt.Fprintf(w, "%s\t%d\t%d\t%d\n", cleanName, *repo.StargazersCount, days, importers) + } +} + +func createGithubClient(ctx context.Context) (*github.Client, error) { + v := os.Getenv("GITHUB_TOKEN") + if v == "" { + return nil, errors.New("environment variable GITHUB_TOKEN is required") + } + ts := oauth2.StaticTokenSource(&oauth2.Token{ + AccessToken: v, + }) + tc := oauth2.NewClient(ctx, ts) + return github.NewClient(tc), nil +} + +func scrapeGodocImports(importPath string) (int, error) { + req, err := http.NewRequest("GET", "https://godoc.org/"+importPath, nil) + if err != nil { + return -1, fmt.Errorf("problem loading godoc.org: %v", err) + } + req.Header.Set("User-Agent", "Gofrs popstalerepo bot") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return -1, fmt.Errorf("problem loading %s: %v", req.URL, err) + } + defer resp.Body.Close() + + // recursive search, from /x/net/html docs + var f func(n *html.Node) (int, error) + f = func(n *html.Node) (int, error) { + if n.Type == html.ElementNode && n.Data == "a" { + for _, a := range n.Attr { + // TODO(adam): we should try and refresh importers + // when running into errors. + if a.Key == "href" && strings.Contains(a.Val, "?importers") { + parts := strings.Fields(n.FirstChild.Data) + n, err := strconv.Atoi(parts[0]) + if err != nil { + return -1, fmt.Errorf("couldn't parse %q: %v", parts[0], err) + } + return n, nil + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + n, err := f(c) + if err == nil && n > 0 { + return n, err + } + } + return -1, errors.New(`didn't find `) + } + + doc, err := html.Parse(resp.Body) + if err != nil { + return -1, fmt.Errorf("couldn't parse html: %v", err) + } + return f(doc) +}