Permalink
Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
313 lines (282 sloc) 8.26 KB
package retrogit
import (
"fmt"
"time"
"appengine"
"appengine/datastore"
"appengine/delay"
"appengine/taskqueue"
"github.com/google/go-github/github"
)
const (
VintageDateFormat = "January 2, 2006"
VintageChunkSize = 1000
)
type RepoVintage struct {
UserId int `datastore:",noindex"`
RepoId int `datastore:",noindex"`
Vintage time.Time `datastore:",noindex"`
}
func getVintageKey(c appengine.Context, userId int, repoId int) *datastore.Key {
return datastore.NewKey(c, "RepoVintage", fmt.Sprintf("%d-%d", userId, repoId), 0, nil)
}
var computeVintageFunc *delay.Function
func computeVintage(c appengine.Context, userId int, userLogin string, repoId int, repoOwnerLogin string, repoName string) error {
account, err := getAccount(c, userId)
if err != nil {
c.Errorf("Could not load account %d: %s. Presumed deleted, aborting computing vintage for %s/%s", userId, err.Error(), repoOwnerLogin, repoName)
return nil
}
oauthTransport := githubOAuthTransport(c)
oauthTransport.Token = &account.OAuthToken
githubClient := github.NewClient(oauthTransport.Client())
repo, response, err := githubClient.Repositories.Get(repoOwnerLogin, repoName)
if response.StatusCode == 403 || response.StatusCode == 404 {
c.Warningf("Got a %d when trying to look up %s/%s (%d)", response.StatusCode, repoOwnerLogin, repoName, repoId)
_, err = datastore.Put(c, getVintageKey(c, userId, repoId), &RepoVintage{
UserId: userId,
RepoId: repoId,
Vintage: time.Unix(0, 0),
})
return err
} else if err != nil {
c.Errorf("Could not load repo %s/%s (%d): %s", repoOwnerLogin, repoName, repoId, err.Error())
return err
}
// Cheap check to see if there are commits before the creation time.
vintage := repo.CreatedAt.UTC()
beforeCreationTime := repo.CreatedAt.UTC().AddDate(0, 0, -1)
commits, response, err := githubClient.Repositories.ListCommits(
repoOwnerLogin,
repoName,
&github.CommitsListOptions{
ListOptions: github.ListOptions{PerPage: 1},
Author: userLogin,
Until: beforeCreationTime,
})
if response != nil && response.StatusCode == 409 {
// GitHub returns with a 409 when a repository is empty.
commits = make([]github.RepositoryCommit, 0)
} else if err != nil {
c.Errorf("Could not load commits for repo %s (%d): %s", *repo.FullName, repoId, err.Error())
return err
}
// If there are, then we use the contributor stats API to figure out when
// the user's first commit in the repository was.
if len(commits) > 0 {
stats, response, err := githubClient.Repositories.ListContributorsStats(repoOwnerLogin, repoName)
if response.StatusCode == 202 {
c.Infof("Stats were not available for %s, will try again later", *repo.FullName)
task, err := computeVintageFunc.Task(userId, userLogin, repoId, repoOwnerLogin, repoName)
if err != nil {
c.Errorf("Could create delayed task for %s: %s", *repo.FullName, err.Error())
return err
}
task.Delay = time.Second * 10
taskqueue.Add(c, task, "")
return nil
}
if err != nil {
c.Errorf("Could not load stats for repo %s: %s", *repo.FullName, err.Error())
return err
}
for _, stat := range stats {
if *stat.Author.ID == userId {
for i := range stat.Weeks {
weekTimestamp := stat.Weeks[i].Week.UTC()
if weekTimestamp.Before(vintage) {
vintage = weekTimestamp
}
}
break
}
}
}
_, err = datastore.Put(c, getVintageKey(c, userId, repoId), &RepoVintage{
UserId: userId,
RepoId: repoId,
Vintage: vintage,
})
if err != nil {
c.Errorf("Could save vintage for repo %s: %s", *repo.FullName, err.Error())
return err
}
return nil
}
func init() {
computeVintageFunc = delay.Func("computeVintage", computeVintage)
}
func fillVintages(c appengine.Context, user *github.User, repos []*Repo) error {
if len(repos) > VintageChunkSize {
for chunkStart := 0; chunkStart < len(repos); chunkStart += VintageChunkSize {
chunkEnd := chunkStart + VintageChunkSize
if chunkEnd > len(repos) {
chunkEnd = len(repos)
}
err := fillVintages(c, user, repos[chunkStart:chunkEnd])
if err != nil {
return err
}
}
return nil
}
keys := make([]*datastore.Key, len(repos))
for i := range repos {
keys[i] = getVintageKey(c, *user.ID, *repos[i].ID)
}
vintages := make([]*RepoVintage, len(repos))
for i := range vintages {
vintages[i] = new(RepoVintage)
}
err := datastore.GetMulti(c, keys, vintages)
if err != nil {
if errs, ok := err.(appengine.MultiError); ok {
for i, err := range errs {
if err == datastore.ErrNoSuchEntity {
vintages[i] = nil
} else if err != nil {
c.Errorf("%d/%s vintage fetch error: %s", i, *repos[i].FullName, err.Error())
return err
}
}
} else {
return err
}
}
for i := range vintages {
repo := repos[i]
vintage := vintages[i]
if vintage != nil {
if vintage.Vintage.Unix() != 0 {
repo.Vintage = vintage.Vintage
}
continue
}
computeVintageFunc.Call(c, *user.ID, *user.Login, *repo.ID, *repo.Owner.Login, *repo.Name)
}
return nil
}
type Repos struct {
AllRepos []*Repo
UserRepos []*Repo
OtherUserRepos []*UserRepos
OldestVintage time.Time
}
func (repos *Repos) Redact() {
for _, repo := range repos.UserRepos {
*repo.HTMLURL = "https://redacted"
*repo.FullName = "redacted/redacted"
}
for _, otherUserRepos := range repos.OtherUserRepos {
*otherUserRepos.User.Login = "redacted"
*otherUserRepos.User.AvatarURL = "https://redacted"
for _, repo := range otherUserRepos.Repos {
*repo.HTMLURL = "https://redacted"
*repo.FullName = "redacted/redacted"
}
}
}
type Repo struct {
*github.Repository
Vintage time.Time
IncludeInDigest bool
}
func newRepo(githubRepo *github.Repository, account *Account) *Repo {
return &Repo{
Repository: githubRepo,
Vintage: githubRepo.CreatedAt.UTC(),
IncludeInDigest: !account.IsRepoIdExcluded(*githubRepo.ID),
}
}
func (repo *Repo) TypeAsOcticonName() string {
if *repo.Fork {
return "repo-forked"
}
if *repo.Private {
return "lock"
}
return "repo"
}
func (repo *Repo) TypeAsClassName() string {
if *repo.Fork {
return "fork"
}
if *repo.Private {
return "private"
}
return ""
}
func (repo *Repo) DisplayVintage() string {
return repo.Vintage.Format(VintageDateFormat)
}
type UserRepos struct {
User *github.User
Repos []*Repo
}
func getRepos(c appengine.Context, githubClient *github.Client, account *Account, user *github.User) (*Repos, error) {
clientUserRepos := make([]github.Repository, 0)
page := 1
for {
pageClientUserRepos, response, err := githubClient.Repositories.List(
// The username parameter must be left blank so that we can get all
// of the repositories the user has access to, not just ones that
// they own.
"",
&github.RepositoryListOptions{
ListOptions: github.ListOptions{
Page: page,
PerPage: 100,
},
})
if err != nil {
return nil, err
}
clientUserRepos = append(clientUserRepos, pageClientUserRepos...)
if response.NextPage == 0 {
break
}
page = response.NextPage
}
repos := &Repos{}
repos.UserRepos = make([]*Repo, 0, len(clientUserRepos))
repos.OtherUserRepos = make([]*UserRepos, 0)
for i := range clientUserRepos {
ownerID := *clientUserRepos[i].Owner.ID
if ownerID == *user.ID {
repos.UserRepos = append(repos.UserRepos, newRepo(&clientUserRepos[i], account))
} else {
var userRepos *UserRepos
for j := range repos.OtherUserRepos {
if *repos.OtherUserRepos[j].User.ID == ownerID {
userRepos = repos.OtherUserRepos[j]
break
}
}
if userRepos == nil {
userRepos = &UserRepos{
User: clientUserRepos[i].Owner,
Repos: make([]*Repo, 0),
}
repos.OtherUserRepos = append(repos.OtherUserRepos, userRepos)
}
userRepos.Repos = append(userRepos.Repos, newRepo(&clientUserRepos[i], account))
}
}
repos.AllRepos = make([]*Repo, 0, len(clientUserRepos))
repos.AllRepos = append(repos.AllRepos, repos.UserRepos...)
for _, userRepos := range repos.OtherUserRepos {
repos.AllRepos = append(repos.AllRepos, userRepos.Repos...)
}
err := fillVintages(c, user, repos.AllRepos)
if err != nil {
return nil, err
}
repos.OldestVintage = time.Now().UTC()
for _, repo := range repos.AllRepos {
repoVintage := repo.Vintage
if repoVintage.Before(repos.OldestVintage) {
repos.OldestVintage = repoVintage
}
}
return repos, nil
}