/
search.go
119 lines (103 loc) · 2.55 KB
/
search.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
/*
* Revision History:
* Initial: 2018/08/04 Li Zebang
*/
package app
import (
"context"
"errors"
"time"
github "github.com/google/go-github/github"
"github.com/TechCatsLab/logging/logrus"
"github.com/TechCatsLab/github-detector/pkg/filetool"
pool "github.com/TechCatsLab/github-detector/pkg/github"
)
// SearchTaskInfo -
type SearchTaskInfo struct {
RepoDir string
Language string
Pushed time.Duration
Min int
Max int
GPool pool.Pool
}
// NewSearchTaskContext -
func NewSearchTaskContext(info *SearchTaskInfo) context.Context {
ctx := context.Background()
ctx = context.WithValue(ctx, SearchTaskKey, info)
return ctx
}
// SearchTaskFunc -
func SearchTaskFunc(ctx context.Context) (err error) {
info, ok := ctx.Value(SearchTaskKey).(*SearchTaskInfo)
if !ok {
return errors.New("assert fail")
}
client := info.GPool.Get(pool.DefualtClientTag)
defer info.GPool.Put(client)
if client == nil {
return errors.New("no available client")
}
// store stores
store := func(dir string, repos *[]github.Repository) error {
f, err := filetool.Open(dir, filetool.TRUNC, 0)
if err != nil {
return err
}
defer f.Close()
err = filetool.NewEncoder(f).Encode(*repos)
if err != nil {
return err
}
return nil
}
// specify gets right index.
specify := func(pre []github.Repository, rsr *github.RepositoriesSearchResult) int {
index := 0
for len(pre) != 0 && index < len(rsr.Repositories) {
switch {
case pre[len(pre)-1].GetFullName() == rsr.Repositories[index].GetFullName():
index++
return index
case pre[len(pre)-1].GetStargazersCount() == rsr.Repositories[index].GetStargazersCount():
index++
break
default:
return index
}
}
return index
}
var (
rsr *github.RepositoriesSearchResult
resp *github.Response
repos []github.Repository
max = info.Max
)
defer func() {
if err == nil {
err = store(info.RepoDir+ReposJSON, &repos)
}
}()
for {
rsr, resp, err = client.Search(info.Language, info.Pushed, info.Min, max, 1)
if err != nil && resp.Remaining == 0 {
logrus.Errorf("Search %s failed, %v", err)
<-time.After(time.Until(resp.Reset.Time))
continue
} else if err != nil {
return err
} else if rsr.GetIncompleteResults() {
err = errors.New("search results incomplete")
return
}
index := specify(repos, rsr)
repos = append(repos, rsr.Repositories[index:]...)
logrus.Infof("Search number: %d", len(repos))
if rsr.GetTotal() > 100 {
max = rsr.Repositories[len(rsr.Repositories)-1].GetStargazersCount()
continue
}
return nil
}
}