-
Notifications
You must be signed in to change notification settings - Fork 0
/
db.go
177 lines (155 loc) · 4.21 KB
/
db.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
package prow
import (
"database/sql"
"log"
"net/url"
"os"
"path"
"regexp"
"time"
"github.com/spf13/cobra"
_ "github.com/mattn/go-sqlite3"
)
func NewDBCommand() *cobra.Command {
var command = &cobra.Command{
Use: "db",
Short: "Prow build database functions.",
}
command.AddCommand(newCreateCommand())
return command
}
var storagePattern = regexp.MustCompile(`.*/(origin-ci-test/.*)`)
type createOptions struct {
BaseURL string
StorageBaseURL string
From time.Duration
Jobs []string
OutputFile string
}
func newCreateCommand() *cobra.Command {
var options createOptions
var command = &cobra.Command{
Use: "create",
Short: "Creates a sqlite database of CI build history.",
Run: func(cmd *cobra.Command, args []string) {
err := create(options)
if err != nil {
panic(err)
}
},
}
command.Flags().StringVarP(&options.BaseURL, "base-url", "", "https://prow.ci.openshift.org", "")
command.Flags().StringVarP(&options.StorageBaseURL, "storage-base-url", "", "https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs", "GCS storage base")
command.Flags().DurationVarP(&options.From, "from", "", 24*time.Hour, "how far back to find builds")
command.Flags().StringArrayVarP(&options.Jobs, "job", "", []string{"release-openshift-ocp-installer-e2e-aws-4.6"}, "jobs to find")
command.Flags().StringVarP(&options.OutputFile, "output-file", "f", path.Join(os.Getenv("HOME"), ".dowser.db"), "output database file location")
return command
}
type build struct {
Build
Job string
URL string
}
func create(options createOptions) error {
buildC := make(chan []build)
for _, job := range options.Jobs {
go func(job string) {
builds, err := findBuilds(options.From, options.BaseURL, options.StorageBaseURL, job)
if err != nil {
log.Printf("couldn't find builds for job %s: %s", job, err)
buildC <- []build{}
} else {
buildC <- builds
}
}(job)
}
db, err := sql.Open("sqlite3", options.OutputFile)
if err != nil {
return err
}
defer db.Close()
sqlStmt := `
create table if not exists jobs (id text not null primary key, name text, result text, url text, started text, duration numeric);
`
_, err = db.Exec(sqlStmt)
if err != nil {
return err
}
tx, err := db.Begin()
if err != nil {
return err
}
stmt, err := tx.Prepare(`
insert into jobs(id, name, result, url, started, duration)
values(?, ?, ?, ?, ?, ?)
on conflict(id) do update set name=excluded.name, result=excluded.result, url=excluded.url, started=excluded.started, duration=excluded.duration;
`)
if err != nil {
return err
}
defer stmt.Close()
for range options.Jobs {
for _, build := range <-buildC {
_, err = stmt.Exec(build.ID, build.Job, build.Result, build.URL, build.Started.UTC().Format(time.RFC3339), build.Duration)
if err != nil {
log.Printf("error inserting:\nbuild: %#v\nerror: %v\n", build, err)
}
}
}
err = tx.Commit()
if err != nil {
return err
}
log.Printf("wrote build database to %s", options.OutputFile)
return nil
}
func findBuilds(from time.Duration, baseURL string, storageBaseURL string, job string) ([]build, error) {
var builds []build
jobURL, err := url.Parse(baseURL)
if err != nil {
return nil, err
}
jobURL.Path = path.Join(jobURL.Path, "job-history/gs/origin-ci-test/logs", job)
prowBuilds, err := GetJobHistory(from, jobURL.String())
if err != nil {
return nil, err
}
log.Printf("found %d prow builds for job %s", len(prowBuilds), job)
prowBuildC := make(chan Build, len(prowBuilds))
for i := range prowBuilds {
prowBuildC <- prowBuilds[i]
}
buildC := make(chan *build)
done := make(chan bool)
for i := 0; i < 5; i++ {
go func() {
for {
select {
case <-done:
return
case prowBuild := <-prowBuildC:
buildURL, err := url.Parse(baseURL)
if err != nil {
log.Printf("invalid prow build url %q: %s", baseURL, err)
buildC <- nil
break
}
buildURL.Path = path.Join(buildURL.Path, prowBuild.SpyglassLink)
build := build{
Build: prowBuild,
Job: job,
URL: buildURL.String(),
}
buildC <- &build
}
}
}()
}
for range prowBuilds {
if build := <-buildC; build != nil {
builds = append(builds, *build)
}
}
close(done)
return builds, nil
}