-
Notifications
You must be signed in to change notification settings - Fork 16
/
map.go
208 lines (180 loc) · 6.5 KB
/
map.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
package cmd
import (
"context"
"encoding/json"
"fmt"
"os"
"sort"
"time"
"github.com/pkg/errors"
"cloud.google.com/go/civil"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/openshift-eng/ci-test-mapping/cmd/ci-test-mapping/flags"
v1 "github.com/openshift-eng/ci-test-mapping/pkg/api/types/v1"
"github.com/openshift-eng/ci-test-mapping/pkg/bigquery"
"github.com/openshift-eng/ci-test-mapping/pkg/components"
"github.com/openshift-eng/ci-test-mapping/pkg/jira"
"github.com/openshift-eng/ci-test-mapping/pkg/obsoletetests"
"github.com/openshift-eng/ci-test-mapping/pkg/registry"
)
const ModeBigQuery = "bigquery"
const ModeLocal = "local"
var mapCmd = &cobra.Command{
Use: "map",
Short: "Map tests to components and capabilities",
RunE: func(cmd *cobra.Command, args []string) error {
if err := verifyParams(); err != nil {
_ = cmd.Usage()
return err
}
var tests []v1.TestInfo
var tableManager *bigquery.MappingTableManager
if f.mode == ModeBigQuery {
// Get a bigquery client
bigqueryClient, err := bigquery.NewClient(context.Background(),
f.bigqueryFlags.ServiceAccountCredentialFile,
f.bigqueryFlags.OAuthClientCredentialFile)
if err != nil {
return errors.WithMessage(err, "could not obtain bigquery client")
}
// Create or update schema for mapping table
tableManager = bigquery.NewMappingTableManager(context.Background(), bigqueryClient)
if err := tableManager.Migrate(); err != nil {
return errors.WithMessage(err, "could not migrate mapping table")
}
// Get a list of all tests from bigquery - this could be swapped out with other
// mechanisms to get test details later on.
testLister := bigquery.NewTestTableManager(context.Background(), bigqueryClient)
tests, err = testLister.ListTests()
if err != nil {
return errors.WithMessage(err, "could not list tests")
}
if err := writeRecords(tests, "bigquery_tests.json"); err != nil {
return errors.WithMessage(err, "couldn't write records")
}
} else {
data, err := os.ReadFile(f.testsFile)
if err != nil {
return errors.WithMessage(err, "could not fetch tests from file")
}
if err := json.Unmarshal(data, &tests); err != nil {
return errors.WithMessage(err, "could not marshal tests from file")
}
}
// Create a registry of components
componentRegistry := registry.NewComponentRegistry()
// Query each component for each test
now := time.Now()
createdAt := civil.DateTimeOf(now)
log.Infof("mapping tests to ownership")
jiraComponentIDs, err := jira.GetJiraComponents()
if err != nil {
return errors.WithMessage(err, "could not get jira component mapping")
}
testObsoleter := &obsoletetests.OCPObsoleteTestManager{}
testIdentifier := components.New(componentRegistry, jiraComponentIDs)
var newMappings []v1.TestOwnership
var matched, unmatched int
success := true
for i := range tests {
ownership, err := testIdentifier.Identify(&tests[i])
if err != nil {
log.WithError(err).Warningf("encountered error in component identification")
success = false
continue
}
if ownership != nil {
if ownership.Component == components.DefaultComponent {
unmatched++
} else {
matched++
}
ownership.CreatedAt = createdAt
ownership.StaffApprovedObsolete = testObsoleter.IsObsolete(&tests[i])
newMappings = append(newMappings, *ownership)
}
}
if !success {
return fmt.Errorf("encountered errors while trying to identify tests")
}
// Ensure slice is sorted
sort.Slice(newMappings, func(i, j int) bool {
return newMappings[i].Name < newMappings[j].Name && newMappings[i].Suite < newMappings[j].Suite
})
log.WithFields(log.Fields{
"matched": matched,
"unmatched": unmatched,
}).Infof("mapping tests to ownership complete in %v", time.Since(now))
if f.mode == ModeBigQuery && f.pushToBQ {
now = time.Now()
log.Infof("pushing to bigquery...")
if err := tableManager.PushMappings(newMappings); err != nil {
return errors.WithMessage(err, "could not push records to bigquery")
}
log.Infof("push finished in %+v", time.Since(now))
}
if err := writeRecords(newMappings, f.mappingFile); err != nil {
return errors.WithMessage(err, "could not write records to mapping file")
}
return nil
},
}
type MapFlags struct {
mode string
mappingFile string
testsFile string
pushToBQ bool
bigqueryFlags *flags.Flags
}
var f = NewMapFlags()
func NewMapFlags() *MapFlags {
return &MapFlags{
bigqueryFlags: flags.NewFlags(),
}
}
func (f *MapFlags) BindFlags(fs *pflag.FlagSet) {
f.bigqueryFlags.BindFlags(fs)
}
func init() {
mapCmd.PersistentFlags().StringVar(&f.mappingFile, "mapping-file", "mapping.json",
"File containing existing mappings")
mapCmd.PersistentFlags().StringVar(&f.testsFile, "tests-file", "bigquery_tests.json", "File containing a list of tests to process, see bigquery_tests.json. For local testing without access to canonical test data from BigQuery.")
mapCmd.PersistentFlags().StringVar(&f.mode, "mode", "local", "Mode (one of: local, bigquery). Local mode doesn't require access to BigQuery and is suitable for local development.")
mapCmd.PersistentFlags().BoolVar(&f.pushToBQ, "push-to-bigquery", false, "whether or not to push the updated records to bigquery")
f.BindFlags(mapCmd.Flags())
rootCmd.AddCommand(mapCmd)
}
func verifyParams() error {
switch f.mode {
case ModeBigQuery:
if f.bigqueryFlags.ServiceAccountCredentialFile == "" && f.bigqueryFlags.OAuthClientCredentialFile == "" {
return fmt.Errorf("please supply bigquery credentials, or use --mode=local") //nolint
}
case ModeLocal:
if f.pushToBQ {
return fmt.Errorf("cannot push to bigquery in --mode=local") //nolint
}
if f.bigqueryFlags.ServiceAccountCredentialFile != "" || f.bigqueryFlags.OAuthClientCredentialFile != "" {
return fmt.Errorf("bigquery credentials not required for local mode, maybe you meant to specify --mode=bigquery") //nolint
}
default:
return fmt.Errorf("invalid mode, must be one of: bigquery, local. got: %q", f.mode) //nolint
}
return nil
}
func writeRecords(records interface{}, filename string) error {
now := time.Now()
log.Infof("writing results to file")
f, err := os.OpenFile(filename, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0o644)
if err != nil {
log.WithError(err).Errorf("could not open file for writing")
return err
}
jsonEncoder := json.NewEncoder(f)
jsonEncoder.SetIndent("", " ")
err = jsonEncoder.Encode(records)
log.Infof("write complete in %+v", time.Since(now))
return err
}