/
metadata.go
233 lines (203 loc) · 7.86 KB
/
metadata.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
// Copyright 2019 The WPT Dashboard Project. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package shared
import (
"strings"
"gopkg.in/yaml.v3"
)
// ShowMetadataParam determines whether Metadata Information returns along
// with a test result query request.
const ShowMetadataParam = "metadataInfo"
// MetadataFileName is the name of Metadata files in the wpt-metadata repo.
const MetadataFileName = "META.yml"
// MetadataResults is a map from test paths to all of the links under that test path.
// It represents a flattened copy of the wpt-metadata repository, which has metadata
// sharded across as large number of files in a directory structure.
type MetadataResults map[string]MetadataLinks
// Metadata represents a wpt-metadata META.yml file.
type Metadata struct {
Links MetadataLinks `yaml:"links"`
}
// MetadataLinks is a helper type for a MetadataLink slice.
type MetadataLinks []MetadataLink
// MetadataLink is an item in the `links` node of a wpt-metadata
// META.yml file, which lists an external reference, optionally
// filtered by product and a specific test.
type MetadataLink struct {
Product ProductSpec `yaml:"product,omitempty" json:"product,omitempty"`
URL string `yaml:"url,omitempty" json:"url"`
Label string `yaml:"label,omitempty" json:"label,omitempty"`
Results []MetadataTestResult `yaml:"results" json:"results,omitempty"`
}
// MetadataTestResult is a filter for test results to which the Metadata link
// should apply.
type MetadataTestResult struct {
TestPath string `yaml:"test" json:"test,omitempty"`
SubtestName *string `yaml:"subtest,omitempty" json:"subtest,omitempty"`
Status *TestStatus `yaml:"status,omitempty" json:"status,omitempty"`
}
// GetMetadataResponse retrieves the response to a WPT Metadata query. Metadata
// is included for any product that matches a passed TestRun. Test-level
// metadata (i.e. that is not associated with any product) may be fetched by
// passing true for includeTestLevel.
func GetMetadataResponse(testRuns []TestRun, includeTestLevel bool, log Logger, fetcher MetadataFetcher) (MetadataResults, error) {
var productSpecs = make([]ProductSpec, len(testRuns))
for i, run := range testRuns {
productSpecs[i] = ProductSpec{ProductAtRevision: run.ProductAtRevision, Labels: run.LabelsSet()}
}
// TODO(kyleju): Include the SHA information in API response;
// see https://github.com/web-platform-tests/wpt.fyi/issues/1938
_, metadata, err := GetMetadataByteMap(log, fetcher)
if err != nil {
return nil, err
}
return constructMetadataResponse(productSpecs, includeTestLevel, metadata), nil
}
// GetMetadataResponseOnProducts constructs the response to a WPT Metadata
// query, given ProductSpecs. Metdata is included for any product that matches
// a passed ProductSpec. Test-level metadata (i.e. that is not associated with
// any product) may be fetched by passing true for includeTestLevel.
func GetMetadataResponseOnProducts(productSpecs ProductSpecs, includeTestLevel bool, log Logger, fetcher MetadataFetcher) (MetadataResults, error) {
// TODO(kyleju): Include the SHA information in API response;
// see https://github.com/web-platform-tests/wpt.fyi/issues/1938
_, metadata, err := GetMetadataByteMap(log, fetcher)
if err != nil {
return nil, err
}
return constructMetadataResponse(productSpecs, includeTestLevel, metadata), nil
}
// GetMetadataByteMap collects and parses all META.yml files from
// the wpt-metadata repository.
func GetMetadataByteMap(log Logger, fetcher MetadataFetcher) (sha *string, metadata map[string]Metadata, err error) {
sha, metadataByteMap, err := fetcher.Fetch()
if err != nil {
log.Errorf("Error from FetchMetadata: %s", err.Error())
return nil, nil, err
}
metadata = parseMetadata(metadataByteMap, log)
return sha, metadata, nil
}
func parseMetadata(metadataByteMap map[string][]byte, log Logger) map[string]Metadata {
var metadataMap = make(map[string]Metadata)
for path, data := range metadataByteMap {
var metadata Metadata
err := yaml.Unmarshal(data, &metadata)
if err != nil {
log.Warningf("Failed to unmarshal %s.", path)
continue
}
metadataMap[path] = metadata
}
return metadataMap
}
// addResponseLink is a helper method for constructMetadataResponse. It creates a new MetadataLink
// object corresponding to a specific MetadataTestResult for a given test, and adds it to a
// MetadataResults map.
func addResponseLink(fullTestName string, link MetadataLink, result MetadataTestResult, outMap MetadataResults) {
newLink := MetadataLink{
Product: link.Product,
URL: link.URL,
Label: link.Label,
}
if result.SubtestName != nil || result.Status != nil {
newLink.Results = []MetadataTestResult{
{
SubtestName: result.SubtestName,
Status: result.Status,
// TestPath is redundant (it's the map key in outMap)
},
}
}
if _, ok := outMap[fullTestName]; !ok {
outMap[fullTestName] = MetadataLinks{newLink}
} else {
outMap[fullTestName] = append(outMap[fullTestName], newLink)
}
}
// constructMetadataResponse constructs the response to a WPT Metadata query, given ProductSpecs.
func constructMetadataResponse(productSpecs ProductSpecs, includeTestLevel bool, metadata map[string]Metadata) MetadataResults {
res := make(MetadataResults)
for folderPath, data := range metadata {
for i := range data.Links {
link := data.Links[i]
for _, result := range link.Results {
//TODO(kyleju): Concatenate test path on WPT Metadata repository instead of here.
fullTestName := GetWPTTestPath(folderPath, result.TestPath)
if link.Product.BrowserName == "" && includeTestLevel {
addResponseLink(fullTestName, link, result, res)
}
// Find any matching product for this link result (there can be at most one).
for _, productSpec := range productSpecs {
// Matches on browser type if a version is not specified.
if link.Product.MatchesProductSpec(productSpec) {
addResponseLink(fullTestName, link, result, res)
break
}
}
}
}
}
return res
}
// PrepareLinkFilter maps a MetadataResult test name to its URLs.
func PrepareLinkFilter(metadata MetadataResults) map[string][]string {
metadataMap := make(map[string][]string)
for test, links := range metadata {
for _, link := range links {
if link.URL == "" {
continue
}
if urls, ok := metadataMap[test]; !ok {
metadataMap[test] = []string{link.URL}
} else {
metadataMap[test] = append(urls, link.URL)
}
}
}
return metadataMap
}
// PrepareTestLabelFilter maps a MetadataResult test name to its labels.
func PrepareTestLabelFilter(metadata MetadataResults) map[string][]string {
metadataMap := make(map[string][]string)
for test, links := range metadata {
for _, link := range links {
if link.Label == "" {
continue
}
if labels, ok := metadataMap[test]; !ok {
metadataMap[test] = []string{link.Label}
} else {
metadataMap[test] = append(labels, link.Label)
}
}
}
return metadataMap
}
// GetWPTTestPath concatenates a folder path and a test name into a WPT test path.
func GetWPTTestPath(folderPath string, testname string) string {
if folderPath == "" {
return "/" + testname
}
return "/" + folderPath + "/" + testname
}
// SplitWPTTestPath splits a WPT test path into a folder path and a test name.
func SplitWPTTestPath(githubPath string) (string, string) {
if !strings.HasPrefix(githubPath, "/") {
return "", ""
}
pathArray := strings.Split(githubPath, "/")[1:]
if len(pathArray) == 1 {
return "", pathArray[0]
}
folderPath := strings.Join(pathArray[:len(pathArray)-1], "/")
testName := pathArray[len(pathArray)-1]
return folderPath, testName
}
// GetMetadataFilePath appends MetadataFileName to a Metadata folder path.
func GetMetadataFilePath(folderName string) string {
if folderName == "" {
return MetadataFileName
}
return folderName + "/" + MetadataFileName
}