/
feed.go
282 lines (234 loc) · 7.62 KB
/
feed.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
package bond
import (
"context"
"encoding/json"
"io/ioutil"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/mongodb/grip"
"github.com/pkg/errors"
)
// ArtifactsFeed represents the entire structure of the MongoDB build information feed.
// See http://downloads.mongodb.org/full.json for an example.
type ArtifactsFeed struct {
Versions []*ArtifactVersion
mutex sync.RWMutex
table map[string]*ArtifactVersion
dir string
path string
}
// GetArtifactsFeed parses a ArtifactsFeed object from a file on the file system.
// This operation will automatically refresh the feed from
// http://downloads.mongodb.org/full.json if the modification time of
// the file on the file system is more than 48 hours old.
func GetArtifactsFeed(ctx context.Context, path string) (*ArtifactsFeed, error) {
feed, err := NewArtifactsFeed(path)
if err != nil {
return nil, errors.Wrap(err, "building feed")
}
if err := feed.Populate(ctx, 4*time.Hour); err != nil {
return nil, errors.Wrap(err, "getting feed data")
}
return feed, nil
}
// NewArtifactsFeed takes the path of a file and returns an empty
// ArtifactsFeed object. You may specify an empty string as an argument
// to return a feed object homed on a temporary directory.
func NewArtifactsFeed(path string) (*ArtifactsFeed, error) {
f := &ArtifactsFeed{
table: make(map[string]*ArtifactVersion),
path: path,
}
if path == "" {
// no value for feed, let's write it to the tempDir
tmpDir, err := ioutil.TempDir("", "mongodb-downloads")
if err != nil {
return nil, err
}
f.dir = tmpDir
f.path = filepath.Join(tmpDir, "full.json")
} else if strings.HasSuffix(path, ".json") {
f.dir = filepath.Dir(path)
f.path = path
} else {
f.dir = path
f.path = filepath.Join(f.dir, "full.json")
}
if stat, err := os.Stat(f.path); !os.IsNotExist(err) && stat.IsDir() {
// if the thing we think should be the json file
// exists but isn't a file (i.e. directory,) then this
// should be an error.
return nil, errors.Errorf("path '%s' not a JSON file directory", path)
}
return f, nil
}
// Populate updates the local copy of the full feed in the the feed's
// cache if the local file doesn't exist or is older than the
// specified TTL. Additional Populate parses the data feed, using the
// Reload method.
func (feed *ArtifactsFeed) Populate(ctx context.Context, ttl time.Duration) error {
data, err := CacheDownload(ctx, ttl, "http://downloads.mongodb.org/full.json", feed.path, false)
if err != nil {
return errors.Wrap(err, "getting feed data")
}
if err = feed.Reload(data); err != nil {
return errors.Wrap(err, "reloading feed")
}
return nil
}
// Reload takes the content of the full.json file and loads this data
// into the current ArtifactsFeed object, overwriting any existing data.
func (feed *ArtifactsFeed) Reload(data []byte) error {
// file exists, remove it if it's more than 48 hours old.
feed.mutex.Lock()
defer feed.mutex.Unlock()
err := json.Unmarshal(data, feed)
if err != nil {
return errors.Wrap(err, "converting data from JSON")
}
if len(feed.table) > 0 {
feed.table = make(map[string]*ArtifactVersion)
}
for _, version := range feed.Versions {
feed.table[version.Version] = version
version.refresh()
}
return err
}
// GetVersion takes a version string and returns the entire Artifacts version.
// The second value indicates if that release exists in the current feed.
func (feed *ArtifactsFeed) GetVersion(release string) (*ArtifactVersion, bool) {
feed.mutex.RLock()
defer feed.mutex.RUnlock()
version, ok := feed.table[release]
return version, ok
}
// GetLatestArchive given a release series (e.g. 3.2, 3.0, or 3.0),
// return the URL of the "latest" (e.g. nightly) build archive. These
// builds are atypical, and given how they're produced, may not
// necessarily reflect the most recent released or unreleased changes on a branch.
func (feed *ArtifactsFeed) GetLatestArchive(series string, options BuildOptions) (string, error) {
series = coerceSeries(series)
if options.Debug {
return "", errors.New("debug symbols are not valid for nightly releases")
}
version, ok := feed.GetVersion(series + ".0")
if !ok {
return "", errors.Errorf("there is no .0 release for series '%s' in the feed", series)
}
dl, err := version.GetDownload(options)
if err != nil {
return "", errors.Wrapf(err, "fetching download information for series '%s'", series)
}
isDev, err := version.isDevelopmentSeries()
if err != nil {
return "", errors.Wrap(err, "determining version type")
}
// If it's a development series, we just replace the version with the word latest.
// Otherwise the branch name is in the file name, and we take the latest from the series release.
if isDev {
return strings.Replace(dl.Archive.URL, version.Version, "latest", -1), nil
}
return strings.Replace(dl.Archive.URL, version.Version, "v"+series+"-latest", -1), nil
}
// GetCurrentArchive is a helper to download the latest stable release for a specific series.
func (feed *ArtifactsFeed) GetCurrentArchive(series string, options BuildOptions) (string, error) {
feed.mutex.RLock()
defer feed.mutex.RUnlock()
version, err := feed.GetLatestRelease(series)
if err != nil {
return "", errors.Wrapf(err, "finding version for series '%s' ", series)
}
dl, err := version.GetDownload(options)
if err != nil {
return "", errors.Wrap(err, "finding version")
}
return dl.Archive.URL, nil
}
// GetLatestRelease returns the latest official release for a specific series.
func (feed *ArtifactsFeed) GetLatestRelease(series string) (*ArtifactVersion, error) {
series = coerceSeries(series)
if series == "2.4" {
version, ok := feed.GetVersion("2.4.14")
if !ok {
return nil, errors.Errorf("could not find current version 2.4.14")
}
return version, nil
}
for _, version := range feed.Versions {
if version.Current && strings.HasPrefix(version.Version, series) {
return version, nil
}
}
return nil, errors.Errorf("could not find a current version for series '%s'", series)
}
// GetArchives provides an iterator for all archives given a list of
// releases (versions) for a specific set of build operations.
// Returns channels of urls (strings) and errors. Read from the error channel,
// after completing all results.
func (feed *ArtifactsFeed) GetArchives(releases []string, options BuildOptions) (<-chan string, <-chan error) {
output := make(chan string)
errOut := make(chan error)
go func() {
catcher := grip.NewCatcher()
for _, rel := range releases {
// this is a series, have to handle it differently
hasLatest := strings.Contains(rel, "latest")
if len(rel) == 3 || hasLatest {
if hasLatest {
rel = strings.Split(rel, "-")[0]
}
url, err := feed.GetLatestArchive(rel, options)
if err != nil {
catcher.Add(err)
continue
}
output <- url
continue
}
if strings.HasSuffix(rel, "-current") || strings.HasSuffix(rel, "-stable") {
rel = strings.Split(rel, "-")[0]
url, err := feed.GetCurrentArchive(rel, options)
if err != nil {
catcher.Add(err)
continue
}
output <- url
continue
}
version, ok := feed.GetVersion(rel)
if !ok {
catcher.Errorf("no version defined for release '%s'", rel)
continue
}
dl, err := version.GetDownload(options)
if err != nil {
catcher.Add(err)
continue
}
if options.Debug {
output <- dl.Archive.Debug
continue
}
output <- dl.Archive.URL
}
close(output)
if catcher.HasErrors() {
errOut <- catcher.Resolve()
}
close(errOut)
}()
return output, errOut
}
func coerceSeries(series string) string {
if series[0] == 'v' {
series = series[1:]
}
if len(series) > 3 {
series = series[:3]
}
return series
}