/
cipd.go
298 lines (259 loc) · 9.13 KB
/
cipd.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cipd
import (
"context"
"go.chromium.org/luci/vpython/api/vpython"
"go.chromium.org/luci/vpython/spec"
"go.chromium.org/luci/vpython/venv"
"go.chromium.org/luci/cipd/client/cipd"
"go.chromium.org/luci/cipd/client/cipd/ensure"
"go.chromium.org/luci/cipd/client/cipd/template"
"go.chromium.org/luci/cipd/common"
"go.chromium.org/luci/common/errors"
"go.chromium.org/luci/common/logging"
"go.chromium.org/luci/common/system/filesystem"
)
// TemplateFunc builds a set of template parameters to augment the default CIPD
// parameter set with.
type TemplateFunc func(context.Context, []*vpython.PEP425Tag) (map[string]string, error)
// PackageLoader is an implementation of venv.PackageLoader that uses the
// CIPD service to fetch packages.
//
// Packages that use the CIPD loader use the CIPD package name as their Path
// and a CIPD version/tag/ref as their Version.
type PackageLoader struct {
// Options are additional client options to use when generating CIPD clients.
Options cipd.ClientOptions
// Template, if not nil, is a callback that will return additional CIPD
// package template parameters. These may be derived from the VirtualEnv's
// runtime environment.
//
// For example, if a user wanted to include the Python PEP425 tag version
// as a CIPD template variable, they could include a "py_pep425_tag"
// template parameter.
Template TemplateFunc
}
var _ venv.PackageLoader = (*PackageLoader)(nil)
// Resolve implements venv.PackageLoader.
//
// The resulting packages slice will be updated in-place with the resolved
// package name and instance ID.
func (pl *PackageLoader) Resolve(c context.Context, e *vpython.Environment) error {
spec := e.Spec
if spec == nil {
return nil
}
expander, err := pl.expanderForTags(c, e.Pep425Tag)
if err != nil {
return err
}
// Generate CIPD client options. If no root is provided, use a temporary root.
if pl.Options.Root != "" {
return pl.resolveWithOpts(c, pl.Options, expander, spec)
}
td := filesystem.TempDir{
Prefix: "vpython_cipd",
CleanupErrFunc: func(tdir string, err error) {
logging.WithError(err).Warningf(c, "Failed to clean up CIPD temporary directory [%s]", tdir)
},
}
return td.With(func(tdir string) error {
opts := pl.Options
opts.Root = tdir
return pl.resolveWithOpts(c, opts, expander, spec)
})
}
// resolveWithOpts resolves the specified packages.
//
// The supplied spec is updated with the resolved packages.
func (pl *PackageLoader) resolveWithOpts(c context.Context, opts cipd.ClientOptions,
expander template.Expander, spec *vpython.Spec) error {
logging.Debugf(c, "Resolving CIPD packages in root [%s]:", opts.Root)
ef, packages := specToEnsureFile(spec)
// Log our unresolved packages. Note that "specToEnsureFile" only creates
// a subdir entry for the root (""), so we don't need to deterministically
// iterate over the full map.
if logging.IsLogging(c, logging.Debug) {
for _, pkg := range ef.PackagesBySubdir[""] {
logging.Debugf(c, "\tUnresolved package: %s", pkg)
}
}
client, err := cipd.NewClient(opts)
if err != nil {
return errors.Annotate(err, "failed to generate CIPD client").Err()
}
// Start a CIPD client batch.
client.BeginBatch(c)
defer client.EndBatch(c)
// Resolve our ensure file.
resolver := cipd.Resolver{Client: client}
resolved, err := resolver.Resolve(c, ef, expander)
if err != nil {
return err
}
// Write the results to "packages". All of them should have been installed
// into the root subdir.
for i, pkg := range resolved.PackagesBySubdir[""] {
packages[i].Name = pkg.PackageName
packages[i].Version = pkg.InstanceID
}
return nil
}
// Ensure implement venv.PackageLoader.
//
// The packages must be valid (PackageIsComplete). If they aren't, Ensure will
// panic.
//
// The CIPD client that is used for the operation is generated from the supplied
// options, opts.
func (pl *PackageLoader) Ensure(c context.Context, root string, packages []*vpython.Spec_Package) error {
pins, err := packagesToPins(packages)
if err != nil {
return errors.Annotate(err, "failed to convert packages to CIPD pins").Err()
}
pinSlice := common.PinSliceBySubdir{
"": pins,
}
// Generate a CIPD client. Use the supplied root.
opts := pl.Options
opts.Root = root
client, err := cipd.NewClient(opts)
if err != nil {
return errors.Annotate(err, "failed to generate CIPD client").Err()
}
// Start a CIPD client batch.
client.BeginBatch(c)
defer client.EndBatch(c)
actionMap, err := client.EnsurePackages(c, pinSlice, cipd.NotParanoid, 1, false)
if err != nil {
return errors.Annotate(err, "failed to install CIPD packages").Err()
}
if len(actionMap) > 0 {
errorCount := 0
for root, action := range actionMap {
errorCount += len(action.Errors)
for _, err := range action.Errors {
logging.Errorf(c, "CIPD root %q action %q for pin %q encountered error: %s", root, err.Action, err.Pin, err)
}
}
if errorCount > 0 {
return errors.Reason("CIPD package installation encountered %d error(s)", errorCount).Err()
}
}
return nil
}
// Verify implements venv.PackageLoader.
func (pl *PackageLoader) Verify(c context.Context, sp *vpython.Spec, tags []*vpython.PEP425Tag) error {
client, err := cipd.NewClient(pl.Options)
if err != nil {
return errors.Annotate(err, "failed to generate CIPD client").Err()
}
client.BeginBatch(c)
defer client.EndBatch(c)
resolver := cipd.Resolver{
Client: client,
VerifyPresence: true,
}
// Build an Ensure file for our specification under each tag and register it
// with our Resolver.
ensureFileErrors := 0
for _, tag := range tags {
tagSlice := []*vpython.PEP425Tag{tag}
tagSpec := sp.Clone()
if err := spec.NormalizeSpec(tagSpec, tagSlice); err != nil {
return errors.Annotate(err, "failed to normalize spec for %q", tag).Err()
}
// Convert our spec into an ensure file.
ef, _ := specToEnsureFile(tagSpec)
expander, err := pl.expanderForTags(c, tagSlice)
if err != nil {
ensureFileErrors++
logging.Errorf(c, "Failed to generate template expander for: %s", tag.TagString())
continue
}
if _, err := resolver.Resolve(c, ef, expander); err != nil {
ensureFileErrors++
ts := tag.TagString()
if merr, ok := err.(errors.MultiError); ok {
for _, err := range merr {
logging.Errorf(c, "For %s - %s", ts, err)
}
} else {
logging.Errorf(c, "For %s - %s", ts, err)
}
}
}
if ensureFileErrors > 0 {
logging.Errorf(c, "Spec could not be resolved for %d tag(s).", ensureFileErrors)
return errors.New("verification failed")
}
logging.Infof(c, "Successfully verified all packages.")
return nil
}
// specToEnsureFile translates the packages named in spec into a CIPD ensure
// file.
//
// It returns an ensure file, ef, containing a specification that loads the
// contents of each package into the CIPD root (""). It also returns packages,
// a slice of pointers to spec's "vpython.Spec_Package" entries. Each PackageDef
// index in ef corresponds to the same index source vpython.Spec_Package from
// spec.
func specToEnsureFile(spec *vpython.Spec) (ef *ensure.File, packages []*vpython.Spec_Package) {
// Create a single package list. Our VirtualEnv will be index 0 (need
// this so we can back-port it into its VirtualEnv property).
//
// These will be updated to their resolved values in-place.
packages = make([]*vpython.Spec_Package, 1, 1+len(spec.Wheel))
packages[0] = spec.Virtualenv
packages = append(packages, spec.Wheel...)
pslice := make(ensure.PackageSlice, len(packages))
for i, pkg := range packages {
pslice[i] = ensure.PackageDef{
PackageTemplate: pkg.Name,
UnresolvedVersion: pkg.Version,
}
}
ef = &ensure.File{
PackagesBySubdir: map[string]ensure.PackageSlice{"": pslice},
}
return
}
func (pl *PackageLoader) expanderForTags(c context.Context, tags []*vpython.PEP425Tag) (template.Expander, error) {
// Build our aggregate template parameters. We prefer our template parameters
// over the local system parameters. This allows us to override CIPD template
// parameters elsewhere, and in the production case we will not override
// any CIPD template parameters.
expander := template.DefaultExpander()
if pl.Template != nil {
loaderTemplate, err := pl.Template(c, tags)
if err != nil {
return nil, errors.Annotate(err, "failed to get CIPD template arguments").Err()
}
for k, v := range loaderTemplate {
expander[k] = v
}
}
return expander, nil
}
func packagesToPins(packages []*vpython.Spec_Package) ([]common.Pin, error) {
pins := make([]common.Pin, len(packages))
for i, pkg := range packages {
pins[i] = common.Pin{
PackageName: pkg.Name,
InstanceID: pkg.Version,
}
}
return pins, nil
}