-
Notifications
You must be signed in to change notification settings - Fork 246
/
registry.go
525 lines (460 loc) · 15.1 KB
/
registry.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
package registry
import (
"context"
"fmt"
"io/ioutil"
"os"
"github.com/sirupsen/logrus"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"github.com/operator-framework/operator-registry/pkg/containertools"
"github.com/operator-framework/operator-registry/pkg/image"
"github.com/operator-framework/operator-registry/pkg/image/containerdregistry"
"github.com/operator-framework/operator-registry/pkg/image/execregistry"
"github.com/operator-framework/operator-registry/pkg/lib/certs"
"github.com/operator-framework/operator-registry/pkg/registry"
"github.com/operator-framework/operator-registry/pkg/sqlite"
)
type RegistryUpdater struct {
Logger *logrus.Entry
}
type AddToRegistryRequest struct {
Permissive bool
SkipTLS bool
CaFile string
InputDatabase string
Bundles []string
Mode registry.Mode
ContainerTool containertools.ContainerTool
Overwrite bool
EnableAlpha bool
}
func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
db, err := sqlite.Open(request.InputDatabase)
if err != nil {
return err
}
defer db.Close()
dbLoader, err := sqlite.NewSQLLiteLoader(db, sqlite.WithEnableAlpha(request.EnableAlpha))
if err != nil {
return err
}
if err := dbLoader.Migrate(context.TODO()); err != nil {
return err
}
graphLoader, err := sqlite.NewSQLGraphLoaderFromDB(db)
if err != nil {
return err
}
dbQuerier := sqlite.NewSQLLiteQuerierFromDb(db)
// add custom ca certs to resolver
var reg image.Registry
var rerr error
switch request.ContainerTool {
case containertools.NoneTool:
rootCAs, err := certs.RootCAs(request.CaFile)
if err != nil {
return fmt.Errorf("failed to get RootCAs: %v", err)
}
reg, rerr = containerdregistry.NewRegistry(containerdregistry.SkipTLS(request.SkipTLS), containerdregistry.WithRootCAs(rootCAs))
case containertools.PodmanTool:
fallthrough
case containertools.DockerTool:
reg, rerr = execregistry.NewRegistry(request.ContainerTool, r.Logger, containertools.SkipTLS(request.SkipTLS))
}
if rerr != nil {
return rerr
}
defer func() {
if err := reg.Destroy(); err != nil {
r.Logger.WithError(err).Warn("error destroying local cache")
}
}()
simpleRefs := make([]image.Reference, 0)
for _, ref := range request.Bundles {
simpleRefs = append(simpleRefs, image.SimpleReference(ref))
}
if err := populate(context.TODO(), dbLoader, graphLoader, dbQuerier, reg, simpleRefs, request.Mode, request.Overwrite); err != nil {
r.Logger.Debugf("unable to populate database: %s", err)
if !request.Permissive {
r.Logger.WithError(err).Error("permissive mode disabled")
return err
}
r.Logger.WithError(err).Warn("permissive mode enabled")
}
return nil
}
func unpackImage(ctx context.Context, reg image.Registry, ref image.Reference) (image.Reference, string, func(), error) {
var errs []error
workingDir, err := ioutil.TempDir("./", "bundle_tmp")
if err != nil {
errs = append(errs, err)
}
if err = reg.Pull(ctx, ref); err != nil {
errs = append(errs, err)
}
if err = reg.Unpack(ctx, ref, workingDir); err != nil {
errs = append(errs, err)
}
cleanup := func() {
if err := os.RemoveAll(workingDir); err != nil {
logrus.Error(err)
}
}
if len(errs) > 0 {
return nil, "", cleanup, utilerrors.NewAggregate(errs)
}
return ref, workingDir, cleanup, nil
}
func populate(ctx context.Context, loader registry.Load, graphLoader registry.GraphLoader, querier registry.Query, reg image.Registry, refs []image.Reference, mode registry.Mode, overwrite bool) error {
unpackedImageMap := make(map[image.Reference]string, 0)
for _, ref := range refs {
to, from, cleanup, err := unpackImage(ctx, reg, ref)
if err != nil {
return err
}
unpackedImageMap[to] = from
defer cleanup()
}
overwriteImageMap := make(map[string]map[image.Reference]string, 0)
if overwrite {
// find all bundles that are attempting to overwrite
for to, from := range unpackedImageMap {
img, err := registry.NewImageInput(to, from)
if err != nil {
return err
}
overwritten, err := querier.GetBundlePathIfExists(ctx, img.Bundle.Name)
if err != nil {
if err == registry.ErrBundleImageNotInDatabase {
continue
}
return err
}
if overwritten == "" {
return fmt.Errorf("index add --overwrite-latest is only supported when using bundle images")
}
// get all bundle paths for that package - we will re-add these to regenerate the graph
bundles, err := querier.GetBundlesForPackage(ctx, img.Bundle.Package)
if err != nil {
return err
}
type unpackedImage struct {
to image.Reference
from string
cleanup func()
err error
}
unpacked := make(chan unpackedImage)
for bundle := range bundles {
// parallelize image pulls
go func(bundle registry.BundleKey, img *registry.ImageInput) {
if bundle.CsvName != img.Bundle.Name {
to, from, cleanup, err := unpackImage(ctx, reg, image.SimpleReference(bundle.BundlePath))
unpacked <- unpackedImage{to: to, from: from, cleanup: cleanup, err: err}
} else {
unpacked <- unpackedImage{to: to, from: from, cleanup: func() { return }, err: nil}
}
}(bundle, img)
}
if _, ok := overwriteImageMap[img.Bundle.Package]; !ok {
overwriteImageMap[img.Bundle.Package] = make(map[image.Reference]string, 0)
}
for i := 0; i < len(bundles); i++ {
unpack := <-unpacked
if unpack.err != nil {
return unpack.err
}
overwriteImageMap[img.Bundle.Package][unpack.to] = unpack.from
if _, ok := unpackedImageMap[unpack.to]; ok {
delete(unpackedImageMap, unpack.to)
}
defer unpack.cleanup()
}
}
}
populator := registry.NewDirectoryPopulator(loader, graphLoader, querier, unpackedImageMap, overwriteImageMap, overwrite)
if err := populator.Populate(mode); err != nil {
return err
}
for _, imgMap := range overwriteImageMap {
for to, from := range imgMap {
unpackedImageMap[to] = from
}
}
return checkForBundles(ctx, querier.(*sqlite.SQLQuerier), graphLoader, unpackedImageMap)
}
type DeleteFromRegistryRequest struct {
Permissive bool
InputDatabase string
Packages []string
}
func (r RegistryUpdater) DeleteFromRegistry(request DeleteFromRegistryRequest) error {
db, err := sqlite.Open(request.InputDatabase)
if err != nil {
return err
}
defer db.Close()
dbLoader, err := sqlite.NewDeprecationAwareLoader(db)
if err != nil {
return err
}
if err := dbLoader.Migrate(context.TODO()); err != nil {
return err
}
for _, pkg := range request.Packages {
remover := sqlite.NewSQLRemoverForPackages(dbLoader, pkg)
if err := remover.Remove(); err != nil {
err = fmt.Errorf("error deleting packages from database: %s", err)
if !request.Permissive {
logrus.WithError(err).Fatal("permissive mode disabled")
return err
}
logrus.WithError(err).Warn("permissive mode enabled")
}
}
// remove any stranded bundles from the database
// TODO: This is unnecessary if the db schema can prevent this orphaned data from existing
remover := sqlite.NewSQLStrandedBundleRemover(dbLoader)
if err := remover.Remove(); err != nil {
return fmt.Errorf("error removing stranded packages from database: %s", err)
}
return nil
}
type PruneStrandedFromRegistryRequest struct {
InputDatabase string
}
func (r RegistryUpdater) PruneStrandedFromRegistry(request PruneStrandedFromRegistryRequest) error {
db, err := sqlite.Open(request.InputDatabase)
if err != nil {
return err
}
defer db.Close()
dbLoader, err := sqlite.NewSQLLiteLoader(db)
if err != nil {
return err
}
if err := dbLoader.Migrate(context.TODO()); err != nil {
return err
}
remover := sqlite.NewSQLStrandedBundleRemover(dbLoader)
if err := remover.Remove(); err != nil {
return fmt.Errorf("error removing stranded packages from database: %s", err)
}
return nil
}
type PruneFromRegistryRequest struct {
Permissive bool
InputDatabase string
Packages []string
}
func (r RegistryUpdater) PruneFromRegistry(request PruneFromRegistryRequest) error {
db, err := sqlite.Open(request.InputDatabase)
if err != nil {
return err
}
defer db.Close()
dbLoader, err := sqlite.NewSQLLiteLoader(db)
if err != nil {
return err
}
if err := dbLoader.Migrate(context.TODO()); err != nil {
return err
}
// get all the packages
lister := sqlite.NewSQLLiteQuerierFromDb(db)
packages, err := lister.ListPackages(context.TODO())
if err != nil {
return err
}
// make it inexpensive to find packages
pkgMap := make(map[string]bool)
for _, pkg := range request.Packages {
pkgMap[pkg] = true
}
// prune packages from registry
for _, pkg := range packages {
if _, found := pkgMap[pkg]; !found {
remover := sqlite.NewSQLRemoverForPackages(dbLoader, pkg)
if err := remover.Remove(); err != nil {
err = fmt.Errorf("error deleting packages from database: %s", err)
if !request.Permissive {
logrus.WithError(err).Fatal("permissive mode disabled")
return err
}
logrus.WithError(err).Warn("permissive mode enabled")
}
}
}
return nil
}
type DeprecateFromRegistryRequest struct {
Permissive bool
InputDatabase string
Bundles []string
AllowPackageRemoval bool
}
func (r RegistryUpdater) DeprecateFromRegistry(request DeprecateFromRegistryRequest) error {
db, err := sqlite.Open(request.InputDatabase)
if err != nil {
return err
}
defer db.Close()
dbLoader, err := sqlite.NewSQLLiteLoader(db)
if err != nil {
return err
}
if err := dbLoader.Migrate(context.TODO()); err != nil {
return fmt.Errorf("unable to migrate database: %s", err)
}
// Check if all bundlepaths are valid
var toDeprecate []string
dbQuerier := sqlite.NewSQLLiteQuerierFromDb(db)
toDeprecate, _, err = checkForBundlePaths(dbQuerier, request.Bundles)
if err != nil {
if !request.Permissive {
r.Logger.WithError(err).Error("permissive mode disabled")
return err
}
r.Logger.WithError(err).Warn("permissive mode enabled")
}
deprecator := sqlite.NewSQLDeprecatorForBundles(dbLoader, toDeprecate)
// Check for deprecation of head of default channel. If deprecation request includes heads of all other channels,
// then remove the package entirely. Otherwise, deprecate provided bundles. This enables deprecating an entire package.
// By default deprecating the head of default channel is not permitted.
if request.AllowPackageRemoval {
packageDeprecator := sqlite.NewSQLDeprecatorForBundlesAndPackages(deprecator, dbQuerier)
if err := packageDeprecator.MaybeRemovePackages(); err != nil {
r.Logger.Debugf("unable to deprecate package from database: %s", err)
if !request.Permissive {
r.Logger.WithError(err).Error("permissive mode disabled")
return err
}
r.Logger.WithError(err).Warn("permissive mode enabled")
}
}
// Any bundles associated with removed packages are now removed from the list of bundles to deprecate.
if err := deprecator.Deprecate(); err != nil {
r.Logger.Debugf("unable to deprecate bundles from database: %s", err)
if !request.Permissive {
r.Logger.WithError(err).Error("permissive mode disabled")
return err
}
r.Logger.WithError(err).Warn("permissive mode enabled")
}
return nil
}
// checkForBundlePaths verifies presence of a list of bundle paths in the registry.
func checkForBundlePaths(querier registry.GRPCQuery, bundlePaths []string) ([]string, []string, error) {
if len(bundlePaths) == 0 {
return bundlePaths, nil, nil
}
registryBundles, err := querier.ListBundles(context.TODO())
if err != nil {
return bundlePaths, nil, err
}
if len(registryBundles) == 0 {
return nil, bundlePaths, nil
}
registryBundlePaths := map[string]struct{}{}
for _, b := range registryBundles {
registryBundlePaths[b.BundlePath] = struct{}{}
}
var found, missing []string
for _, b := range bundlePaths {
if _, ok := registryBundlePaths[b]; ok {
found = append(found, b)
continue
}
missing = append(missing, b)
}
if len(missing) > 0 {
return found, missing, fmt.Errorf("target bundlepaths for deprecation missing from registry: %v", missing)
}
return found, missing, nil
}
// packagesFromUnpackedRefs creates packages from a set of unpacked ref dirs without their upgrade edges.
func packagesFromUnpackedRefs(bundles map[image.Reference]string) (map[string]registry.Package, error) {
graph := map[string]registry.Package{}
for to, from := range bundles {
b, err := registry.NewImageInput(to, from)
if err != nil {
return nil, fmt.Errorf("failed to parse unpacked bundle image %s: %v", to, err)
}
v, err := b.Bundle.Version()
if err != nil {
return nil, fmt.Errorf("failed to parse version for %s (%s): %v", b.Bundle.Name, b.Bundle.BundleImage, err)
}
key := registry.BundleKey{
CsvName: b.Bundle.Name,
Version: v,
BundlePath: b.Bundle.BundleImage,
}
if _, ok := graph[b.Bundle.Package]; !ok {
graph[b.Bundle.Package] = registry.Package{
Name: b.Bundle.Package,
Channels: map[string]registry.Channel{},
}
}
for _, c := range b.Bundle.Channels {
if _, ok := graph[b.Bundle.Package].Channels[c]; !ok {
graph[b.Bundle.Package].Channels[c] = registry.Channel{
Nodes: map[registry.BundleKey]map[registry.BundleKey]struct{}{},
}
}
graph[b.Bundle.Package].Channels[c].Nodes[key] = nil
}
}
return graph, nil
}
// replaces mode selects highest version as channel head and
// prunes any bundles in the upgrade chain after the channel head.
// check for the presence of all bundles after a replaces-mode add.
func checkForBundles(ctx context.Context, q *sqlite.SQLQuerier, g registry.GraphLoader, bundles map[image.Reference]string) error {
if len(bundles) == 0 {
return nil
}
required, err := packagesFromUnpackedRefs(bundles)
if err != nil {
return err
}
var errs []error
for _, pkg := range required {
graph, err := g.Generate(pkg.Name)
if err != nil {
errs = append(errs, fmt.Errorf("unable to verify added bundles for package %s: %v", pkg.Name, err))
continue
}
for channel, missing := range pkg.Channels {
// trace replaces chain for reachable bundles
for next := []registry.BundleKey{graph.Channels[channel].Head}; len(next) > 0; next = next[1:] {
delete(missing.Nodes, next[0])
for edge := range graph.Channels[channel].Nodes[next[0]] {
next = append(next, edge)
}
}
for bundle := range missing.Nodes {
// check if bundle is deprecated. Bundles readded after deprecation should not be present in index and can be ignored.
deprecated, err := isDeprecated(ctx, q, bundle)
if err != nil {
errs = append(errs, fmt.Errorf("could not validate pruned bundle %s (%s) as deprecated: %v", bundle.CsvName, bundle.BundlePath, err))
}
if !deprecated {
errs = append(errs, fmt.Errorf("added bundle %s pruned from package %s, channel %s: this may be due to incorrect channel head (%s)", bundle.BundlePath, pkg.Name, channel, graph.Channels[channel].Head.CsvName))
}
}
}
}
return utilerrors.NewAggregate(errs)
}
func isDeprecated(ctx context.Context, q *sqlite.SQLQuerier, bundle registry.BundleKey) (bool, error) {
props, err := q.GetPropertiesForBundle(ctx, bundle.CsvName, bundle.Version, bundle.BundlePath)
if err != nil {
return false, err
}
for _, prop := range props {
if prop.Type == registry.DeprecatedType {
return true, nil
}
}
return false, nil
}