Skip to content

Commit

Permalink
(feat) Add semver insert mode
Browse files Browse the repository at this point in the history
This adds a new flag to `opm index add` and `opm registry add` which
defines two new insert modes to add bundles to the existing update graph

Rather than rely on parsing the CSV for the replaces field to define the
channel update graph explicitly, in these new modes `opm` implicitly
infers the update graph based on the version attached to the bundle and
semantic versioning rules (https://semver.org/#summary)
  • Loading branch information
kevinrizza committed Apr 6, 2020
1 parent 1a514b0 commit bf35fab
Show file tree
Hide file tree
Showing 22 changed files with 997 additions and 65 deletions.
10 changes: 10 additions & 0 deletions cmd/opm/index/add.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"k8s.io/kubectl/pkg/util/templates"

"github.com/operator-framework/operator-registry/pkg/lib/indexer"
"github.com/operator-framework/operator-registry/pkg/registry"
)

var (
Expand Down Expand Up @@ -55,6 +56,7 @@ func addIndexAddCmd(parent *cobra.Command) {
indexCmd.Flags().StringP("container-tool", "c", "podman", "tool to interact with container images (save, build, etc.). One of: [docker, podman]")
indexCmd.Flags().StringP("tag", "t", "", "custom tag for container image being built")
indexCmd.Flags().Bool("permissive", false, "allow registry load errors")
indexCmd.Flags().StringP("mode", "", "replaces", "graph update mode that defines how channel graphs are updated. One of: [replaces, semver, semver-skippatch]")

if err := indexCmd.Flags().MarkHidden("debug"); err != nil {
logrus.Panic(err.Error())
Expand Down Expand Up @@ -107,6 +109,13 @@ func runIndexAddCmdFunc(cmd *cobra.Command, args []string) error {
return err
}

mode, err := cmd.Flags().GetString("mode")
if err != nil {
return err
}

modeEnum := registry.GetModeFromString(mode)

logger := logrus.WithFields(logrus.Fields{"bundles": bundles})

logger.Info("building the index")
Expand All @@ -121,6 +130,7 @@ func runIndexAddCmdFunc(cmd *cobra.Command, args []string) error {
Tag: tag,
Bundles: bundles,
Permissive: permissive,
Mode: modeEnum,
}

err = indexAdder.AddToIndex(request)
Expand Down
10 changes: 10 additions & 0 deletions cmd/opm/registry/add.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"github.com/spf13/cobra"

"github.com/operator-framework/operator-registry/pkg/lib/registry"
reg "github.com/operator-framework/operator-registry/pkg/registry"
)

func newRegistryAddCmd() *cobra.Command {
Expand All @@ -28,6 +29,7 @@ func newRegistryAddCmd() *cobra.Command {
rootCmd.Flags().StringSliceP("bundle-images", "b", []string{}, "comma separated list of links to bundle image")
rootCmd.Flags().Bool("permissive", false, "allow registry load errors")
rootCmd.Flags().Bool("skip-tls", false, "skip TLS certificate verification for container image registries while pulling bundles")
rootCmd.Flags().StringP("mode", "", "replaces", "graph update mode that defines how channel graphs are updated. One of: [replaces, semver, semver-skippatch]")

rootCmd.Flags().StringP("container-tool", "c", "", "")
if err := rootCmd.Flags().MarkDeprecated("container-tool", "ignored in favor of standalone image manipulation"); err != nil {
Expand Down Expand Up @@ -55,11 +57,19 @@ func addFunc(cmd *cobra.Command, args []string) error {
return err
}

mode, err := cmd.Flags().GetString("mode")
if err != nil {
return err
}

modeEnum := reg.GetModeFromString(mode)

request := registry.AddToRegistryRequest{
Permissive: permissive,
SkipTLS: skipTLS,
InputDatabase: fromFilename,
Bundles: bundleImages,
Mode: modeEnum,
}

logger := logrus.WithFields(logrus.Fields{"bundles": bundleImages})
Expand Down
2 changes: 1 addition & 1 deletion pkg/lib/bundle/validate.go
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ func (i imageValidator) ValidateBundleContent(manifestDir string) error {

// Validate the bundle object
if len(unstObjs) > 0 {
bundle := registry.NewBundle(csvName, "", "", unstObjs...)
bundle := registry.NewBundle(csvName, "", nil, unstObjs...)
bundleValidator := v.BundleValidator
results := bundleValidator.Validate(bundle)
if len(results) > 0 {
Expand Down
2 changes: 2 additions & 0 deletions pkg/lib/indexer/indexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ type AddToIndexRequest struct {
OutDockerfile string
Bundles []string
Tag string
Mode pregistry.Mode
}

// AddToIndex is an aggregate API used to generate a registry index image with additional bundles
Expand Down Expand Up @@ -89,6 +90,7 @@ func (i ImageIndexer) AddToIndex(request AddToIndexRequest) error {
Bundles: request.Bundles,
InputDatabase: databaseFile,
Permissive: request.Permissive,
Mode: request.Mode,
}

// Add the bundles to the registry
Expand Down
14 changes: 10 additions & 4 deletions pkg/lib/registry/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ type AddToRegistryRequest struct {
SkipTLS bool
InputDatabase string
Bundles []string
Mode registry.Mode
}

func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
Expand All @@ -44,6 +45,11 @@ func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
return err
}

graphLoader, err := sqlite.NewSQLGraphLoaderFromDB(db)
if err != nil {
return err
}

// TODO: Dependency inject the registry if we want to swap it out.
reg, destroy, err := containerdregistry.NewRegistry(
containerdregistry.SkipTLS(request.SkipTLS),
Expand All @@ -59,7 +65,7 @@ func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {

// TODO(njhale): Parallelize this once bundle add is commutative
for _, ref := range request.Bundles {
if err := populate(context.TODO(), dbLoader, reg, image.SimpleReference(ref)); err != nil {
if err := populate(context.TODO(), dbLoader, graphLoader, reg, image.SimpleReference(ref), request.Mode); err != nil {
err = fmt.Errorf("error loading bundle from image: %s", err)
if !request.Permissive {
r.Logger.WithError(err).Error("permissive mode disabled")
Expand All @@ -73,7 +79,7 @@ func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
return utilerrors.NewAggregate(errs) // nil if no errors
}

func populate(ctx context.Context, loader registry.Load, reg image.Registry, ref image.Reference) error {
func populate(ctx context.Context, loader registry.Load, graphLoader registry.GraphLoader, reg image.Registry, ref image.Reference, mode registry.Mode) error {
workingDir, err := ioutil.TempDir("./", "bundle_tmp")
if err != nil {
return err
Expand All @@ -88,9 +94,9 @@ func populate(ctx context.Context, loader registry.Load, reg image.Registry, ref
return err
}

populator := registry.NewDirectoryPopulator(loader, ref, workingDir)
populator := registry.NewDirectoryPopulator(loader, graphLoader, ref, workingDir)

return populator.Populate()
return populator.Populate(mode)
}

type DeleteFromRegistryRequest struct {
Expand Down
10 changes: 5 additions & 5 deletions pkg/registry/bundle.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,22 +31,22 @@ type Bundle struct {
Name string
Objects []*unstructured.Unstructured
Package string
Channel string
Channels []string
BundleImage string
csv *ClusterServiceVersion
crds []*v1beta1.CustomResourceDefinition
cacheStale bool
}

func NewBundle(name, pkgName, channelName string, objs ...*unstructured.Unstructured) *Bundle {
bundle := &Bundle{Name: name, Package: pkgName, Channel: channelName, cacheStale: false}
func NewBundle(name, pkgName string, channels []string, objs ...*unstructured.Unstructured) *Bundle {
bundle := &Bundle{Name: name, Package: pkgName, Channels: channels, cacheStale: false}
for _, o := range objs {
bundle.Add(o)
}
return bundle
}

func NewBundleFromStrings(name, pkgName, channelName string, objs []string) (*Bundle, error) {
func NewBundleFromStrings(name, pkgName string, channels []string, objs []string) (*Bundle, error) {
unstObjs := []*unstructured.Unstructured{}
for _, o := range objs {
dec := yaml.NewYAMLOrJSONDecoder(strings.NewReader(o), 10)
Expand All @@ -56,7 +56,7 @@ func NewBundleFromStrings(name, pkgName, channelName string, objs []string) (*Bu
}
unstObjs = append(unstObjs, unst)
}
return NewBundle(name, pkgName, channelName, unstObjs...), nil
return NewBundle(name, pkgName, channels, unstObjs...), nil
}

func (b *Bundle) Size() int {
Expand Down
147 changes: 147 additions & 0 deletions pkg/registry/bundlegraphloader.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
package registry

import (
"fmt"

"github.com/blang/semver"
)

// BundleGraphLoader generates updated graphs by adding bundles to them, updating
// the graph implicitly via semantic version of each bundle
type BundleGraphLoader struct {
}

// AddBundleToGraph takes a bundle and an existing graph and updates the graph to insert the new bundle
// into each channel it is included in
func (g *BundleGraphLoader) AddBundleToGraph(bundle *Bundle, graph *Package, newDefaultChannel string, skippatch bool) (*Package, error) {
bundleVersion, err := bundle.Version()
if err != nil {
return nil, fmt.Errorf("Unable to extract bundle version from bundle %s, can't insert in semver mode", bundle.BundleImage)
}

versionToAdd, err := semver.Make(bundleVersion)
if err != nil {
return nil, fmt.Errorf("Bundle version %s is not valid", bundleVersion)
}

newBundleKey := BundleKey{
CsvName: bundle.Name,
Version: versionToAdd.String(),
BundlePath: bundle.BundleImage,
}

// initialize the graph if it started empty
if graph.Name == "" {
graph.Name = bundle.Package
}
if newDefaultChannel != "" {
graph.DefaultChannel = newDefaultChannel
}

// generate the DAG for each channel the new bundle is being insert into
for _, channel := range bundle.Channels {
replaces := make(map[BundleKey]struct{}, 0)

// If the channel doesn't exist yet, initialize it
if !graph.HasChannel(channel) {
// create the channel and add a single node
newChannelGraph := Channel{
Head: newBundleKey,
Nodes: map[BundleKey]map[BundleKey]struct{}{
newBundleKey: nil,
},
}
if graph.Channels == nil {
graph.Channels = make(map[string]Channel, 1)
}
graph.Channels[channel] = newChannelGraph
continue
}

// find the version(s) it should sit between
channelGraph := graph.Channels[channel]
if channelGraph.Nodes == nil {
channelGraph.Nodes = make(map[BundleKey]map[BundleKey]struct{}, 1)
}

lowestAhead := BundleKey{}
greatestBehind := BundleKey{}
skipPatchCandidates := []BundleKey{}

// Iterate over existing nodes and compare the new node's version to find the
// lowest version above it and highest version below it (to insert between these nodes)
for node := range channelGraph.Nodes {
nodeVersion, err := semver.Make(node.Version)
if err != nil {
return nil, fmt.Errorf("Unable to parse existing bundle version stored in index %s %s %s",
node.CsvName, node.Version, node.BundlePath)
}

switch comparison := nodeVersion.Compare(versionToAdd); comparison {
case 0:
return nil, fmt.Errorf("Bundle version %s already added to index", bundleVersion)
case 1:
if lowestAhead.IsEmpty() {
lowestAhead = node
} else {
lowestAheadSemver, _ := semver.Make(lowestAhead.Version)
if nodeVersion.LT(lowestAheadSemver) {
lowestAhead = node
}
}
case -1:
if greatestBehind.IsEmpty() {
greatestBehind = node
} else {
greatestBehindSemver, _ := semver.Make(greatestBehind.Version)
if nodeVersion.GT(greatestBehindSemver) {
greatestBehind = node
}
}
}

// if skippatch mode is enabled, check each node to determine if z-updates should
// be replaced as well. Keep track of them to delete those nodes from the graph itself,
// just be aware of them for replacements
if skippatch {
if isSkipPatchCandidate(versionToAdd, nodeVersion) {
skipPatchCandidates = append(skipPatchCandidates, node)
replaces[node] = struct{}{}
}
}
}

// If we found a node behind the one we're adding, make the new node replace it
if !greatestBehind.IsEmpty() {
replaces[greatestBehind] = struct{}{}
}

// If we found a node ahead of the one we're adding, make the lowest to replace
// the new node. If we didn't find a node semantically ahead, the new node is
// the new channel head
if !lowestAhead.IsEmpty() {
channelGraph.Nodes[lowestAhead] = map[BundleKey]struct{}{
newBundleKey: struct{}{},
}
} else {
channelGraph.Head = newBundleKey
}

if skippatch {
// Remove the nodes that are now being skipped by a new patch version update
for _, candidate := range skipPatchCandidates {
delete(channelGraph.Nodes, candidate)
}
}

// add the node and update the graph
channelGraph.Nodes[newBundleKey] = replaces
graph.Channels[channel] = channelGraph
}

return graph, nil
}

func isSkipPatchCandidate(version, toCompare semver.Version) bool {
return (version.Major == toCompare.Major) && (version.Minor == toCompare.Minor) && (version.Patch > toCompare.Patch)
}

0 comments on commit bf35fab

Please sign in to comment.