Skip to content

Commit

Permalink
Merge pull request #247 from kevinrizza/semver-mode
Browse files Browse the repository at this point in the history
Semver index insert
  • Loading branch information
openshift-merge-robot committed Apr 7, 2020
2 parents 09daf8d + bce843d commit cac3168
Show file tree
Hide file tree
Showing 22 changed files with 1,004 additions and 65 deletions.
13 changes: 13 additions & 0 deletions cmd/opm/index/add.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"k8s.io/kubectl/pkg/util/templates"

"github.com/operator-framework/operator-registry/pkg/lib/indexer"
"github.com/operator-framework/operator-registry/pkg/registry"
)

var (
Expand Down Expand Up @@ -55,6 +56,7 @@ func addIndexAddCmd(parent *cobra.Command) {
indexCmd.Flags().StringP("container-tool", "c", "podman", "tool to interact with container images (save, build, etc.). One of: [docker, podman]")
indexCmd.Flags().StringP("tag", "t", "", "custom tag for container image being built")
indexCmd.Flags().Bool("permissive", false, "allow registry load errors")
indexCmd.Flags().StringP("mode", "", "replaces", "graph update mode that defines how channel graphs are updated. One of: [replaces, semver, semver-skippatch]")

if err := indexCmd.Flags().MarkHidden("debug"); err != nil {
logrus.Panic(err.Error())
Expand Down Expand Up @@ -107,6 +109,16 @@ func runIndexAddCmdFunc(cmd *cobra.Command, args []string) error {
return err
}

mode, err := cmd.Flags().GetString("mode")
if err != nil {
return err
}

modeEnum, err := registry.GetModeFromString(mode)
if err != nil {
return err
}

logger := logrus.WithFields(logrus.Fields{"bundles": bundles})

logger.Info("building the index")
Expand All @@ -121,6 +133,7 @@ func runIndexAddCmdFunc(cmd *cobra.Command, args []string) error {
Tag: tag,
Bundles: bundles,
Permissive: permissive,
Mode: modeEnum,
}

err = indexAdder.AddToIndex(request)
Expand Down
13 changes: 13 additions & 0 deletions cmd/opm/registry/add.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"github.com/spf13/cobra"

"github.com/operator-framework/operator-registry/pkg/lib/registry"
reg "github.com/operator-framework/operator-registry/pkg/registry"
)

func newRegistryAddCmd() *cobra.Command {
Expand All @@ -28,6 +29,7 @@ func newRegistryAddCmd() *cobra.Command {
rootCmd.Flags().StringSliceP("bundle-images", "b", []string{}, "comma separated list of links to bundle image")
rootCmd.Flags().Bool("permissive", false, "allow registry load errors")
rootCmd.Flags().Bool("skip-tls", false, "skip TLS certificate verification for container image registries while pulling bundles")
rootCmd.Flags().StringP("mode", "", "replaces", "graph update mode that defines how channel graphs are updated. One of: [replaces, semver, semver-skippatch]")

rootCmd.Flags().StringP("container-tool", "c", "", "")
if err := rootCmd.Flags().MarkDeprecated("container-tool", "ignored in favor of standalone image manipulation"); err != nil {
Expand Down Expand Up @@ -55,11 +57,22 @@ func addFunc(cmd *cobra.Command, args []string) error {
return err
}

mode, err := cmd.Flags().GetString("mode")
if err != nil {
return err
}

modeEnum, err := reg.GetModeFromString(mode)
if err != nil {
return err
}

request := registry.AddToRegistryRequest{
Permissive: permissive,
SkipTLS: skipTLS,
InputDatabase: fromFilename,
Bundles: bundleImages,
Mode: modeEnum,
}

logger := logrus.WithFields(logrus.Fields{"bundles": bundleImages})
Expand Down
2 changes: 1 addition & 1 deletion pkg/lib/bundle/validate.go
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ func (i imageValidator) ValidateBundleContent(manifestDir string) error {

// Validate the bundle object
if len(unstObjs) > 0 {
bundle := registry.NewBundle(csvName, "", "", unstObjs...)
bundle := registry.NewBundle(csvName, "", nil, unstObjs...)
bundleValidator := v.BundleValidator
results := bundleValidator.Validate(bundle)
if len(results) > 0 {
Expand Down
2 changes: 2 additions & 0 deletions pkg/lib/indexer/indexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ type AddToIndexRequest struct {
OutDockerfile string
Bundles []string
Tag string
Mode pregistry.Mode
}

// AddToIndex is an aggregate API used to generate a registry index image with additional bundles
Expand Down Expand Up @@ -89,6 +90,7 @@ func (i ImageIndexer) AddToIndex(request AddToIndexRequest) error {
Bundles: request.Bundles,
InputDatabase: databaseFile,
Permissive: request.Permissive,
Mode: request.Mode,
}

// Add the bundles to the registry
Expand Down
14 changes: 10 additions & 4 deletions pkg/lib/registry/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ type AddToRegistryRequest struct {
SkipTLS bool
InputDatabase string
Bundles []string
Mode registry.Mode
}

func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
Expand All @@ -44,6 +45,11 @@ func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
return err
}

graphLoader, err := sqlite.NewSQLGraphLoaderFromDB(db)
if err != nil {
return err
}

// TODO: Dependency inject the registry if we want to swap it out.
reg, destroy, err := containerdregistry.NewRegistry(
containerdregistry.SkipTLS(request.SkipTLS),
Expand All @@ -59,7 +65,7 @@ func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {

// TODO(njhale): Parallelize this once bundle add is commutative
for _, ref := range request.Bundles {
if err := populate(context.TODO(), dbLoader, reg, image.SimpleReference(ref)); err != nil {
if err := populate(context.TODO(), dbLoader, graphLoader, reg, image.SimpleReference(ref), request.Mode); err != nil {
err = fmt.Errorf("error loading bundle from image: %s", err)
if !request.Permissive {
r.Logger.WithError(err).Error("permissive mode disabled")
Expand All @@ -73,7 +79,7 @@ func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
return utilerrors.NewAggregate(errs) // nil if no errors
}

func populate(ctx context.Context, loader registry.Load, reg image.Registry, ref image.Reference) error {
func populate(ctx context.Context, loader registry.Load, graphLoader registry.GraphLoader, reg image.Registry, ref image.Reference, mode registry.Mode) error {
workingDir, err := ioutil.TempDir("./", "bundle_tmp")
if err != nil {
return err
Expand All @@ -88,9 +94,9 @@ func populate(ctx context.Context, loader registry.Load, reg image.Registry, ref
return err
}

populator := registry.NewDirectoryPopulator(loader, ref, workingDir)
populator := registry.NewDirectoryPopulator(loader, graphLoader, ref, workingDir)

return populator.Populate()
return populator.Populate(mode)
}

type DeleteFromRegistryRequest struct {
Expand Down
10 changes: 5 additions & 5 deletions pkg/registry/bundle.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,22 +31,22 @@ type Bundle struct {
Name string
Objects []*unstructured.Unstructured
Package string
Channel string
Channels []string
BundleImage string
csv *ClusterServiceVersion
crds []*v1beta1.CustomResourceDefinition
cacheStale bool
}

func NewBundle(name, pkgName, channelName string, objs ...*unstructured.Unstructured) *Bundle {
bundle := &Bundle{Name: name, Package: pkgName, Channel: channelName, cacheStale: false}
func NewBundle(name, pkgName string, channels []string, objs ...*unstructured.Unstructured) *Bundle {
bundle := &Bundle{Name: name, Package: pkgName, Channels: channels, cacheStale: false}
for _, o := range objs {
bundle.Add(o)
}
return bundle
}

func NewBundleFromStrings(name, pkgName, channelName string, objs []string) (*Bundle, error) {
func NewBundleFromStrings(name, pkgName string, channels []string, objs []string) (*Bundle, error) {
unstObjs := []*unstructured.Unstructured{}
for _, o := range objs {
dec := yaml.NewYAMLOrJSONDecoder(strings.NewReader(o), 10)
Expand All @@ -56,7 +56,7 @@ func NewBundleFromStrings(name, pkgName, channelName string, objs []string) (*Bu
}
unstObjs = append(unstObjs, unst)
}
return NewBundle(name, pkgName, channelName, unstObjs...), nil
return NewBundle(name, pkgName, channels, unstObjs...), nil
}

func (b *Bundle) Size() int {
Expand Down
147 changes: 147 additions & 0 deletions pkg/registry/bundlegraphloader.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
package registry

import (
"fmt"

"github.com/blang/semver"
)

// BundleGraphLoader generates updated graphs by adding bundles to them, updating
// the graph implicitly via semantic version of each bundle
type BundleGraphLoader struct {
}

// AddBundleToGraph takes a bundle and an existing graph and updates the graph to insert the new bundle
// into each channel it is included in
func (g *BundleGraphLoader) AddBundleToGraph(bundle *Bundle, graph *Package, newDefaultChannel string, skippatch bool) (*Package, error) {
bundleVersion, err := bundle.Version()
if err != nil {
return nil, fmt.Errorf("Unable to extract bundle version from bundle %s, can't insert in semver mode", bundle.BundleImage)
}

versionToAdd, err := semver.Make(bundleVersion)
if err != nil {
return nil, fmt.Errorf("Bundle version %s is not valid", bundleVersion)
}

newBundleKey := BundleKey{
CsvName: bundle.Name,
Version: versionToAdd.String(),
BundlePath: bundle.BundleImage,
}

// initialize the graph if it started empty
if graph.Name == "" {
graph.Name = bundle.Package
}
if newDefaultChannel != "" {
graph.DefaultChannel = newDefaultChannel
}

// generate the DAG for each channel the new bundle is being insert into
for _, channel := range bundle.Channels {
replaces := make(map[BundleKey]struct{}, 0)

// If the channel doesn't exist yet, initialize it
if !graph.HasChannel(channel) {
// create the channel and add a single node
newChannelGraph := Channel{
Head: newBundleKey,
Nodes: map[BundleKey]map[BundleKey]struct{}{
newBundleKey: nil,
},
}
if graph.Channels == nil {
graph.Channels = make(map[string]Channel, 1)
}
graph.Channels[channel] = newChannelGraph
continue
}

// find the version(s) it should sit between
channelGraph := graph.Channels[channel]
if channelGraph.Nodes == nil {
channelGraph.Nodes = make(map[BundleKey]map[BundleKey]struct{}, 1)
}

lowestAhead := BundleKey{}
greatestBehind := BundleKey{}
skipPatchCandidates := []BundleKey{}

// Iterate over existing nodes and compare the new node's version to find the
// lowest version above it and highest version below it (to insert between these nodes)
for node := range channelGraph.Nodes {
nodeVersion, err := semver.Make(node.Version)
if err != nil {
return nil, fmt.Errorf("Unable to parse existing bundle version stored in index %s %s %s",
node.CsvName, node.Version, node.BundlePath)
}

switch comparison := nodeVersion.Compare(versionToAdd); comparison {
case 0:
return nil, fmt.Errorf("Bundle version %s already added to index", bundleVersion)
case 1:
if lowestAhead.IsEmpty() {
lowestAhead = node
} else {
lowestAheadSemver, _ := semver.Make(lowestAhead.Version)
if nodeVersion.LT(lowestAheadSemver) {
lowestAhead = node
}
}
case -1:
if greatestBehind.IsEmpty() {
greatestBehind = node
} else {
greatestBehindSemver, _ := semver.Make(greatestBehind.Version)
if nodeVersion.GT(greatestBehindSemver) {
greatestBehind = node
}
}
}

// if skippatch mode is enabled, check each node to determine if z-updates should
// be replaced as well. Keep track of them to delete those nodes from the graph itself,
// just be aware of them for replacements
if skippatch {
if isSkipPatchCandidate(versionToAdd, nodeVersion) {
skipPatchCandidates = append(skipPatchCandidates, node)
replaces[node] = struct{}{}
}
}
}

// If we found a node behind the one we're adding, make the new node replace it
if !greatestBehind.IsEmpty() {
replaces[greatestBehind] = struct{}{}
}

// If we found a node ahead of the one we're adding, make the lowest to replace
// the new node. If we didn't find a node semantically ahead, the new node is
// the new channel head
if !lowestAhead.IsEmpty() {
channelGraph.Nodes[lowestAhead] = map[BundleKey]struct{}{
newBundleKey: struct{}{},
}
} else {
channelGraph.Head = newBundleKey
}

if skippatch {
// Remove the nodes that are now being skipped by a new patch version update
for _, candidate := range skipPatchCandidates {
delete(channelGraph.Nodes, candidate)
}
}

// add the node and update the graph
channelGraph.Nodes[newBundleKey] = replaces
graph.Channels[channel] = channelGraph
}

return graph, nil
}

func isSkipPatchCandidate(version, toCompare semver.Version) bool {
return (version.Major == toCompare.Major) && (version.Minor == toCompare.Minor) && (version.Patch > toCompare.Patch)
}

0 comments on commit cac3168

Please sign in to comment.