Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[feature] Understanding packages from bundle directory #241

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
32 changes: 18 additions & 14 deletions cmd/opm/alpha/bundle/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,33 +44,37 @@ func newBundleBuildCmd() *cobra.Command {
RunE: buildFunc,
}

bundleBuildCmd.Flags().StringVarP(&dirBuildArgs, "directory", "d", "", "The directory where bundle manifests and metadata for a specific version are located")
bundleBuildCmd.Flags().StringVarP(&dirBuildArgs, "directory", "d", "",
"The directory where bundle manifests and metadata for a specific version are located")
if err := bundleBuildCmd.MarkFlagRequired("directory"); err != nil {
log.Fatalf("Failed to mark `directory` flag for `build` subcommand as required")
}

bundleBuildCmd.Flags().StringVarP(&tagBuildArgs, "tag", "t", "", "The image tag applied to the bundle image")
bundleBuildCmd.Flags().StringVarP(&tagBuildArgs, "tag", "t", "",
"The image tag applied to the bundle image")
if err := bundleBuildCmd.MarkFlagRequired("tag"); err != nil {
log.Fatalf("Failed to mark `tag` flag for `build` subcommand as required")
}

bundleBuildCmd.Flags().StringVarP(&packageNameArgs, "package", "p", "", "The name of the package that bundle image belongs to")
if err := bundleBuildCmd.MarkFlagRequired("package"); err != nil {
log.Fatalf("Failed to mark `package` flag for `build` subcommand as required")
}
bundleBuildCmd.Flags().StringVarP(&packageNameArgs, "package", "p", "",
"The name of the package that bundle image belongs to "+
"(Required if `directory` is not pointing to a bundle in the nested bundle format)")

bundleBuildCmd.Flags().StringVarP(&channelsArgs, "channels", "c", "", "The list of channels that bundle image belongs to")
if err := bundleBuildCmd.MarkFlagRequired("channels"); err != nil {
log.Fatalf("Failed to mark `channels` flag for `build` subcommand as required")
}
bundleBuildCmd.Flags().StringVarP(&channelsArgs, "channels", "c", "",
"The list of channels that bundle image belongs to"+
"(Required if `directory` is not pointing to a bundle in the nested bundle format)")

bundleBuildCmd.Flags().StringVarP(&imageBuilderArgs, "image-builder", "b", "docker", "Tool to build container images. One of: [docker, podman, buildah]")
bundleBuildCmd.Flags().StringVarP(&imageBuilderArgs, "image-builder", "b", "docker",
"Tool to build container images. One of: [docker, podman, buildah]")

bundleBuildCmd.Flags().StringVarP(&channelDefaultArgs, "default", "e", "", "The default channel for the bundle image")
bundleBuildCmd.Flags().StringVarP(&channelDefaultArgs, "default", "e", "",
"The default channel for the bundle image")

bundleBuildCmd.Flags().BoolVarP(&overwriteArgs, "overwrite", "o", false, "To overwrite annotations.yaml locally if existed. By default, overwrite is set to `false`.")
bundleBuildCmd.Flags().BoolVarP(&overwriteArgs, "overwrite", "o", false,
"To overwrite annotations.yaml locally if existed. By default, overwrite is set to `false`.")

bundleBuildCmd.Flags().StringVarP(&outputDirArgs, "output-dir", "u", "", "Optional output directory for operator manifests")
bundleBuildCmd.Flags().StringVarP(&outputDirArgs, "output-dir", "u", "",
"Optional output directory for operator manifests")

return bundleBuildCmd
}
Expand Down
23 changes: 12 additions & 11 deletions cmd/opm/alpha/bundle/generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,24 +24,25 @@ func newBundleGenerateCmd() *cobra.Command {
RunE: generateFunc,
}

bundleGenerateCmd.Flags().StringVarP(&dirBuildArgs, "directory", "d", "", "The directory where bundle manifests for a specific version are located.")
bundleGenerateCmd.Flags().StringVarP(&dirBuildArgs, "directory", "d", "",
"The directory where bundle manifests for a specific version are located.")
if err := bundleGenerateCmd.MarkFlagRequired("directory"); err != nil {
log.Fatalf("Failed to mark `directory` flag for `generate` subcommand as required")
}

bundleGenerateCmd.Flags().StringVarP(&packageNameArgs, "package", "p", "", "The name of the package that bundle image belongs to")
if err := bundleGenerateCmd.MarkFlagRequired("package"); err != nil {
log.Fatalf("Failed to mark `package` flag for `generate` subcommand as required")
}
bundleGenerateCmd.Flags().StringVarP(&packageNameArgs, "package", "p", "",
"The name of the package that bundle image belongs to "+
"(Required if `directory` is not pointing to a bundle in the nested bundle format)")

bundleGenerateCmd.Flags().StringVarP(&channelsArgs, "channels", "c", "", "The list of channels that bundle image belongs to")
if err := bundleGenerateCmd.MarkFlagRequired("channels"); err != nil {
log.Fatalf("Failed to mark `channels` flag for `generate` subcommand as required")
}
bundleGenerateCmd.Flags().StringVarP(&channelsArgs, "channels", "c", "",
"The list of channels that bundle image belongs to"+
"(Required if `directory` is not pointing to a bundle in the nested bundle format)")

bundleGenerateCmd.Flags().StringVarP(&channelDefaultArgs, "default", "e", "", "The default channel for the bundle image")
bundleGenerateCmd.Flags().StringVarP(&channelDefaultArgs, "default", "e", "",
"The default channel for the bundle image")

bundleGenerateCmd.Flags().StringVarP(&outputDirArgs, "output-dir", "u", "", "Optional output directory for operator manifests")
bundleGenerateCmd.Flags().StringVarP(&outputDirArgs, "output-dir", "u", "",
"Optional output directory for operator manifests")

return bundleGenerateCmd
}
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6 h1:uZuxRZCz65cG1o6K/xUqImNcYKtmk9ylqaH0itMSvzA=
Expand Down Expand Up @@ -389,6 +391,7 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
Expand Down Expand Up @@ -598,6 +601,7 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
Expand Down
3 changes: 2 additions & 1 deletion pkg/lib/bundle/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ func ExecuteCommand(cmd *exec.Cmd) error {
// @channels: The list of channels that bundle image belongs to
// @channelDefault: The default channel for the bundle image
// @overwrite: Boolean flag to enable overwriting annotations.yaml locally if existed
func BuildFunc(directory, outputDir, imageTag, imageBuilder, packageName, channels, channelDefault string, overwrite bool) error {
func BuildFunc(directory, outputDir, imageTag, imageBuilder, packageName, channels, channelDefault string,
overwrite bool) error {
_, err := os.Stat(directory)
if os.IsNotExist(err) {
return err
Expand Down
39 changes: 39 additions & 0 deletions pkg/lib/bundle/generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,45 @@ func GenerateFunc(directory, outputDir, packageName, channels, channelDefault st
return err
}

// Channels and packageName are required fields where as default channel is automatically filled if unspecified
// and that either of the required field is missing. We are interpreting the bundle information through
// bundle directory embedded in the package folder.
if channels == "" || packageName == "" {
var notProvided []string
if channels == "" {
notProvided = append(notProvided, "channels")
}
if packageName == "" {
notProvided = append(notProvided, "package name")
}
log.Infof("Bundle %s information not provided, inferring from parent package directory",
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

log/notification added at the info level to suggestion inference took place.

strings.Join(notProvided, " and "))

i, err := NewBundleDirInterperter(directory)
if err != nil {
return fmt.Errorf("please manually input channels and packageName, "+
"error interpreting bundle from directory %s, %v", directory, err)
}

if channels == "" {
channels = strings.Join(i.GetBundleChannels(), ",")
if channels == "" {
return fmt.Errorf("error interpreting channels, please manually input channels instead")
}
log.Infof("Inferred channels: %s", channels)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The inference results are also posted.

}

if packageName == "" {
packageName = i.GetPackageName()
log.Infof("Inferred package name: %s", packageName)
}

if channelDefault == "" {
channelDefault = i.GetDefaultChannel()
log.Infof("Inferred default channel: %s", channelDefault)
}
}

log.Info("Building annotations.yaml")

// Generate annotations.yaml
Expand Down
54 changes: 54 additions & 0 deletions pkg/lib/bundle/interpreter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package bundle

import (
"fmt"
"path"
"sort"

"github.com/operator-framework/operator-registry/pkg/registry"
)

type bundleDirInterpreter struct {
bundleCsvName string
pkg *registry.Package
}

func NewBundleDirInterperter(bundleDir string) (*bundleDirInterpreter, error) {
csv, err := registry.ReadCSVFromBundleDirectory(bundleDir)
if err != nil {
return nil, fmt.Errorf("error loading CSV from bundle directory, %v", err)
}

pkgDir, err := registry.NewPackageGraphLoaderFromDir(path.Join(bundleDir, ".."))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure this is what we want to do here.

This reads in the directory of files and generates a graph, and tells you if there are problems.

But the generate command is used for a single bundle. That single bundle may be applied to any target index, and has nothing to do with the bundles that happen to be in in the filesystem next to it.

I think this would be fine without the graph loader in this PR at all. Thoughts?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But I thought the whole purpose of this though is to automagically decide what channels a given bundle is. How can we do that without attempting to build the graph for a given package?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same ask!

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the explanation - commented before ☕️ !

I think there's a slight impedance mismatch here - this option is now only really useful if you're attempting convert an entire directory of the old format, right?

Should this really be part of the normal command for generating/buliding single bundles when it's so tied to the set?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the trigger for it should be "if you didn't pass in these parameters explicitly, let's go see if we can find them for you" rather than it being an explicit opt in/opt out feature. IMO that's a better experience

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I second that this should be an experience enhancement type of feature based on the fact that you have the full-blown format with bundles nested in a package along with other bundles. Without it, it does not have any information we would be looking for. The default channel and channels are only obtainable in the *.package.yaml file. To figure out which channels a bundle belongs to require relationships of this bundle with respect to every channel heads and see if it is directly/transitively replaced by those channel heads to be included as part of that channel. Without the package and rest of the bundles, all we can be certain from a single bundle is the package name.

After all, this feature is just auto-filling package name, channels, and default channels for a bundle.

if err != nil {
return nil, fmt.Errorf("error loading package from directory, %v", err)
}

p, err := pkgDir.Generate()
if err != nil {
return nil, err
}

return &bundleDirInterpreter{bundleCsvName: csv.GetName(), pkg: p}, nil
}

func (b *bundleDirInterpreter) GetBundleChannels() (channelNames []string) {
for channelName, channel := range b.pkg.Channels {
for bundle, _ := range channel.Nodes {
if bundle.CsvName == b.bundleCsvName {
channelNames = append(channelNames, channelName)
break
}
}
}
sort.Strings(channelNames)
return
}

func (b *bundleDirInterpreter) GetDefaultChannel() string {
return b.pkg.DefaultChannel
}

func (b *bundleDirInterpreter) GetPackageName() string {
return b.pkg.Name
}
68 changes: 68 additions & 0 deletions pkg/lib/bundle/interpreter_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package bundle

import (
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestBundleDirectoryInterpreter(t *testing.T) {
tests := []struct {
dir string
fail bool
bundleChannels []string
defaultChannel string
PackageName string
}{
{
dir: "../../registry/testdata/validPackages/etcd/0.6.1",
fail: false,
bundleChannels: []string{"alpha", "beta", "stable"},
defaultChannel: "alpha",
PackageName: "etcd",
},
{
dir: "../../registry/testdata/validPackages/etcd/0.9.0",
fail: false,
bundleChannels: []string{"alpha", "beta", "stable"},
defaultChannel: "alpha",
PackageName: "etcd",
},
{
dir: "../../registry/testdata/validPackages/etcd/0.9.2",
fail: false,
bundleChannels: []string{"alpha", "stable"},
defaultChannel: "alpha",
PackageName: "etcd",
},
{
dir: "../../registry/testdata/validPackages/prometheus/0.14.0",
fail: false,
bundleChannels: []string{"preview"},
defaultChannel: "preview",
PackageName: "prometheus",
},
{
dir: "../../registry/testdata/invalidPackges/3scale-community-operator/0.3.0",
fail: true,
},
}

for _, tt := range tests {
t.Run("Loading Package Graph from "+tt.dir, func(t *testing.T) {
bundle, err := NewBundleDirInterperter(tt.dir)
if tt.fail {
assert.Error(t, err)
return
}
require.NoError(t, err)

assert.EqualValues(t, tt.bundleChannels, bundle.GetBundleChannels())

assert.EqualValues(t, tt.defaultChannel, bundle.GetDefaultChannel())

assert.EqualValues(t, tt.PackageName, bundle.GetPackageName())
})
}
}
53 changes: 53 additions & 0 deletions pkg/registry/csv.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,18 @@ package registry

import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path"

v1 "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/yaml"

"github.com/operator-framework/api/pkg/operators"
)

const (
Expand Down Expand Up @@ -53,6 +62,50 @@ type ClusterServiceVersion struct {
Spec json.RawMessage `json:"spec"`
}

// ReadCSVFromBundleDirectory tries to parse every YAML file in the directory and see if they are CSV.
// According to the strict one CSV rule for every bundle, we return the first file that is considered a CSV type.
func ReadCSVFromBundleDirectory(bundleDir string) (*ClusterServiceVersion, error) {
dirContent, err := ioutil.ReadDir(bundleDir)
if err != nil {
return nil, fmt.Errorf("error reading bundle directory %s, %v", bundleDir, err)
}

files := []string{}
for _, f := range dirContent {
if !f.IsDir() {
files = append(files, f.Name())
}
}

for _, file := range files {
yamlReader, err := os.Open(path.Join(bundleDir, file))
if err != nil {
continue
}

unstructuredCSV := unstructured.Unstructured{}
csv := ClusterServiceVersion{}

decoder := yaml.NewYAMLOrJSONDecoder(yamlReader, 30)
if err = decoder.Decode(&unstructuredCSV); err != nil {
continue
}

if unstructuredCSV.GetKind() != operators.ClusterServiceVersionKind {
continue
}

if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredCSV.UnstructuredContent(),
&csv); err != nil {
return nil, err
}

return &csv, nil
}
return nil, fmt.Errorf("no ClusterServiceVersion object found in %s", bundleDir)

}

// GetReplaces returns the name of the older ClusterServiceVersion object that
// is replaced by this ClusterServiceVersion object.
//
Expand Down