Skip to content

Commit

Permalink
Merge pull request #241 from Bowenislandsong/understand-packages
Browse files Browse the repository at this point in the history
[feature] Understanding packages from bundle directory
  • Loading branch information
openshift-merge-robot committed Apr 1, 2020
2 parents 1186793 + 99fc008 commit d5f8836
Show file tree
Hide file tree
Showing 49 changed files with 20,168 additions and 26 deletions.
32 changes: 18 additions & 14 deletions cmd/opm/alpha/bundle/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,33 +44,37 @@ func newBundleBuildCmd() *cobra.Command {
RunE: buildFunc,
}

bundleBuildCmd.Flags().StringVarP(&dirBuildArgs, "directory", "d", "", "The directory where bundle manifests and metadata for a specific version are located")
bundleBuildCmd.Flags().StringVarP(&dirBuildArgs, "directory", "d", "",
"The directory where bundle manifests and metadata for a specific version are located")
if err := bundleBuildCmd.MarkFlagRequired("directory"); err != nil {
log.Fatalf("Failed to mark `directory` flag for `build` subcommand as required")
}

bundleBuildCmd.Flags().StringVarP(&tagBuildArgs, "tag", "t", "", "The image tag applied to the bundle image")
bundleBuildCmd.Flags().StringVarP(&tagBuildArgs, "tag", "t", "",
"The image tag applied to the bundle image")
if err := bundleBuildCmd.MarkFlagRequired("tag"); err != nil {
log.Fatalf("Failed to mark `tag` flag for `build` subcommand as required")
}

bundleBuildCmd.Flags().StringVarP(&packageNameArgs, "package", "p", "", "The name of the package that bundle image belongs to")
if err := bundleBuildCmd.MarkFlagRequired("package"); err != nil {
log.Fatalf("Failed to mark `package` flag for `build` subcommand as required")
}
bundleBuildCmd.Flags().StringVarP(&packageNameArgs, "package", "p", "",
"The name of the package that bundle image belongs to "+
"(Required if `directory` is not pointing to a bundle in the nested bundle format)")

bundleBuildCmd.Flags().StringVarP(&channelsArgs, "channels", "c", "", "The list of channels that bundle image belongs to")
if err := bundleBuildCmd.MarkFlagRequired("channels"); err != nil {
log.Fatalf("Failed to mark `channels` flag for `build` subcommand as required")
}
bundleBuildCmd.Flags().StringVarP(&channelsArgs, "channels", "c", "",
"The list of channels that bundle image belongs to"+
"(Required if `directory` is not pointing to a bundle in the nested bundle format)")

bundleBuildCmd.Flags().StringVarP(&imageBuilderArgs, "image-builder", "b", "docker", "Tool to build container images. One of: [docker, podman, buildah]")
bundleBuildCmd.Flags().StringVarP(&imageBuilderArgs, "image-builder", "b", "docker",
"Tool to build container images. One of: [docker, podman, buildah]")

bundleBuildCmd.Flags().StringVarP(&channelDefaultArgs, "default", "e", "", "The default channel for the bundle image")
bundleBuildCmd.Flags().StringVarP(&channelDefaultArgs, "default", "e", "",
"The default channel for the bundle image")

bundleBuildCmd.Flags().BoolVarP(&overwriteArgs, "overwrite", "o", false, "To overwrite annotations.yaml locally if existed. By default, overwrite is set to `false`.")
bundleBuildCmd.Flags().BoolVarP(&overwriteArgs, "overwrite", "o", false,
"To overwrite annotations.yaml locally if existed. By default, overwrite is set to `false`.")

bundleBuildCmd.Flags().StringVarP(&outputDirArgs, "output-dir", "u", "", "Optional output directory for operator manifests")
bundleBuildCmd.Flags().StringVarP(&outputDirArgs, "output-dir", "u", "",
"Optional output directory for operator manifests")

return bundleBuildCmd
}
Expand Down
23 changes: 12 additions & 11 deletions cmd/opm/alpha/bundle/generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,24 +24,25 @@ func newBundleGenerateCmd() *cobra.Command {
RunE: generateFunc,
}

bundleGenerateCmd.Flags().StringVarP(&dirBuildArgs, "directory", "d", "", "The directory where bundle manifests for a specific version are located.")
bundleGenerateCmd.Flags().StringVarP(&dirBuildArgs, "directory", "d", "",
"The directory where bundle manifests for a specific version are located.")
if err := bundleGenerateCmd.MarkFlagRequired("directory"); err != nil {
log.Fatalf("Failed to mark `directory` flag for `generate` subcommand as required")
}

bundleGenerateCmd.Flags().StringVarP(&packageNameArgs, "package", "p", "", "The name of the package that bundle image belongs to")
if err := bundleGenerateCmd.MarkFlagRequired("package"); err != nil {
log.Fatalf("Failed to mark `package` flag for `generate` subcommand as required")
}
bundleGenerateCmd.Flags().StringVarP(&packageNameArgs, "package", "p", "",
"The name of the package that bundle image belongs to "+
"(Required if `directory` is not pointing to a bundle in the nested bundle format)")

bundleGenerateCmd.Flags().StringVarP(&channelsArgs, "channels", "c", "", "The list of channels that bundle image belongs to")
if err := bundleGenerateCmd.MarkFlagRequired("channels"); err != nil {
log.Fatalf("Failed to mark `channels` flag for `generate` subcommand as required")
}
bundleGenerateCmd.Flags().StringVarP(&channelsArgs, "channels", "c", "",
"The list of channels that bundle image belongs to"+
"(Required if `directory` is not pointing to a bundle in the nested bundle format)")

bundleGenerateCmd.Flags().StringVarP(&channelDefaultArgs, "default", "e", "", "The default channel for the bundle image")
bundleGenerateCmd.Flags().StringVarP(&channelDefaultArgs, "default", "e", "",
"The default channel for the bundle image")

bundleGenerateCmd.Flags().StringVarP(&outputDirArgs, "output-dir", "u", "", "Optional output directory for operator manifests")
bundleGenerateCmd.Flags().StringVarP(&outputDirArgs, "output-dir", "u", "",
"Optional output directory for operator manifests")

return bundleGenerateCmd
}
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6 h1:uZuxRZCz65cG1o6K/xUqImNcYKtmk9ylqaH0itMSvzA=
Expand Down Expand Up @@ -389,6 +391,7 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
Expand Down Expand Up @@ -598,6 +601,7 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
Expand Down
3 changes: 2 additions & 1 deletion pkg/lib/bundle/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ func ExecuteCommand(cmd *exec.Cmd) error {
// @channels: The list of channels that bundle image belongs to
// @channelDefault: The default channel for the bundle image
// @overwrite: Boolean flag to enable overwriting annotations.yaml locally if existed
func BuildFunc(directory, outputDir, imageTag, imageBuilder, packageName, channels, channelDefault string, overwrite bool) error {
func BuildFunc(directory, outputDir, imageTag, imageBuilder, packageName, channels, channelDefault string,
overwrite bool) error {
_, err := os.Stat(directory)
if os.IsNotExist(err) {
return err
Expand Down
39 changes: 39 additions & 0 deletions pkg/lib/bundle/generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,45 @@ func GenerateFunc(directory, outputDir, packageName, channels, channelDefault st
return err
}

// Channels and packageName are required fields where as default channel is automatically filled if unspecified
// and that either of the required field is missing. We are interpreting the bundle information through
// bundle directory embedded in the package folder.
if channels == "" || packageName == "" {
var notProvided []string
if channels == "" {
notProvided = append(notProvided, "channels")
}
if packageName == "" {
notProvided = append(notProvided, "package name")
}
log.Infof("Bundle %s information not provided, inferring from parent package directory",
strings.Join(notProvided, " and "))

i, err := NewBundleDirInterperter(directory)
if err != nil {
return fmt.Errorf("please manually input channels and packageName, "+
"error interpreting bundle from directory %s, %v", directory, err)
}

if channels == "" {
channels = strings.Join(i.GetBundleChannels(), ",")
if channels == "" {
return fmt.Errorf("error interpreting channels, please manually input channels instead")
}
log.Infof("Inferred channels: %s", channels)
}

if packageName == "" {
packageName = i.GetPackageName()
log.Infof("Inferred package name: %s", packageName)
}

if channelDefault == "" {
channelDefault = i.GetDefaultChannel()
log.Infof("Inferred default channel: %s", channelDefault)
}
}

log.Info("Building annotations.yaml")

// Generate annotations.yaml
Expand Down
54 changes: 54 additions & 0 deletions pkg/lib/bundle/interpreter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package bundle

import (
"fmt"
"path"
"sort"

"github.com/operator-framework/operator-registry/pkg/registry"
)

type bundleDirInterpreter struct {
bundleCsvName string
pkg *registry.Package
}

func NewBundleDirInterperter(bundleDir string) (*bundleDirInterpreter, error) {
csv, err := registry.ReadCSVFromBundleDirectory(bundleDir)
if err != nil {
return nil, fmt.Errorf("error loading CSV from bundle directory, %v", err)
}

pkgDir, err := registry.NewPackageGraphLoaderFromDir(path.Join(bundleDir, ".."))
if err != nil {
return nil, fmt.Errorf("error loading package from directory, %v", err)
}

p, err := pkgDir.Generate()
if err != nil {
return nil, err
}

return &bundleDirInterpreter{bundleCsvName: csv.GetName(), pkg: p}, nil
}

func (b *bundleDirInterpreter) GetBundleChannels() (channelNames []string) {
for channelName, channel := range b.pkg.Channels {
for bundle, _ := range channel.Nodes {
if bundle.CsvName == b.bundleCsvName {
channelNames = append(channelNames, channelName)
break
}
}
}
sort.Strings(channelNames)
return
}

func (b *bundleDirInterpreter) GetDefaultChannel() string {
return b.pkg.DefaultChannel
}

func (b *bundleDirInterpreter) GetPackageName() string {
return b.pkg.Name
}
68 changes: 68 additions & 0 deletions pkg/lib/bundle/interpreter_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package bundle

import (
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestBundleDirectoryInterpreter(t *testing.T) {
tests := []struct {
dir string
fail bool
bundleChannels []string
defaultChannel string
PackageName string
}{
{
dir: "../../registry/testdata/validPackages/etcd/0.6.1",
fail: false,
bundleChannels: []string{"alpha", "beta", "stable"},
defaultChannel: "alpha",
PackageName: "etcd",
},
{
dir: "../../registry/testdata/validPackages/etcd/0.9.0",
fail: false,
bundleChannels: []string{"alpha", "beta", "stable"},
defaultChannel: "alpha",
PackageName: "etcd",
},
{
dir: "../../registry/testdata/validPackages/etcd/0.9.2",
fail: false,
bundleChannels: []string{"alpha", "stable"},
defaultChannel: "alpha",
PackageName: "etcd",
},
{
dir: "../../registry/testdata/validPackages/prometheus/0.14.0",
fail: false,
bundleChannels: []string{"preview"},
defaultChannel: "preview",
PackageName: "prometheus",
},
{
dir: "../../registry/testdata/invalidPackges/3scale-community-operator/0.3.0",
fail: true,
},
}

for _, tt := range tests {
t.Run("Loading Package Graph from "+tt.dir, func(t *testing.T) {
bundle, err := NewBundleDirInterperter(tt.dir)
if tt.fail {
assert.Error(t, err)
return
}
require.NoError(t, err)

assert.EqualValues(t, tt.bundleChannels, bundle.GetBundleChannels())

assert.EqualValues(t, tt.defaultChannel, bundle.GetDefaultChannel())

assert.EqualValues(t, tt.PackageName, bundle.GetPackageName())
})
}
}
53 changes: 53 additions & 0 deletions pkg/registry/csv.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,18 @@ package registry

import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path"

v1 "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/yaml"

"github.com/operator-framework/api/pkg/operators"
)

const (
Expand Down Expand Up @@ -53,6 +62,50 @@ type ClusterServiceVersion struct {
Spec json.RawMessage `json:"spec"`
}

// ReadCSVFromBundleDirectory tries to parse every YAML file in the directory and see if they are CSV.
// According to the strict one CSV rule for every bundle, we return the first file that is considered a CSV type.
func ReadCSVFromBundleDirectory(bundleDir string) (*ClusterServiceVersion, error) {
dirContent, err := ioutil.ReadDir(bundleDir)
if err != nil {
return nil, fmt.Errorf("error reading bundle directory %s, %v", bundleDir, err)
}

files := []string{}
for _, f := range dirContent {
if !f.IsDir() {
files = append(files, f.Name())
}
}

for _, file := range files {
yamlReader, err := os.Open(path.Join(bundleDir, file))
if err != nil {
continue
}

unstructuredCSV := unstructured.Unstructured{}
csv := ClusterServiceVersion{}

decoder := yaml.NewYAMLOrJSONDecoder(yamlReader, 30)
if err = decoder.Decode(&unstructuredCSV); err != nil {
continue
}

if unstructuredCSV.GetKind() != operators.ClusterServiceVersionKind {
continue
}

if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredCSV.UnstructuredContent(),
&csv); err != nil {
return nil, err
}

return &csv, nil
}
return nil, fmt.Errorf("no ClusterServiceVersion object found in %s", bundleDir)

}

// GetReplaces returns the name of the older ClusterServiceVersion object that
// is replaced by this ClusterServiceVersion object.
//
Expand Down

0 comments on commit d5f8836

Please sign in to comment.