Skip to content

Commit

Permalink
feat(bbolt): serialization for capabilities/requirements
Browse files Browse the repository at this point in the history
  • Loading branch information
@ecordell
ecordell committed Feb 4, 2020
1 parent 4b282d3 commit 2b01c8f
Show file tree
Hide file tree
Showing 8 changed files with 289 additions and 144 deletions.
37 changes: 19 additions & 18 deletions pkg/boltdb/load.go
View file
Expand Up @@ -2,6 +2,7 @@ package boltdb

import (
"fmt"
"github.com/operator-framework/operator-registry/pkg/boltdb/model"

"github.com/asdine/storm/v3"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
Expand Down Expand Up @@ -49,8 +50,8 @@ func (s *StormLoader) AddOperatorBundle(bundle *registry.Bundle) error {
}

for image := range images {
err = tx.Save(&RelatedImage{
ImageUser: ImageUser{
err = tx.Save(&model.RelatedImage{
ImageUser: model.ImageUser{
OperatorBundleName: opBundle.Name,
Image: image,
},
Expand All @@ -75,7 +76,7 @@ func (s *StormLoader) AddPackageChannels(manifest registry.PackageManifest) erro
}
defer tx.Rollback()

pkg := Package{
pkg := model.Package{
Name: manifest.PackageName,
DefaultChannel: manifest.DefaultChannelName,
}
Expand All @@ -86,11 +87,11 @@ func (s *StormLoader) AddPackageChannels(manifest registry.PackageManifest) erro
var errs []error
for _, channel := range manifest.Channels {
// Get and store the update graph starting at the channel head
pkgChannel := PackageChannel{
pkgChannel := model.PackageChannel{
ChannelName: channel.Name,
PackageName: pkg.Name,
}
err = tx.Save(&Channel{
err = tx.Save(&model.Channel{
PackageChannel: pkgChannel,
HeadOperatorBundleName: channel.CurrentCSVName,
})
Expand Down Expand Up @@ -126,13 +127,13 @@ func (s *StormLoader) ClearNonDefaultBundles(packageName string) error {
panic("implement me")
}

func (s *StormLoader) updateGraph(pkgName, channelName, operatorBundleName string) (entries []ChannelEntry, err error) {
var head OperatorBundle
func (s *StormLoader) updateGraph(pkgName, channelName, operatorBundleName string) (entries []model.ChannelEntry, err error) {
var head model.OperatorBundle
if err = s.db.One("Name", operatorBundleName, &head); err != nil {
return
}

pkgChannel := PackageChannel{
pkgChannel := model.PackageChannel{
PackageName: pkgName,
ChannelName: channelName,
}
Expand All @@ -145,8 +146,8 @@ func (s *StormLoader) updateGraph(pkgName, channelName, operatorBundleName strin
return
}

entry := ChannelEntry{
ChannelReplacement: ChannelReplacement{
entry := model.ChannelEntry{
ChannelReplacement: model.ChannelReplacement{
PackageChannel: pkgChannel,
BundleName: o.Name,
Replaces: o.Replaces,
Expand Down Expand Up @@ -189,7 +190,7 @@ func (s *StormLoader) updateGraph(pkgName, channelName, operatorBundleName strin
return
}

func newOperatorBundle(bundle *registry.Bundle) (*OperatorBundle, error) {
func newOperatorBundle(bundle *registry.Bundle) (*model.OperatorBundle, error) {
// Add the core bundle
csvName, bundleImage, csvBytes, bundleBytes, err := bundle.Serialize()
if err != nil {
Expand Down Expand Up @@ -223,24 +224,24 @@ func newOperatorBundle(bundle *registry.Bundle) (*OperatorBundle, error) {
return nil, err
}

capabilities := make([]Capability, 0)
capabilities := make([]model.Capability, 0)
for api := range providedApis {
capabilities = append(capabilities, Capability{
Name: GvkCapability,
capabilities = append(capabilities, model.Capability{
Name: model.GvkCapability,
Value: api.String(),
})
}

requirements := make([]Requirement, 0)
requirements := make([]model.Requirement, 0)
for api := range requiredApis {
requirements = append(requirements, Requirement{
requirements = append(requirements, model.Requirement{
Optional: false,
Name: GvkCapability,
Name: model.GvkCapability,
Selector: api.String(),
})
}

opBundle := &OperatorBundle{
opBundle := &model.OperatorBundle{
Name: csvName,
Version: version,
Replaces: replaces,
Expand Down
106 changes: 50 additions & 56 deletions pkg/boltdb/migrate.go
View file
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"github.com/alicebob/sqlittle"
"github.com/asdine/storm/v3"
"github.com/operator-framework/operator-registry/pkg/boltdb/model"
bolt "go.etcd.io/bbolt"
"os"
)
Expand Down Expand Up @@ -35,9 +36,9 @@ func EnsureBolt(file string, backupFile string) error {

// TODO: check if sqlite file is at latest migration

bdb, err := storm.Open(file)
bdb, err := storm.Open(file, storm.Codec(model.Codec))
if err != nil {
return nil
return err
}

if err := migrateSqliteToBolt(sqlDb, bdb); err != nil {
Expand Down Expand Up @@ -83,10 +84,10 @@ func migrateSqliteToBolt(sqlDb *sqlittle.DB, bdb *storm.DB) error {
return tx.Commit()
}

func migrate(node storm.Node, sqlDb *sqlittle.DB, rowMigrator rowMigrator) error {
func migrate(node storm.Node, sqlDb *sqlittle.DB, rm rowMigrator) error {
// migrate package table
migrateError := make([]error, 0)
migrator, table, columns := rowMigrator(node, migrateError)
migrator, table, columns := rm(node, migrateError)
if err := sqlDb.Select(table, migrator, columns...); err != nil {
return err
}
Expand All @@ -110,7 +111,7 @@ func migratePackageRow(node storm.Node, errs []error) (sqlittle.RowCB, string, [
errs = append(errs, err)
return
}
pkg := Package{
pkg := model.Package{
Name: name,
DefaultChannel: defaultChannel,
}
Expand All @@ -135,8 +136,8 @@ func migrateChannelRow(node storm.Node, errs []error) (sqlittle.RowCB, string, [
return
}
fmt.Printf("migrating channel %s %s %s\n", name, pkgName, headBundleName)
ch := Channel{
PackageChannel: PackageChannel{
ch := model.Channel{
PackageChannel: model.PackageChannel{
ChannelName: name,
PackageName: pkgName,
},
Expand Down Expand Up @@ -169,7 +170,7 @@ func migrateBundleRow(node storm.Node, errs []error) (sqlittle.RowCB, string, []

// TODO: replaces
// TODO: skips
ob := OperatorBundle{
ob := model.OperatorBundle{
Name: name,
Version: version,
//Replaces: ,
Expand Down Expand Up @@ -205,8 +206,8 @@ func migrateRelatedImageRow(node storm.Node, errs []error) (sqlittle.RowCB, stri
errs = append(errs, err)
return
}
relatedImg := RelatedImage{
ImageUser: ImageUser{
relatedImg := model.RelatedImage{
ImageUser: model.ImageUser{
Image: image,
OperatorBundleName: operatorbundle_name,
},
Expand All @@ -223,7 +224,7 @@ func migrateChannelEntries(sqlDb *sqlittle.DB, node storm.Node) error {
table := "channel_entry"

type unpack struct {
ChannelEntry
model.ChannelEntry
replaces int64
}
unpacks := make(map[int64]unpack, 0)
Expand All @@ -241,9 +242,9 @@ func migrateChannelEntries(sqlDb *sqlittle.DB, node storm.Node) error {
return
}
unpacks[entry_id] = unpack{
ChannelEntry: ChannelEntry{
ChannelReplacement: ChannelReplacement{
PackageChannel: PackageChannel{
ChannelEntry: model.ChannelEntry{
ChannelReplacement: model.ChannelReplacement{
PackageChannel: model.PackageChannel{
PackageName: package_name,
ChannelName: channel_name,
},
Expand Down Expand Up @@ -282,7 +283,7 @@ func migrateApiProviders(sqlDb *sqlittle.DB, node storm.Node) error {
table := "api_provider"

type unpack struct {
Capability
model.Capability
channel_entry_id int64
operatorbundle_name string
}
Expand All @@ -300,14 +301,11 @@ func migrateApiProviders(sqlDb *sqlittle.DB, node storm.Node) error {
return
}
unpacks = append(unpacks, unpack{
Capability: Capability{
Name: GvkCapability,
Value: Api{
Group: groupName,
Version: version,
Kind: kind,
}.String(),
},
Capability: model.NewApiCapability(&model.Api{
Group: groupName,
Version: version,
Kind: kind,
}),
channel_entry_id: channel_entry_id,
})
}
Expand Down Expand Up @@ -349,15 +347,15 @@ func migrateApiProviders(sqlDb *sqlittle.DB, node storm.Node) error {
errs = append(errs, err)
return
}
capValue, err := ApiFromString(u.Value)
if err != nil {
errs = append(errs, err)
api, ok := u.Capability.Value.(*model.Api)
if !ok {
errs = append(errs, fmt.Errorf("couldn't parse gvk"))
return
}
if u.Name == GvkCapability && capValue.Group == groupName && capValue.Version == version && capValue.Kind == kind {
capValue.Plural = plural
if u.Name == model.GvkCapability && api.Group == groupName && api.Version == version && api.Kind == kind {
api.Plural = plural
}
u.Value = capValue.String()
u.Value = api
unpacks[i] = u
}, "group_name", "version", "kind", "plural")
if err != nil {
Expand All @@ -370,33 +368,33 @@ func migrateApiProviders(sqlDb *sqlittle.DB, node storm.Node) error {
}

// connect provided apis to their owner operator bundles
bundleCapabilityFilter := make(map[string]map[string]struct{})
bundleCapabilityFilter := make(map[string]map[model.Api]struct{})
for _, u := range unpacks {
// Filter out duplicates
if capabilities, ok := bundleCapabilityFilter[u.operatorbundle_name]; ok {
if _, ok := capabilities[u.Capability.Value]; ok {
if _, ok := capabilities[*u.Capability.Value.(*model.Api)]; ok {
continue
}
} else { // initialize the set
capabilitySet := make(map[string]struct{})
capabilitySet := make(map[model.Api]struct{})
bundleCapabilityFilter[u.operatorbundle_name] = capabilitySet
}

var ob OperatorBundle
var ob model.OperatorBundle
err := node.One("Name", u.operatorbundle_name, &ob)
if err != nil {
return err
}
if ob.Capabilities == nil {
ob.Capabilities = make([]Capability, 0)
ob.Capabilities = make([]model.Capability, 0)
}
ob.Capabilities = append(ob.Capabilities, u.Capability)
if err := node.Save(&ob); err != nil {
return err
}

// add unique item to filter once it's added to the set
bundleCapabilityFilter[u.operatorbundle_name][u.Capability.Value] = struct{}{}
bundleCapabilityFilter[u.operatorbundle_name][*u.Capability.Value.(*model.Api)] = struct{}{}
}

return nil
Expand All @@ -407,7 +405,7 @@ func migrateApiRequirers(sqlDb *sqlittle.DB, node storm.Node) error {
table := "api_requirer"

type unpack struct {
Requirement
model.Requirement
channel_entry_id int64
operatorbundle_name string
}
Expand All @@ -425,15 +423,11 @@ func migrateApiRequirers(sqlDb *sqlittle.DB, node storm.Node) error {
return
}
unpacks = append(unpacks, unpack{
Requirement: Requirement{
Name: GvkCapability,
Selector: Api{
Group: groupName,
Version: version,
Kind: kind,
}.String(),
Optional: false,
},
Requirement: model.NewApiEqualityRequirement(&model.ApiEqualitySelector{
Group: groupName,
Version: version,
Kind: kind,
}),
channel_entry_id: channel_entry_id,
})
}
Expand Down Expand Up @@ -475,16 +469,16 @@ func migrateApiRequirers(sqlDb *sqlittle.DB, node storm.Node) error {
errs = append(errs, err)
return
}
if u.Name == GvkCapability {
reqSelector, err := ApiFromString(u.Selector)
if err != nil {
errs = append(errs, err)
if u.Name == model.GvkCapability {
reqSelector, ok := u.Selector.(*model.ApiEqualitySelector)
if !ok {
errs = append(errs, fmt.Errorf("couldn't parse gvk selector"))
return
}
if reqSelector.Group == groupName && reqSelector.Version == version && reqSelector.Kind == kind {
reqSelector.Plural = plural
}
u.Selector = reqSelector.String()
u.Selector = reqSelector
unpacks[i] = u
} else {
err := fmt.Errorf("Unsupported requirement type: %s", u.Name)
Expand All @@ -502,33 +496,33 @@ func migrateApiRequirers(sqlDb *sqlittle.DB, node storm.Node) error {
}

// connect required apis to their owner operator bundles
bundleRequirementFilter := make(map[string]map[string]struct{})
bundleRequirementFilter := make(map[string]map[model.ApiEqualitySelector]struct{})
for _, u := range unpacks {
// Filter out duplicates
if capabilities, ok := bundleRequirementFilter[u.operatorbundle_name]; ok {
if _, ok := capabilities[u.Requirement.Selector]; ok {
if _, ok := capabilities[*u.Requirement.Selector.(*model.ApiEqualitySelector)]; ok {
continue
}
} else { // initialize the set
capabilitySet := make(map[string]struct{})
capabilitySet := make(map[model.ApiEqualitySelector]struct{})
bundleRequirementFilter[u.operatorbundle_name] = capabilitySet
}

var ob OperatorBundle
var ob model.OperatorBundle
err := node.One("Name", u.operatorbundle_name, &ob)
if err != nil {
return err
}
if ob.Requirements == nil {
ob.Requirements = make([]Requirement, 0)
ob.Requirements = make([]model.Requirement, 0)
}
ob.Requirements = append(ob.Requirements, u.Requirement)
if err := node.Save(&ob); err != nil {
return err
}

// add unique item to filter once it's added to the set
bundleRequirementFilter[u.operatorbundle_name][u.Requirement.Selector] = struct{}{}
bundleRequirementFilter[u.operatorbundle_name][*u.Requirement.Selector.(*model.ApiEqualitySelector)] = struct{}{}
}

return nil
Expand Down

0 comments on commit 2b01c8f

Please sign in to comment.